From 8db3763b106f773101f1745096ce6248aaa25190 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 10 Sep 2024 14:08:41 +0000 Subject: [PATCH 001/124] v2 version for the icrc ledger --- rs/rosetta-api/icrc1/BUILD.bazel | 2 -- rs/rosetta-api/icrc1/ledger/BUILD.bazel | 21 ------------------ rs/rosetta-api/icrc1/ledger/Cargo.toml | 1 - .../icrc1/ledger/sm-tests/src/lib.rs | 13 ----------- rs/rosetta-api/icrc1/ledger/src/main.rs | 22 ------------------- rs/rosetta-api/icrc1/ledger/tests/tests.rs | 14 +++--------- 6 files changed, 3 insertions(+), 70 deletions(-) diff --git a/rs/rosetta-api/icrc1/BUILD.bazel b/rs/rosetta-api/icrc1/BUILD.bazel index c1bfebd61e1..2498a54a294 100644 --- a/rs/rosetta-api/icrc1/BUILD.bazel +++ b/rs/rosetta-api/icrc1/BUILD.bazel @@ -175,7 +175,6 @@ rust_test( data = [ "//rs/rosetta-api/icrc1/index-ng:index_ng_canister" + name_suffix + ".wasm.gz", "//rs/rosetta-api/icrc1/ledger:ledger_canister" + name_suffix + ".wasm", - "//rs/rosetta-api/icrc1/ledger:ledger_canister" + name_suffix + "_nextmigrationversionmemorymanager.wasm", "@" + mainnet_ledger + "//file", "@" + mainnet_index + "//file", ], @@ -185,7 +184,6 @@ rust_test( "IC_ICRC1_INDEX_NG_WASM_PATH": "$(rootpath //rs/rosetta-api/icrc1/index-ng:index_ng_canister" + name_suffix + ".wasm.gz)", "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @" + mainnet_ledger + "//file)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath //rs/rosetta-api/icrc1/ledger:ledger_canister" + name_suffix + ".wasm)", - "IC_ICRC1_LEDGER_MEM_MGR_WASM_PATH": "$(rootpath //rs/rosetta-api/icrc1/ledger:ledger_canister" + name_suffix + "_nextmigrationversionmemorymanager.wasm)", }, deps = [ # Keep sorted. diff --git a/rs/rosetta-api/icrc1/ledger/BUILD.bazel b/rs/rosetta-api/icrc1/ledger/BUILD.bazel index 142fb88132c..fa08a641acd 100644 --- a/rs/rosetta-api/icrc1/ledger/BUILD.bazel +++ b/rs/rosetta-api/icrc1/ledger/BUILD.bazel @@ -156,25 +156,6 @@ package(default_visibility = ["//visibility:public"]) "//rs/rosetta-api/icrc1/tokens_u256", ], ), - ( - "_nextmigrationversionmemorymanager", - ["next-migration-version-memory-manager"], - [ - ":ledger", - "//rs/rosetta-api/icrc1/tokens_u64", - ], - ), - ( - "_u256_nextmigrationversionmemorymanager", - [ - "next-migration-version-memory-manager", - "u256-tokens", - ], - [ - ":ledger_u256", - "//rs/rosetta-api/icrc1/tokens_u256", - ], - ), ] ] @@ -198,7 +179,6 @@ rust_test( data = [ ":block.cddl", ":ledger_canister" + name_suffix + ".wasm", - ":ledger_canister" + name_suffix + "_nextmigrationversionmemorymanager.wasm", "//rs/rosetta-api/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz", "@ic-icrc1-ledger-first-version.wasm.gz//file", "@mainnet_ckbtc_ic-icrc1-ledger//file", @@ -213,7 +193,6 @@ rust_test( "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)", "IC_ICRC1_LEDGER_FIRST_VERSION_WASM_PATH": "$(rootpath @ic-icrc1-ledger-first-version.wasm.gz//file)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath :ledger_canister" + name_suffix + ".wasm)", - "IC_ICRC1_LEDGER_MEM_MGR_WASM_PATH": "$(rootpath :ledger_canister" + name_suffix + "_nextmigrationversionmemorymanager.wasm)", }, deps = [ # Keep sorted. diff --git a/rs/rosetta-api/icrc1/ledger/Cargo.toml b/rs/rosetta-api/icrc1/ledger/Cargo.toml index ac748641442..32fa8be5bd0 100644 --- a/rs/rosetta-api/icrc1/ledger/Cargo.toml +++ b/rs/rosetta-api/icrc1/ledger/Cargo.toml @@ -55,4 +55,3 @@ default = [] get-blocks-disabled = [] u256-tokens = ["dep:ic-icrc1-tokens-u256"] canbench-rs = ["dep:canbench-rs", "dep:assert_matches"] -next-migration-version-memory-manager = [] diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index 458aec32cfd..08a74c02cb1 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2591,7 +2591,6 @@ pub fn test_upgrade_serialization( pub fn icrc1_test_upgrade_serialization_fixed_tx( ledger_wasm_mainnet: Vec, ledger_wasm_current: Vec, - ledger_wasm_nextmigrationversionmemorymanager: Vec, encode_init_args: fn(InitArgs) -> T, ) where T: CandidType, @@ -2685,18 +2684,6 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( test_upgrade(ledger_wasm_current.clone(), balances.clone()); // Test the new wasm serialization test_upgrade(ledger_wasm_current.clone(), balances.clone()); - // Test serializing to the memory manager - test_upgrade( - ledger_wasm_nextmigrationversionmemorymanager.clone(), - balances.clone(), - ); - // Test upgrade to memory manager again - test_upgrade( - ledger_wasm_nextmigrationversionmemorymanager, - balances.clone(), - ); - // Test deserializing from memory manager - test_upgrade(ledger_wasm_current, balances.clone()); // Add some more approvals for a1 in &accounts { diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 851e52b0fa7..d62c6f13c73 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -6,8 +6,6 @@ use candid::types::number::Nat; use ic_canister_log::{declare_log_buffer, export}; use ic_canisters_http_types::{HttpRequest, HttpResponse, HttpResponseBuilder}; use ic_cdk::api::stable::StableReader; -#[cfg(not(feature = "next-migration-version-memory-manager"))] -use ic_cdk::api::stable::StableWriter; #[cfg(not(feature = "canbench-rs"))] use ic_cdk_macros::init; @@ -27,7 +25,6 @@ use ic_ledger_core::block::BlockIndex; use ic_ledger_core::timestamp::TimeStamp; use ic_ledger_core::tokens::Zero; use ic_stable_structures::reader::{BufferedReader, Reader}; -#[cfg(feature = "next-migration-version-memory-manager")] use ic_stable_structures::writer::{BufferedWriter, Writer}; use icrc_ledger_types::icrc2::approve::{ApproveArgs, ApproveError}; use icrc_ledger_types::icrc21::{ @@ -124,28 +121,9 @@ fn init_state(init_args: InitArgs) { }) } -#[cfg(not(feature = "next-migration-version-memory-manager"))] -#[pre_upgrade] -fn pre_upgrade() { - #[cfg(feature = "canbench-rs")] - let _p = canbench_rs::bench_scope("pre_upgrade"); - - let start = ic_cdk::api::instruction_counter(); - let mut stable_writer = StableWriter::default(); - Access::with_ledger(|ledger| ciborium::ser::into_writer(ledger, &mut stable_writer)) - .expect("failed to encode ledger state"); - let end = ic_cdk::api::instruction_counter(); - let instructions_consumed = end - start; - let counter_bytes: [u8; 8] = instructions_consumed.to_le_bytes(); - stable_writer - .write_all(&counter_bytes) - .expect("failed to write instructions consumed to stable memory"); -} - // We use 8MiB buffer const BUFFER_SIZE: usize = 8388608; -#[cfg(feature = "next-migration-version-memory-manager")] #[pre_upgrade] fn pre_upgrade() { #[cfg(feature = "canbench-rs")] diff --git a/rs/rosetta-api/icrc1/ledger/tests/tests.rs b/rs/rosetta-api/icrc1/ledger/tests/tests.rs index 1bab3531e1e..7dcbbcefd06 100644 --- a/rs/rosetta-api/icrc1/ledger/tests/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/tests/tests.rs @@ -88,10 +88,6 @@ fn ledger_wasm() -> Vec { ) } -fn ledger_wasm_nextmigrationversionmemorymanager() -> Vec { - std::fs::read(std::env::var("IC_ICRC1_LEDGER_MEM_MGR_WASM_PATH").unwrap()).unwrap() -} - fn archive_wasm() -> Vec { ic_test_utilities_load_wasm::load_wasm( PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) @@ -402,7 +398,7 @@ fn icrc1_test_upgrade_serialization() { ic_icrc1_ledger_sm_tests::test_upgrade_serialization( ledger_mainnet_wasm(), ledger_wasm(), - Some(ledger_wasm_nextmigrationversionmemorymanager()), + None, init_args, upgrade_args, minter, @@ -415,16 +411,12 @@ fn icrc1_test_upgrade_serialization_fixed_tx() { ic_icrc1_ledger_sm_tests::icrc1_test_upgrade_serialization_fixed_tx( ledger_mainnet_wasm(), ledger_wasm(), - ledger_wasm_nextmigrationversionmemorymanager(), encode_init_args, ); } mod metrics { - use crate::{ - encode_init_args, encode_upgrade_args, ledger_wasm, - ledger_wasm_nextmigrationversionmemorymanager, - }; + use crate::{encode_init_args, encode_upgrade_args, ledger_wasm}; use ic_icrc1_ledger_sm_tests::metrics::LedgerSuiteType; #[test] @@ -456,7 +448,7 @@ mod metrics { fn should_set_ledger_upgrade_instructions_consumed_metric() { ic_icrc1_ledger_sm_tests::metrics::assert_ledger_upgrade_instructions_consumed_metric_set( ledger_wasm(), - Some(ledger_wasm_nextmigrationversionmemorymanager()), + None, encode_init_args, encode_upgrade_args, ); From 619a887339144299196d7fb95a3c21b40c9aa7e7 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 11 Sep 2024 12:39:30 +0000 Subject: [PATCH 002/124] update mainnet canisters to v1 --- mainnet-canisters.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mainnet-canisters.bzl b/mainnet-canisters.bzl index 206cecfc5d9..b17387c741b 100644 --- a/mainnet-canisters.bzl +++ b/mainnet-canisters.bzl @@ -23,9 +23,9 @@ CANISTER_NAME_TO_WASM_METADATA = { "sns_ledger": ("3d0b3f10417fc6708e8b5d844a0bac5e86f3e17d", "e8942f56f9439b89b13bd8037f357126e24f1e7932cf03018243347505959fd4"), "sns_archive": ("3d0b3f10417fc6708e8b5d844a0bac5e86f3e17d", "5c595c2adc7f6d9971298fee2fa666929711e73341192ab70804c783a0eee03f"), "ck_btc_index": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "cac207cf438df8c9fba46d4445c097f05fd8228a1eeacfe0536b7e9ddefc5f1c"), - "ck_btc_ledger": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "4264ce2952c4e9ff802d81a11519d5e3ffdaed4215d5831a6634e59efd72f7d8"), + "ck_btc_ledger": ("d323465e02b84a0bc3b8c2c6fd362f6072f1a3f2", "cb5fc1cd94cb75791e8c01be3116122d779ef61c6eab41f10b60f6e79fe9f0e9"), "ck_eth_index": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "8104acad6105abb069b2dbc8289692bd63c2d110127f8e91f99db51465962606"), - "ck_eth_ledger": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "e5c8a297d1c0c6d2ab2253c0280aaefd6e23fe3a8a994fc64706a1f3c3116062"), + "ck_eth_ledger": ("d323465e02b84a0bc3b8c2c6fd362f6072f1a3f2", "e61b0c6fcf598ee1e7551cc9ee10869f6dccf685bf9af714476baefd240f9978"), } def canister_url(git_commit_id, filename): From 7f1a685ebb619565e2039b1bd016f6189dfdec1e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 13 Sep 2024 12:10:49 +0000 Subject: [PATCH 003/124] update mainnet canister hash --- mainnet-canisters.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mainnet-canisters.bzl b/mainnet-canisters.bzl index ac6abee919a..b92c745798d 100644 --- a/mainnet-canisters.bzl +++ b/mainnet-canisters.bzl @@ -23,9 +23,9 @@ CANISTER_NAME_TO_WASM_METADATA = { "sns_ledger": ("3d0b3f10417fc6708e8b5d844a0bac5e86f3e17d", "e8942f56f9439b89b13bd8037f357126e24f1e7932cf03018243347505959fd4"), "sns_archive": ("3d0b3f10417fc6708e8b5d844a0bac5e86f3e17d", "5c595c2adc7f6d9971298fee2fa666929711e73341192ab70804c783a0eee03f"), "ck_btc_index": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "cac207cf438df8c9fba46d4445c097f05fd8228a1eeacfe0536b7e9ddefc5f1c"), - "ck_btc_ledger": ("d323465e02b84a0bc3b8c2c6fd362f6072f1a3f2", "cb5fc1cd94cb75791e8c01be3116122d779ef61c6eab41f10b60f6e79fe9f0e9"), + "ck_btc_ledger": ("643e4fb30cdb47cc904c0a5b9e5c97f9179c8104", "5d1b5bef09f5679d01de2682074fdd5b0f5f1382358a9edf9f84b6f6c648dce1"), "ck_eth_index": ("a3831c87440df4821b435050c8a8fcb3745d86f6", "8104acad6105abb069b2dbc8289692bd63c2d110127f8e91f99db51465962606"), - "ck_eth_ledger": ("d323465e02b84a0bc3b8c2c6fd362f6072f1a3f2", "e61b0c6fcf598ee1e7551cc9ee10869f6dccf685bf9af714476baefd240f9978"), + "ck_eth_ledger": ("643e4fb30cdb47cc904c0a5b9e5c97f9179c8104", "b40e42eed058e35fe67ae0ade549d6909258d05b9350adb37e03f403202cf7e2"), } def canister_url(git_commit_id, filename): From 702e5d8a34994c88d94f44448b2f73e220bdef95 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 13 Sep 2024 13:38:28 +0000 Subject: [PATCH 004/124] remove extra clone --- rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index 020429d7d3f..d9713a3e5a4 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2683,7 +2683,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( // Test if the old serialized approvals and balances are correctly deserialized test_upgrade(ledger_wasm_current.clone(), balances.clone()); // Test the new wasm serialization - test_upgrade(ledger_wasm_current.clone(), balances.clone()); + test_upgrade(ledger_wasm_current, balances.clone()); // Add some more approvals for a1 in &accounts { From d89d15df0e83efdcdce5638f7f8bb02aa28c5ac2 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 16 Sep 2024 13:58:38 +0000 Subject: [PATCH 005/124] approvals in stable memory --- packages/icrc-ledger-types/BUILD.bazel | 1 + packages/icrc-ledger-types/Cargo.toml | 3 +- .../icrc-ledger-types/src/icrc1/account.rs | 50 +++++++ rs/rosetta-api/icrc1/ledger/BUILD.bazel | 20 ++- rs/rosetta-api/icrc1/ledger/src/lib.rs | 139 ++++++++++++++++-- rs/rosetta-api/icrc1/ledger/src/main.rs | 10 +- rs/rosetta-api/ledger_core/BUILD.bazel | 1 + rs/rosetta-api/ledger_core/Cargo.toml | 1 + rs/rosetta-api/ledger_core/src/approvals.rs | 47 ++++++ rs/rosetta-api/ledger_core/src/timestamp.rs | 19 +++ 10 files changed, 269 insertions(+), 22 deletions(-) diff --git a/packages/icrc-ledger-types/BUILD.bazel b/packages/icrc-ledger-types/BUILD.bazel index 1e3f8029019..32bc2a991bb 100644 --- a/packages/icrc-ledger-types/BUILD.bazel +++ b/packages/icrc-ledger-types/BUILD.bazel @@ -19,6 +19,7 @@ rust_library( "@crate_index//:crc32fast", "@crate_index//:hex", "@crate_index//:ic-cdk", + "@crate_index//:ic-stable-structures", "@crate_index//:itertools", "@crate_index//:num-bigint", "@crate_index//:num-traits", diff --git a/packages/icrc-ledger-types/Cargo.toml b/packages/icrc-ledger-types/Cargo.toml index 9d8497a2f28..9c4254925ea 100644 --- a/packages/icrc-ledger-types/Cargo.toml +++ b/packages/icrc-ledger-types/Cargo.toml @@ -15,13 +15,14 @@ base32 = "0.4.0" candid = { workspace = true } crc32fast = "1.2.0" hex = { workspace = true } +ic-stable-structures = { workspace = true } itertools = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } sha2 = { workspace = true } -strum = { workspace = true} +strum = { workspace = true } time = { workspace = true } [dev-dependencies] diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 32f23b5838b..9f0649f7fdc 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -6,7 +6,10 @@ use std::{ use base32::Alphabet; use candid::{types::principal::PrincipalError, CandidType, Deserialize, Principal}; +use ic_stable_structures::{storable::Bound, Storable}; use serde::Serialize; +use std::borrow::Cow; +use std::io::{Cursor, Read}; pub type Subaccount = [u8; 32]; @@ -167,6 +170,53 @@ impl FromStr for Account { } } +impl Storable for Account { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buffer: Vec = vec![]; + let mut buffer0: Vec = vec![]; + + if let Some(subaccount) = self.subaccount { + buffer0.extend(subaccount.as_slice()); + } + buffer0.extend(self.owner.as_slice()); + buffer.extend((buffer0.len() as u8).to_le_bytes()); + buffer.append(&mut buffer0); + + Cow::Owned(buffer) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + let mut cursor = Cursor::new(bytes); + + let mut len_bytes = [0u8; 1]; + cursor + .read_exact(&mut len_bytes) + .expect("Unable to read the len of the account"); + let mut len = u8::from_le_bytes(len_bytes); + let subaccount = if len >= 32 { + let mut subaccount_bytes = [0u8; 32]; + cursor + .read_exact(&mut subaccount_bytes) + .expect("Unable to read the bytes of the account's subaccount"); + len -= 32; + Some(subaccount_bytes) + } else { + None + }; + let mut owner_bytes = vec![0; len as usize]; + cursor + .read_exact(&mut owner_bytes) + .expect("Unable to read the bytes of the account's owners"); + let owner = Principal::from_slice(&owner_bytes); + Account { owner, subaccount } + } + + const BOUND: Bound = Bound::Bounded { + max_size: 62, + is_fixed_size: false, + }; +} + #[cfg(test)] mod tests { use assert_matches::assert_matches; diff --git a/rs/rosetta-api/icrc1/ledger/BUILD.bazel b/rs/rosetta-api/icrc1/ledger/BUILD.bazel index fa08a641acd..b833bbd4a2f 100644 --- a/rs/rosetta-api/icrc1/ledger/BUILD.bazel +++ b/rs/rosetta-api/icrc1/ledger/BUILD.bazel @@ -43,13 +43,28 @@ package(default_visibility = ["//visibility:public"]) ] + extra_deps, ) for (name_suffix, archive_name_suffix, features, extra_deps) in [ - ("", "", [], []), - ("_u256", "_u256", [], []), + ( + "", + "", + [], + [ + "//rs/rosetta-api/icrc1/tokens_u64", + ], + ), + ( + "_u256", + "_u256", + ["u256-tokens"], + [ + "//rs/rosetta-api/icrc1/tokens_u256", + ], + ), ( "_canbench", "", ["canbench-rs"], [ + "//rs/rosetta-api/icrc1/tokens_u64", "@crate_index//:canbench-rs", ], ), @@ -61,6 +76,7 @@ package(default_visibility = ["//visibility:public"]) "u256-tokens", ], [ + "//rs/rosetta-api/icrc1/tokens_u256", "@crate_index//:canbench-rs", ], ), diff --git a/rs/rosetta-api/icrc1/ledger/src/lib.rs b/rs/rosetta-api/icrc1/ledger/src/lib.rs index 43e61a4b909..ce1c286f35c 100644 --- a/rs/rosetta-api/icrc1/ledger/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/src/lib.rs @@ -23,15 +23,14 @@ use ic_ledger_canister_core::{ range_utils, }; use ic_ledger_core::{ - approvals::{AllowanceTable, HeapAllowancesData}, + approvals::{Allowance, AllowanceTable, AllowancesData}, balances::Balances, block::{BlockIndex, BlockType, EncodedBlock, FeeCollector}, timestamp::TimeStamp, - tokens::TokensType, }; use ic_ledger_hash_of::HashOf; use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory}; -use ic_stable_structures::DefaultMemoryImpl; +use ic_stable_structures::{DefaultMemoryImpl, StableBTreeMap}; use icrc_ledger_types::icrc3::transactions::Transaction as Tx; use icrc_ledger_types::icrc3::{blocks::GetBlocksResponse, transactions::GetTransactionsResponse}; use icrc_ledger_types::{ @@ -65,6 +64,12 @@ const MAX_TRANSACTIONS_TO_PURGE: usize = 100_000; const DEFAULT_MAX_MEMO_LENGTH: u16 = 32; +#[cfg(not(feature = "u256-tokens"))] +pub type Tokens = ic_icrc1_tokens_u64::U64; + +#[cfg(feature = "u256-tokens")] +pub type Tokens = ic_icrc1_tokens_u256::U256; + #[derive(Clone, Debug)] pub struct Icrc1ArchiveWasm; @@ -330,6 +335,8 @@ pub enum LedgerArgument { } const UPGRADES_MEMORY_ID: MemoryId = MemoryId::new(0); +const ALLOWANCES_MEMORY_ID: MemoryId = MemoryId::new(1); +const ALLOWANCES_EXPIRATIONS_MEMORY_ID: MemoryId = MemoryId::new(2); thread_local! { static MEMORY_MANAGER: RefCell> = RefCell::new( @@ -339,14 +346,24 @@ thread_local! { // The memory where the ledger must write and read its state during an upgrade. pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); + + // (from, spender) -> allowance - map storing ledger allowances. + pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = + MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_MEMORY_ID)))); + + // (timestamp, (from, spender)) - expiration set used for removing expired allowances. + pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = + MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } #[derive(Debug, Deserialize, Serialize)] #[serde(bound = "")] -pub struct Ledger { +pub struct Ledger { balances: LedgerBalances, #[serde(default)] approvals: LedgerAllowances, + #[serde(default)] + stable_approvals: AllowanceTable, blockchain: Blockchain, minting_account: Account, @@ -407,7 +424,7 @@ fn default_decimals() -> u8 { ic_ledger_core::tokens::DECIMAL_PLACES as u8 } -impl Ledger { +impl Ledger { pub fn from_init_args( sink: impl Sink + Clone, InitArgs { @@ -436,6 +453,7 @@ impl Ledger { let mut ledger = Self { balances: LedgerBalances::default(), approvals: Default::default(), + stable_approvals: Default::default(), blockchain: Blockchain::new_with_archive(archive_options), transactions_by_hash: BTreeMap::new(), transactions_by_height: VecDeque::new(), @@ -474,7 +492,7 @@ impl Ledger { ) }); let mint = Transaction::mint(account, amount, Some(now), None); - apply_transaction(&mut ledger, mint, now, Tokens::zero()).unwrap_or_else(|err| { + apply_transaction(&mut ledger, mint, now, Tokens::ZERO).unwrap_or_else(|err| { panic!( "failed to mint {} tokens to {}: {:?}", balance, account, err @@ -486,9 +504,9 @@ impl Ledger { } } -impl LedgerContext for Ledger { +impl LedgerContext for Ledger { type AccountId = Account; - type AllowancesData = HeapAllowancesData; + type AllowancesData = StableAllowancesData; type BalancesStore = BTreeMap; type Tokens = Tokens; @@ -501,11 +519,11 @@ impl LedgerContext for Ledger { } fn approvals(&self) -> &AllowanceTable { - &self.approvals + &self.stable_approvals } fn approvals_mut(&mut self) -> &mut AllowanceTable { - &mut self.approvals + &mut self.stable_approvals } fn fee_collector(&self) -> Option<&FeeCollector> { @@ -513,7 +531,7 @@ impl LedgerContext for Ledger { } } -impl LedgerData for Ledger { +impl LedgerData for Ledger { type Runtime = CdkRuntime; type ArchiveWasm = Icrc1ArchiveWasm; type Transaction = Transaction; @@ -578,7 +596,7 @@ impl LedgerData for Ledger { } } -impl Ledger { +impl Ledger { pub fn minting_account(&self) -> &Account { &self.minting_account } @@ -605,7 +623,8 @@ impl Ledger { records.push(Value::entry("icrc1:decimals", self.decimals() as u64)); records.push(Value::entry("icrc1:name", self.token_name())); records.push(Value::entry("icrc1:symbol", self.token_symbol())); - records.push(Value::entry("icrc1:fee", self.transfer_fee().into())); + let nat_fee: Nat = self.transfer_fee().into(); + records.push(Value::entry("icrc1:fee", nat_fee)); records.push(Value::entry( "icrc1:max_memo_length", self.max_memo_length() as u64, @@ -864,3 +883,97 @@ impl Ledger { } } } + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct StableAllowancesData {} + +impl AllowancesData for StableAllowancesData { + type AccountId = Account; + type Tokens = Tokens; + + fn get_allowance( + &self, + account_spender: &(Self::AccountId, Self::AccountId), + ) -> Option> { + ALLOWANCES_MEMORY.with_borrow(|allowances| allowances.get(account_spender)) + } + + fn set_allowance( + &mut self, + account_spender: (Self::AccountId, Self::AccountId), + allowance: Allowance, + ) { + ALLOWANCES_MEMORY + .with_borrow_mut(|allowances| allowances.insert(account_spender, allowance)); + } + + fn remove_allowance(&mut self, account_spender: &(Self::AccountId, Self::AccountId)) { + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| allowances.remove(account_spender)); + } + + fn insert_expiry( + &mut self, + timestamp: TimeStamp, + account_spender: (Self::AccountId, Self::AccountId), + ) { + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.insert((timestamp, account_spender), ()); + }); + } + + fn remove_expiry( + &mut self, + timestamp: TimeStamp, + account_spender: (Self::AccountId, Self::AccountId), + ) { + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.remove(&(timestamp, account_spender)); + }); + } + + fn insert_arrival( + &mut self, + _timestamp: TimeStamp, + _account_spender: (Self::AccountId, Self::AccountId), + ) { + } + + fn remove_arrival( + &mut self, + _timestamp: TimeStamp, + _account_spender: (Self::AccountId, Self::AccountId), + ) { + } + + fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow(|expirations| expirations.first_key_value().map(|kv| kv.0)) + } + + fn oldest_arrivals(&self, _n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { + vec![] + } + + fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)) + } + + fn len_allowances(&self) -> usize { + ALLOWANCES_MEMORY + .with_borrow(|allowances| allowances.len()) + .try_into() + .unwrap() + } + + fn len_expirations(&self) -> usize { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow(|expirations| expirations.len()) + .try_into() + .unwrap() + } + + fn len_arrivals(&self) -> usize { + 0 + } +} diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 07df5fbb1c2..36e793d5697 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -71,7 +71,7 @@ pub type Tokens = ic_icrc1_tokens_u64::U64; pub type Tokens = ic_icrc1_tokens_u256::U256; thread_local! { - static LEDGER: RefCell>> = const { RefCell::new(None) }; + static LEDGER: RefCell> = const { RefCell::new(None) }; static PRE_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; static POST_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; } @@ -80,7 +80,7 @@ declare_log_buffer!(name = LOG, capacity = 1000); struct Access; impl LedgerAccess for Access { - type Ledger = Ledger; + type Ledger = Ledger; fn with_ledger(f: impl FnOnce(&Self::Ledger) -> R) -> R { LEDGER.with(|cell| { @@ -116,9 +116,7 @@ fn init(args: LedgerArgument) { fn init_state(init_args: InitArgs) { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); - LEDGER.with(|cell| { - *cell.borrow_mut() = Some(Ledger::::from_init_args(&LOG, init_args, now)) - }) + LEDGER.with(|cell| *cell.borrow_mut() = Some(Ledger::from_init_args(&LOG, init_args, now))) } // We use 8MiB buffer @@ -181,7 +179,7 @@ fn post_upgrade(args: Option) { } }; } else { - let state: Ledger = UPGRADES_MEMORY.with_borrow(|bs| { + let state: Ledger = UPGRADES_MEMORY.with_borrow(|bs| { let reader = Reader::new(bs, 0); let mut buffered_reader = BufferedReader::new(BUFFER_SIZE, reader); let state = ciborium::de::from_reader(&mut buffered_reader).expect( diff --git a/rs/rosetta-api/ledger_core/BUILD.bazel b/rs/rosetta-api/ledger_core/BUILD.bazel index 797c654a270..94f48aa34f0 100644 --- a/rs/rosetta-api/ledger_core/BUILD.bazel +++ b/rs/rosetta-api/ledger_core/BUILD.bazel @@ -11,6 +11,7 @@ rust_library( # Keep sorted. "//packages/ic-ledger-hash-of:ic_ledger_hash_of", "@crate_index//:candid", + "@crate_index//:ic-stable-structures", "@crate_index//:num-traits", "@crate_index//:serde", "@crate_index//:serde_bytes", diff --git a/rs/rosetta-api/ledger_core/Cargo.toml b/rs/rosetta-api/ledger_core/Cargo.toml index 408746dd290..cace33c17b8 100644 --- a/rs/rosetta-api/ledger_core/Cargo.toml +++ b/rs/rosetta-api/ledger_core/Cargo.toml @@ -9,6 +9,7 @@ documentation.workspace = true [dependencies] candid = { workspace = true } ic-ledger-hash-of = { path = "../../../packages/ic-ledger-hash-of" } +ic-stable-structures = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index 5939d2a791e..b5c9ff4672d 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -1,7 +1,13 @@ use crate::timestamp::TimeStamp; use crate::tokens::{CheckedSub, TokensType, Zero}; +use candid::Nat; +use ic_stable_structures::{storable::Bound, Storable}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; +use std::{ + borrow::Cow, + io::{Cursor, Read}, +}; #[cfg(test)] mod tests; @@ -474,3 +480,44 @@ where fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } + +impl + TryFrom> Storable for Allowance { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buffer = vec![]; + let amount: Nat = self.amount.clone().into(); + amount + .encode(&mut buffer) + .expect("Unable to serialize amount"); + buffer.extend(self.arrived_at.as_nanos_since_unix_epoch().to_le_bytes()); + if let Some(expires_at) = self.expires_at { + buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); + } + Cow::Owned(buffer) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + let mut cursor = Cursor::new(bytes.into_owned()); + let amount = Nat::decode(&mut cursor).expect("Unable to deserialize amount"); + let amount = Tokens::try_from(amount).expect("Unable to deserialize amount"); + let mut arrived_at_bytes = [0u8; 8]; + cursor + .read_exact(&mut arrived_at_bytes) + .expect("Unable to read arrived_at bytes"); + let arrived_at = + TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes(arrived_at_bytes)); + let mut expires_at_bytes = [0u8; 8]; + let expires_at = match cursor.read_exact(&mut expires_at_bytes) { + Ok(()) => Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( + expires_at_bytes, + ))), + _ => None, + }; + Self { + amount, + arrived_at, + expires_at, + } + } + + const BOUND: Bound = Bound::Unbounded; +} diff --git a/rs/rosetta-api/ledger_core/src/timestamp.rs b/rs/rosetta-api/ledger_core/src/timestamp.rs index 55a7d0e41dd..c4bd1fd06d3 100644 --- a/rs/rosetta-api/ledger_core/src/timestamp.rs +++ b/rs/rosetta-api/ledger_core/src/timestamp.rs @@ -1,5 +1,7 @@ use candid::CandidType; +use ic_stable_structures::{storable::Bound, Storable}; use serde::{Deserialize, Serialize}; +use std::borrow::Cow; use std::convert::TryInto; use std::ops::{Add, Sub}; use std::time::{Duration, SystemTime}; @@ -66,3 +68,20 @@ impl Sub for TimeStamp { } } } + +impl Storable for TimeStamp { + fn to_bytes(&self) -> Cow<[u8]> { + Cow::Owned(self.as_nanos_since_unix_epoch().to_le_bytes().to_vec()) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + Self::from_nanos_since_unix_epoch(u64::from_le_bytes( + bytes.into_owned().as_slice().try_into().unwrap(), + )) + } + + const BOUND: Bound = Bound::Bounded { + max_size: 8, + is_fixed_size: true, + }; +} From ce95fccde5c5e1c8b1cd9d3a4e0540e763a518b3 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Mon, 16 Sep 2024 14:02:17 +0000 Subject: [PATCH 006/124] Automatically updated Cargo*.lock --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index fd527de3f68..c3aaf8e196b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9157,6 +9157,7 @@ version = "0.9.0" dependencies = [ "candid", "ic-ledger-hash-of", + "ic-stable-structures", "num-traits", "serde", "serde_bytes", @@ -13527,6 +13528,7 @@ dependencies = [ "candid", "crc32fast", "hex", + "ic-stable-structures", "itertools 0.12.1", "num-bigint 0.4.6", "num-traits", From df25efd9e154cb992eb0560e1aaab0e14386ce72 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 18 Sep 2024 11:50:52 +0000 Subject: [PATCH 007/124] skip arrived_at serialization --- rs/rosetta-api/ledger_core/src/approvals.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index b5c9ff4672d..f66707d5133 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -481,6 +481,8 @@ fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } +// We do not serialize and deserialize the arrived_at value as it is not used anymore +// after the migration to stable structures. impl + TryFrom> Storable for Allowance { fn to_bytes(&self) -> Cow<[u8]> { let mut buffer = vec![]; @@ -488,7 +490,6 @@ impl + TryFrom> Storable for Allo amount .encode(&mut buffer) .expect("Unable to serialize amount"); - buffer.extend(self.arrived_at.as_nanos_since_unix_epoch().to_le_bytes()); if let Some(expires_at) = self.expires_at { buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); } @@ -498,13 +499,8 @@ impl + TryFrom> Storable for Allo fn from_bytes(bytes: Cow<[u8]>) -> Self { let mut cursor = Cursor::new(bytes.into_owned()); let amount = Nat::decode(&mut cursor).expect("Unable to deserialize amount"); - let amount = Tokens::try_from(amount).expect("Unable to deserialize amount"); - let mut arrived_at_bytes = [0u8; 8]; - cursor - .read_exact(&mut arrived_at_bytes) - .expect("Unable to read arrived_at bytes"); - let arrived_at = - TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes(arrived_at_bytes)); + let amount = Tokens::try_from(amount).expect("Unable to convert Nat to Tokens"); + let arrived_at = TimeStamp::from_nanos_since_unix_epoch(0); let mut expires_at_bytes = [0u8; 8]; let expires_at = match cursor.read_exact(&mut expires_at_bytes) { Ok(()) => Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( From b262c9a7ee365fb433290fdedcd0f8e253a83783 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 18 Sep 2024 12:04:39 +0000 Subject: [PATCH 008/124] comment --- rs/rosetta-api/ledger_core/src/approvals.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index f66707d5133..6d8000629c4 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -481,8 +481,6 @@ fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } -// We do not serialize and deserialize the arrived_at value as it is not used anymore -// after the migration to stable structures. impl + TryFrom> Storable for Allowance { fn to_bytes(&self) -> Cow<[u8]> { let mut buffer = vec![]; @@ -493,6 +491,7 @@ impl + TryFrom> Storable for Allo if let Some(expires_at) = self.expires_at { buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); } + // We don't seriazlize arrived_at - it is not used after stable structures migration. Cow::Owned(buffer) } @@ -500,6 +499,7 @@ impl + TryFrom> Storable for Allo let mut cursor = Cursor::new(bytes.into_owned()); let amount = Nat::decode(&mut cursor).expect("Unable to deserialize amount"); let amount = Tokens::try_from(amount).expect("Unable to convert Nat to Tokens"); + // arrived_at was not serialized, use a default value. let arrived_at = TimeStamp::from_nanos_since_unix_epoch(0); let mut expires_at_bytes = [0u8; 8]; let expires_at = match cursor.read_exact(&mut expires_at_bytes) { From 2ba7c8f200c87d80644bdc57449b71adab5c7046 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 18 Sep 2024 14:58:59 +0000 Subject: [PATCH 009/124] migrate approvals --- .../icp_ledger/ledger/tests/tests.rs | 1 + rs/rosetta-api/icrc1/ledger/BUILD.bazel | 1 + rs/rosetta-api/icrc1/ledger/Cargo.toml | 1 + .../icrc1/ledger/sm-tests/src/lib.rs | 19 ++--- rs/rosetta-api/icrc1/ledger/src/lib.rs | 48 ++++++++++- rs/rosetta-api/icrc1/ledger/src/main.rs | 84 ++++++++++++++++++- rs/rosetta-api/icrc1/ledger/tests/tests.rs | 1 + rs/rosetta-api/ledger_core/src/approvals.rs | 18 +++- 8 files changed, 156 insertions(+), 17 deletions(-) diff --git a/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs b/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs index 972a5187d2b..cd986c1bb8e 100644 --- a/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs +++ b/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs @@ -1005,6 +1005,7 @@ fn test_upgrade_serialization() { upgrade_args, minter, false, + true, ); } diff --git a/rs/rosetta-api/icrc1/ledger/BUILD.bazel b/rs/rosetta-api/icrc1/ledger/BUILD.bazel index b833bbd4a2f..d06492e2caf 100644 --- a/rs/rosetta-api/icrc1/ledger/BUILD.bazel +++ b/rs/rosetta-api/icrc1/ledger/BUILD.bazel @@ -107,6 +107,7 @@ package(default_visibility = ["//visibility:public"]) "@crate_index//:candid", "@crate_index//:ciborium", "@crate_index//:ic-cdk", + "@crate_index//:ic-cdk-timers", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", "@crate_index//:num-traits", diff --git a/rs/rosetta-api/icrc1/ledger/Cargo.toml b/rs/rosetta-api/icrc1/ledger/Cargo.toml index 32fa8be5bd0..22eda7d6abf 100644 --- a/rs/rosetta-api/icrc1/ledger/Cargo.toml +++ b/rs/rosetta-api/icrc1/ledger/Cargo.toml @@ -22,6 +22,7 @@ ic-canister-log = { path = "../../../rust_canisters/canister_log" } ic-canisters-http-types = { path = "../../../rust_canisters/http_types" } ic-cdk = { workspace = true } ic-cdk-macros = { workspace = true } +ic-cdk-timers = { workspace = true } ic-crypto-tree-hash = { path = "../../../crypto/tree_hash" } ic-icrc1 = { path = ".." } ic-icrc1-tokens-u256 = { path = "../tokens_u256", optional = true } diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index d9713a3e5a4..e0621a32bc4 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2494,6 +2494,7 @@ pub fn test_upgrade_serialization( upgrade_args: Vec, minter: Arc, verify_blocks: bool, + downgrade_to_mainnet_possible: bool, ) { let mut runner = TestRunner::new(TestRunnerConfig::with_cases(1)); let now = SystemTime::now(); @@ -2573,8 +2574,10 @@ pub fn test_upgrade_serialization( } // Test deserializing from memory manager test_upgrade(ledger_wasm_current.clone()); - // Test downgrade to mainnet wasm - test_upgrade(ledger_wasm_mainnet.clone()); + if downgrade_to_mainnet_possible { + // Test downgrade to mainnet wasm + test_upgrade(ledger_wasm_mainnet.clone()); + } if verify_blocks { // This will also verify the ledger blocks. // The current implementation of the InMemoryLedger cannot get blocks @@ -2621,11 +2624,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } // Setup ledger as it is deployed on the mainnet. - let (env, canister_id) = setup( - ledger_wasm_mainnet.clone(), - encode_init_args, - initial_balances, - ); + let (env, canister_id) = setup(ledger_wasm_mainnet, encode_init_args, initial_balances); const APPROVE_AMOUNT: u64 = 150_000; let expiration = @@ -2682,8 +2681,6 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( // Test if the old serialized approvals and balances are correctly deserialized test_upgrade(ledger_wasm_current.clone(), balances.clone()); - // Test the new wasm serialization - test_upgrade(ledger_wasm_current, balances.clone()); // Add some more approvals for a1 in &accounts { @@ -2701,8 +2698,8 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } } - // Test downgrade to mainnet wasm - test_upgrade(ledger_wasm_mainnet, balances); + // Test the new wasm serialization + test_upgrade(ledger_wasm_current, balances); // See if the additional approvals are there for a1 in &accounts { diff --git a/rs/rosetta-api/icrc1/ledger/src/lib.rs b/rs/rosetta-api/icrc1/ledger/src/lib.rs index ce1c286f35c..cc77993f19c 100644 --- a/rs/rosetta-api/icrc1/ledger/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/src/lib.rs @@ -356,12 +356,30 @@ thread_local! { MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } +#[derive(Clone, Serialize, Deserialize, Debug)] +pub enum LedgerField { + Allowances, + AllowancesExpirations, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub enum LedgerState { + Migrating(LedgerField), + Ready, +} + +impl Default for LedgerState { + fn default() -> Self { + Self::Ready + } +} + #[derive(Debug, Deserialize, Serialize)] #[serde(bound = "")] pub struct Ledger { balances: LedgerBalances, #[serde(default)] - approvals: LedgerAllowances, + pub approvals: LedgerAllowances, #[serde(default)] stable_approvals: AllowanceTable, blockchain: Blockchain, @@ -389,6 +407,9 @@ pub struct Ledger { maximum_number_of_accounts: usize, #[serde(default = "default_accounts_overflow_trim_quantity")] accounts_overflow_trim_quantity: usize, + + #[serde(default)] + pub state: LedgerState, } fn default_maximum_number_of_accounts() -> usize { @@ -482,6 +503,7 @@ impl Ledger { .unwrap_or_else(|| ACCOUNTS_OVERFLOW_TRIM_QUANTITY.try_into().unwrap()) .try_into() .unwrap(), + state: LedgerState::Ready, }; for (account, balance) in initial_balances.into_iter() { @@ -502,6 +524,20 @@ impl Ledger { ledger } + + pub fn is_migrating(&self) -> bool { + match self.state { + LedgerState::Ready => false, + _ => true, + } + } + + pub fn is_ready(&self) -> bool { + match self.state { + LedgerState::Ready => true, + _ => false, + } + } } impl LedgerContext for Ledger { @@ -959,6 +995,12 @@ impl AllowancesData for StableAllowancesData { .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)) } + fn pop_first_allowance( + &mut self, + ) -> Option<((Self::AccountId, Self::AccountId), Allowance)> { + todo!() + } + fn len_allowances(&self) -> usize { ALLOWANCES_MEMORY .with_borrow(|allowances| allowances.len()) @@ -976,4 +1018,8 @@ impl AllowancesData for StableAllowancesData { fn len_arrivals(&self) -> usize { 0 } + + fn clear_arrivals(&mut self) { + todo!() + } } diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 36e793d5697..8100494fed4 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -3,10 +3,11 @@ mod benches; use candid::candid_method; use candid::types::number::Nat; -use ic_canister_log::{declare_log_buffer, export}; +use ic_canister_log::{declare_log_buffer, export, log}; use ic_canisters_http_types::{HttpRequest, HttpResponse, HttpResponseBuilder}; use ic_cdk::api::stable::StableReader; +use ic_cdk::api::instruction_counter; #[cfg(not(feature = "canbench-rs"))] use ic_cdk_macros::init; use ic_cdk_macros::{post_upgrade, pre_upgrade, query, update}; @@ -15,12 +16,13 @@ use ic_icrc1::{ Operation, Transaction, }; use ic_icrc1_ledger::UPGRADES_MEMORY; -use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument}; +use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; use ic_ledger_canister_core::ledger::{ apply_transaction, archive_blocks, LedgerAccess, LedgerContext, LedgerData, TransferError as CoreTransferError, }; use ic_ledger_canister_core::runtime::total_memory_size_bytes; +use ic_ledger_core::approvals::AllowancesData; use ic_ledger_core::block::BlockIndex; use ic_ledger_core::timestamp::TimeStamp; use ic_ledger_core::tokens::Zero; @@ -59,8 +61,11 @@ use icrc_ledger_types::{ }; use num_traits::{bounds::Bounded, ToPrimitive}; use serde_bytes::ByteBuf; -use std::cell::RefCell; -use std::io::{Read, Write}; +use std::{ + cell::RefCell, + io::{Read, Write}, + time::Duration, +}; const MAX_MESSAGE_SIZE: u64 = 1024 * 1024; @@ -213,11 +218,82 @@ fn post_upgrade(args: Option) { PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); + Access::with_ledger_mut(|ledger| { + ledger.state = LedgerState::Migrating(LedgerField::Allowances); + ledger.approvals.allowances_data.clear_arrivals(); + }); + const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 20_000_000_000; + migrate_next_part(MAX_INSTRUCTIONS_PER_UPGRADE); + let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; POST_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = instructions_consumed); } +fn migrate_next_part(instruction_limit: u64) { + let mut migrated_allowances = 0; + let mut migrated_expirations = 0; + + ic_cdk::println!("Migration started."); + log!(&LOG, "Migration started."); + + Access::with_ledger_mut(|ledger| { + while instruction_counter() < instruction_limit { + let field = match ledger.state.clone() { + LedgerState::Migrating(ledger_field) => ledger_field, + LedgerState::Ready => break, + }; + match field { + LedgerField::Allowances => { + match ledger.approvals.allowances_data.pop_first_allowance() { + Some((account_spender, allowance)) => { + ledger + .approvals_mut() + .allowances_data + .set_allowance(account_spender, allowance); + migrated_allowances += 1; + } + None => { + ledger.state = + LedgerState::Migrating(LedgerField::AllowancesExpirations); + } + }; + } + LedgerField::AllowancesExpirations => { + match ledger.approvals.allowances_data.pop_first_expiry() { + Some((timestamp, account_spender)) => { + ledger + .approvals_mut() + .allowances_data + .insert_expiry(timestamp, account_spender); + migrated_expirations += 1; + } + None => { + ledger.state = LedgerState::Ready; + } + }; + } + } + } + let msg = format!("Number of elements migrated: allowances:{migrated_allowances} expirations:{migrated_expirations}. Instructions used {}.", + instruction_counter()); + if ledger.is_migrating() { + ic_cdk::println!("Migration partially done. Scheduling the next part. {msg}"); + log!( + &LOG, + "Migration partially done. Scheduling the next part. {msg}" + ); + const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 2_000_000_000; + ic_cdk_timers::set_timer(Duration::from_secs(0), || { + migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) + }); + } else { + ic_cdk::println!("Migration completed! {msg}"); + log!(&LOG, "Migration completed! {msg}"); + } + }); +} + fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()> { w.encode_gauge( "ledger_stable_memory_pages", diff --git a/rs/rosetta-api/icrc1/ledger/tests/tests.rs b/rs/rosetta-api/icrc1/ledger/tests/tests.rs index 7dcbbcefd06..c0d133c388e 100644 --- a/rs/rosetta-api/icrc1/ledger/tests/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/tests/tests.rs @@ -403,6 +403,7 @@ fn icrc1_test_upgrade_serialization() { upgrade_args, minter, true, + false, ); } diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index 6d8000629c4..7ff23e9708f 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -74,6 +74,10 @@ pub trait AllowancesData { #[allow(clippy::type_complexity)] fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))>; + fn pop_first_allowance( + &mut self, + ) -> Option<((Self::AccountId, Self::AccountId), Allowance)>; + fn oldest_arrivals(&self, n: usize) -> Vec<(Self::AccountId, Self::AccountId)>; fn len_allowances(&self) -> usize; @@ -81,6 +85,8 @@ pub trait AllowancesData { fn len_expirations(&self) -> usize; fn len_arrivals(&self) -> usize; + + fn clear_arrivals(&mut self); } #[derive(Debug, Deserialize, Serialize)] @@ -173,6 +179,12 @@ where self.expiration_queue.pop_first() } + fn pop_first_allowance( + &mut self, + ) -> Option<((Self::AccountId, Self::AccountId), Allowance)> { + self.allowances.pop_first() + } + fn oldest_arrivals(&self, n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { let mut result = vec![]; for (_t, key) in &self.arrival_queue { @@ -195,6 +207,10 @@ where fn len_arrivals(&self) -> usize { self.arrival_queue.len() } + + fn clear_arrivals(&mut self) { + self.arrival_queue.clear(); + } } #[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)] @@ -217,7 +233,7 @@ impl Default for Allowance { #[derive(Debug, Deserialize, Serialize)] #[serde(transparent)] pub struct AllowanceTable { - allowances_data: AD, + pub allowances_data: AD, } impl Default for AllowanceTable From af7376e7b70d2dfc5106fd6f194464a2422e8a94 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Wed, 18 Sep 2024 15:00:18 +0000 Subject: [PATCH 010/124] Automatically updated Cargo*.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index c3aaf8e196b..398db1ac3ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8822,6 +8822,7 @@ dependencies = [ "ic-canisters-http-types", "ic-cdk 0.13.5", "ic-cdk-macros 0.9.0", + "ic-cdk-timers", "ic-crypto-tree-hash", "ic-icrc1", "ic-icrc1-ledger-sm-tests", From 0fc1d35a1d0580a682caf90d14ab5f2d02d0df1b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 23 Sep 2024 14:55:11 +0000 Subject: [PATCH 011/124] disable ledger while migration and wait for ledger to be ready --- .../icrc1/ledger/sm-tests/src/lib.rs | 26 ++++++++++ rs/rosetta-api/icrc1/ledger/src/main.rs | 52 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index e0621a32bc4..450040ccf1a 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -539,6 +539,29 @@ pub fn balance_of(env: &StateMachine, ledger: CanisterId, acc: impl Into BTreeMap { Decode!( &env.query(ledger, "icrc1_metadata", Encode!().unwrap()) @@ -2543,6 +2566,7 @@ pub fn test_upgrade_serialization( let mut test_upgrade = |ledger_wasm: Vec| { env.upgrade_canister(ledger_id, ledger_wasm, upgrade_args.clone()) .unwrap(); + wait_ledger_ready(&env, ledger_id, 10); add_tx_and_verify(); }; @@ -2661,6 +2685,8 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( ) .unwrap(); + wait_ledger_ready(&env, canister_id, 10); + let mut allowances = vec![]; for i in 0..accounts.len() { for j in i + 1..accounts.len() { diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 8100494fed4..a310a7e43bc 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -492,12 +492,18 @@ fn icrc1_minting_account() -> Option { #[query(name = "icrc1_balance_of")] #[candid_method(query, rename = "icrc1_balance_of")] fn icrc1_balance_of(account: Account) -> Nat { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| ledger.balances().account_balance(&account).into()) } #[query(name = "icrc1_total_supply")] #[candid_method(query, rename = "icrc1_total_supply")] fn icrc1_total_supply() -> Nat { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| ledger.balances().total_supply().into()) } @@ -634,6 +640,9 @@ fn execute_transfer_not_async( #[update] #[candid_method(update)] async fn icrc1_transfer(arg: TransferArg) -> Result { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let from_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.from_subaccount, @@ -661,6 +670,9 @@ async fn icrc1_transfer(arg: TransferArg) -> Result { #[update] #[candid_method(update)] async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let spender_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.spender_subaccount, @@ -687,6 +699,9 @@ async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result Vec { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| { ledger .blockchain() @@ -736,6 +751,9 @@ fn supported_standards() -> Vec { #[query] #[candid_method(query)] fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let (start, length) = req .as_start_and_length() .unwrap_or_else(|msg| ic_cdk::api::trap(&msg)); @@ -746,6 +764,9 @@ fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse { #[query] #[candid_method(query)] fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let (start, length) = req .as_start_and_length() .unwrap_or_else(|msg| ic_cdk::api::trap(&msg)); @@ -755,6 +776,9 @@ fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse { #[query] #[candid_method(query)] fn get_data_certificate() -> DataCertificate { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree()); let mut tree_buf = vec![]; ciborium::ser::into_writer(&hash_tree, &mut tree_buf).unwrap(); @@ -767,6 +791,9 @@ fn get_data_certificate() -> DataCertificate { #[update] #[candid_method(update)] async fn icrc2_approve(arg: ApproveArgs) -> Result { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let block_idx = Access::with_ledger_mut(|ledger| { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); @@ -845,6 +872,9 @@ async fn icrc2_approve(arg: ApproveArgs) -> Result { #[query] #[candid_method(query)] fn icrc2_allowance(arg: AllowanceArgs) -> Allowance { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); let allowance = ledger @@ -860,12 +890,18 @@ fn icrc2_allowance(arg: AllowanceArgs) -> Allowance { #[query] #[candid_method(query)] fn icrc3_get_archives(args: GetArchivesArgs) -> GetArchivesResult { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| ledger.icrc3_get_archives(args)) } #[query] #[candid_method(query)] fn icrc3_get_tip_certificate() -> Option { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let certificate = ByteBuf::from(ic_cdk::api::data_certificate()?); let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree()); let mut tree_buf = vec![]; @@ -908,6 +944,9 @@ fn icrc3_supported_block_types() -> Vec) -> GetBlocksResult { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } Access::with_ledger(|ledger| ledger.icrc3_get_blocks(args)) } @@ -922,6 +961,9 @@ fn icrc10_supported_standards() -> Vec { fn icrc21_canister_call_consent_message( consent_msg_request: ConsentMessageRequest, ) -> Result { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } let caller_principal = ic_cdk::api::caller(); let ledger_fee = icrc1_fee(); let token_symbol = icrc1_symbol(); @@ -936,6 +978,16 @@ fn icrc21_canister_call_consent_message( ) } +fn is_ready() -> bool { + Access::with_ledger(|ledger| ledger.is_ready()) +} + +#[query] +#[candid_method(query)] +fn is_ledger_ready() -> bool { + is_ready() +} + candid::export_service!(); #[query] From 823472c95465e816e84fd5ddf02d3722331ae2e3 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 09:44:03 +0000 Subject: [PATCH 012/124] remove approval trimming test --- rs/rosetta-api/icrc1/ledger/tests/tests.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/tests/tests.rs b/rs/rosetta-api/icrc1/ledger/tests/tests.rs index c0d133c388e..cc39b05704c 100644 --- a/rs/rosetta-api/icrc1/ledger/tests/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/tests/tests.rs @@ -340,11 +340,6 @@ fn test_balances_overflow() { ic_icrc1_ledger_sm_tests::test_balances_overflow(ledger_wasm(), encode_init_args); } -#[test] -fn test_approval_trimming() { - ic_icrc1_ledger_sm_tests::test_approval_trimming(ledger_wasm(), encode_init_args); -} - #[test] fn test_archive_controllers() { ic_icrc1_ledger_sm_tests::test_archive_controllers(ledger_wasm()); From 871d1e1fa660eb88858f45eeb09e6944e0fb3c30 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 09:49:32 +0000 Subject: [PATCH 013/124] clippy --- rs/rosetta-api/ledger_core/src/approvals.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index 7ff23e9708f..8f4764d0b64 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -74,6 +74,7 @@ pub trait AllowancesData { #[allow(clippy::type_complexity)] fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))>; + #[allow(clippy::type_complexity)] fn pop_first_allowance( &mut self, ) -> Option<((Self::AccountId, Self::AccountId), Allowance)>; From ab60f40e120acf46c1ed9f5cc2e8ccdf11bf8c55 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 10:06:49 +0000 Subject: [PATCH 014/124] build fix --- rs/rosetta-api/icrc1/ledger/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/rosetta-api/icrc1/ledger/BUILD.bazel b/rs/rosetta-api/icrc1/ledger/BUILD.bazel index d06492e2caf..b03766d08f5 100644 --- a/rs/rosetta-api/icrc1/ledger/BUILD.bazel +++ b/rs/rosetta-api/icrc1/ledger/BUILD.bazel @@ -305,6 +305,7 @@ rust_test( "@crate_index//:candid", "@crate_index//:ciborium", "@crate_index//:ic-cdk", + "@crate_index//:ic-cdk-timers", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", "@crate_index//:num-traits", From f10866205d022b0701565ae82b13ccb89475bd4e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 11:15:00 +0000 Subject: [PATCH 015/124] fix candid --- rs/rosetta-api/icrc1/ledger/ledger.did | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/rosetta-api/icrc1/ledger/ledger.did b/rs/rosetta-api/icrc1/ledger/ledger.did index ac3a7bff087..dece15d9bba 100644 --- a/rs/rosetta-api/icrc1/ledger/ledger.did +++ b/rs/rosetta-api/icrc1/ledger/ledger.did @@ -509,4 +509,6 @@ service : (ledger_arg : LedgerArg) -> { icrc21_canister_call_consent_message: (icrc21_consent_message_request) -> (icrc21_consent_message_response); icrc10_supported_standards : () -> (vec record { name : text; url : text }) query; + + is_ledger_ready: () -> (bool) query; } From e454fbc803a68eb2a44f8c617cf80be7b3994b5c Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 11:29:26 +0000 Subject: [PATCH 016/124] fix tests --- rs/rosetta-api/icp_ledger/ledger/tests/tests.rs | 2 +- rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs | 8 +++++--- rs/rosetta-api/icrc1/ledger/tests/tests.rs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs b/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs index cd986c1bb8e..fd13d4e8abf 100644 --- a/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs +++ b/rs/rosetta-api/icp_ledger/ledger/tests/tests.rs @@ -1005,7 +1005,7 @@ fn test_upgrade_serialization() { upgrade_args, minter, false, - true, + false, ); } diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index 450040ccf1a..e47300f638d 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2517,7 +2517,7 @@ pub fn test_upgrade_serialization( upgrade_args: Vec, minter: Arc, verify_blocks: bool, - downgrade_to_mainnet_possible: bool, + migration_to_stable_structures: bool, ) { let mut runner = TestRunner::new(TestRunnerConfig::with_cases(1)); let now = SystemTime::now(); @@ -2566,7 +2566,9 @@ pub fn test_upgrade_serialization( let mut test_upgrade = |ledger_wasm: Vec| { env.upgrade_canister(ledger_id, ledger_wasm, upgrade_args.clone()) .unwrap(); - wait_ledger_ready(&env, ledger_id, 10); + if migration_to_stable_structures { + wait_ledger_ready(&env, ledger_id, 10); + } add_tx_and_verify(); }; @@ -2598,7 +2600,7 @@ pub fn test_upgrade_serialization( } // Test deserializing from memory manager test_upgrade(ledger_wasm_current.clone()); - if downgrade_to_mainnet_possible { + if !migration_to_stable_structures { // Test downgrade to mainnet wasm test_upgrade(ledger_wasm_mainnet.clone()); } diff --git a/rs/rosetta-api/icrc1/ledger/tests/tests.rs b/rs/rosetta-api/icrc1/ledger/tests/tests.rs index cc39b05704c..b419148af9d 100644 --- a/rs/rosetta-api/icrc1/ledger/tests/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/tests/tests.rs @@ -398,7 +398,7 @@ fn icrc1_test_upgrade_serialization() { upgrade_args, minter, true, - false, + true, ); } From 378c316211ff2d394b719a1edd3cc68336f1356d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 12:18:54 +0000 Subject: [PATCH 017/124] fix tests --- rs/rosetta-api/icrc1/ledger/src/tests.rs | 218 ++++++++++---------- rs/rosetta-api/ledger_core/src/approvals.rs | 6 - 2 files changed, 107 insertions(+), 117 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/tests.rs b/rs/rosetta-api/icrc1/ledger/src/tests.rs index b7ee02eb137..c1c8ff249fc 100644 --- a/rs/rosetta-api/icrc1/ledger/src/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/src/tests.rs @@ -2,11 +2,11 @@ use crate::{InitArgs, Ledger}; use ic_base_types::PrincipalId; use ic_canister_log::Sink; use ic_icrc1::{Operation, Transaction}; +use ic_icrc1_tokens_u64::U64; use ic_ledger_canister_core::archive::ArchiveOptions; use ic_ledger_canister_core::ledger::{LedgerContext, LedgerTransaction, TxApplyError}; use ic_ledger_core::approvals::Allowance; use ic_ledger_core::timestamp::TimeStamp; -use ic_ledger_core::Tokens; use icrc_ledger_types::icrc::generic_metadata_value::MetadataValue as Value; use icrc_ledger_types::icrc1::account::Account; @@ -32,10 +32,6 @@ fn test_account_id(n: u64) -> Account { } } -fn tokens(n: u64) -> Tokens { - Tokens::from_e8s(n) -} - fn ts(n: u64) -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(n) } @@ -81,10 +77,10 @@ fn test_approvals_are_not_cumulative() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, tokens(100_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(100_000)).unwrap(); - let approved_amount = tokens(150_000); - let fee = tokens(10_000); + let approved_amount = U64::from(150_000); + let fee = U64::from(10_000); let tr = Transaction { operation: Operation::Approve { @@ -98,21 +94,21 @@ fn test_approvals_are_not_cumulative() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); - assert_eq!(ctx.balances().account_balance(&spender), tokens(0)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), U64::from(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { amount: approved_amount, expires_at: None, - arrived_at: now, + arrived_at: ts(0), }, ); - let new_allowance = tokens(200_000); + let new_allowance = U64::from(200_000); let expiration = now + Duration::from_secs(300); let tr = Transaction { @@ -127,16 +123,16 @@ fn test_approvals_are_not_cumulative() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(80_000)); - assert_eq!(ctx.balances().account_balance(&spender), tokens(0)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000)); + assert_eq!(ctx.balances().account_balance(&spender), U64::from(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { amount: new_allowance, expires_at: Some(expiration), - arrived_at: now, + arrived_at: ts(0), } ); } @@ -151,24 +147,24 @@ fn test_approval_transfer_from() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); - let fee = tokens(10_000); + ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); + let fee = U64::from(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: tokens(0) + allowance: U64::from(0) } ); @@ -176,7 +172,7 @@ fn test_approval_transfer_from() { operation: Operation::Approve { from, spender, - amount: tokens(150_000), + amount: U64::from(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -184,33 +180,33 @@ fn test_approval_transfer_from() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(190_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(190_000)); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&to), tokens(100_000)); - assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().account_balance(&from), tokens(80_000)); + assert_eq!(ctx.balances().account_balance(&to), U64::from(100_000)); + assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); + assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(40_000), + amount: U64::from(40_000), expires_at: None, - arrived_at: now, + arrived_at: ts(0), }, ); @@ -219,29 +215,29 @@ fn test_approval_transfer_from() { from, to, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: tokens(40_000) + allowance: U64::from(40_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(40_000), + amount: U64::from(40_000), expires_at: None, - arrived_at: now, + arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), tokens(80_000),); - assert_eq!(ctx.balances().account_balance(&to), tokens(100_000),); + assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000),); + assert_eq!(ctx.balances().account_balance(&to), U64::from(100_000),); } #[test] @@ -253,29 +249,29 @@ fn test_approval_expiration_override() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); let approve = |amount: u64, expires_at: Option| Operation::Approve { from, spender, - amount: tokens(amount), + amount: U64::from(amount), expected_allowance: None, expires_at: expires_at.map(|e| e.as_nanos_since_unix_epoch()), - fee: Some(tokens(10_000)), + fee: Some(U64::from(10_000)), }; let tr = Transaction { operation: approve(100_000, Some(ts(2000))), created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(100_000), + amount: U64::from(100_000), expires_at: Some(ts(2000)), - arrived_at: now, + arrived_at: ts(0), }, ); @@ -284,14 +280,14 @@ fn test_approval_expiration_override() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(200_000), + amount: U64::from(200_000), expires_at: Some(ts(1500)), - arrived_at: now, + arrived_at: ts(0), }, ); @@ -300,14 +296,14 @@ fn test_approval_expiration_override() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(300_000), + amount: U64::from(300_000), expires_at: Some(ts(2500)), - arrived_at: now, + arrived_at: ts(0), }, ); @@ -318,16 +314,16 @@ fn test_approval_expiration_override() { memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::ExpiredApproval { now } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(300_000), + amount: U64::from(300_000), expires_at: Some(ts(2500)), - arrived_at: now, + arrived_at: ts(0), }, ); } @@ -341,23 +337,23 @@ fn test_approval_no_fee_on_reject() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, tokens(20_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(20_000)).unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: tokens(1_000), + amount: U64::from(1_000), expected_allowance: None, expires_at: Some(1), - fee: Some(tokens(10_000)), + fee: Some(U64::from(10_000)), }, created_at_time: Some(1000), memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::ExpiredApproval { now } ); @@ -366,7 +362,7 @@ fn test_approval_no_fee_on_reject() { Allowance::default(), ); - assert_eq!(ctx.balances().account_balance(&from), tokens(20_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(20_000)); } #[test] @@ -378,7 +374,7 @@ fn test_self_transfer_from() { let from = test_account_id(1); let to = test_account_id(2); - ctx.balances_mut().mint(&from, tokens(100_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(100_000)).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &from, now), @@ -390,16 +386,16 @@ fn test_self_transfer_from() { from, to, spender: Some(from), - amount: tokens(20_000), - fee: Some(tokens(10_000)), + amount: U64::from(20_000), + fee: Some(U64::from(10_000)), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(70_000)); - assert_eq!(ctx.balances().account_balance(&to), tokens(20_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(70_000)); + assert_eq!(ctx.balances().account_balance(&to), U64::from(20_000)); } #[test] @@ -412,13 +408,13 @@ fn test_approval_allowance_covers_fee() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut().mint(&from, tokens(20_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(20_000)).unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: tokens(10_000), + amount: U64::from(10_000), expected_allowance: None, expires_at: None, fee: None, @@ -426,24 +422,24 @@ fn test_approval_allowance_covers_fee() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - let fee = tokens(10_000); + let fee = U64::from(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: tokens(10_000), + amount: U64::from(10_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: tokens(10_000) + allowance: U64::from(10_000) } ); @@ -451,7 +447,7 @@ fn test_approval_allowance_covers_fee() { operation: Operation::Approve { from, spender, - amount: tokens(20_000), + amount: U64::from(20_000), expected_allowance: None, expires_at: None, fee: None, @@ -459,28 +455,28 @@ fn test_approval_allowance_covers_fee() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: tokens(10_000), + amount: U64::from(10_000), fee: Some(fee), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(0)); - assert_eq!(ctx.balances().account_balance(&to), tokens(10_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(0)); + assert_eq!(ctx.balances().account_balance(&to), U64::from(10_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(0), + amount: U64::from(0), expires_at: None, arrived_at: ts(0), }, @@ -495,23 +491,23 @@ fn test_burn_smoke() { let from = test_account_id(1); - ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); + ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); - assert_eq!(ctx.balances().total_supply().get_e8s(), 200_000); + assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); let tr = Transaction { operation: Operation::Burn { from, spender: None, - amount: tokens(100_000), + amount: U64::from(100_000), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(100_000)); - assert_eq!(ctx.balances().total_supply(), tokens(100_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(100_000)); + assert_eq!(ctx.balances().total_supply(), U64::from(100_000)); } #[test] @@ -523,34 +519,34 @@ fn test_approval_burn_from() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); - let fee = tokens(10_000); + ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); + let fee = U64::from(10_000); - assert_eq!(ctx.balances().total_supply().get_e8s(), 200_000); + assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); let tr = Transaction { operation: Operation::Burn { from, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: Tokens::ZERO + allowance: U64::ZERO } ); - assert_eq!(ctx.balances().total_supply().get_e8s(), 200_000); + assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: tokens(150_000), + amount: U64::from(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -558,32 +554,32 @@ fn test_approval_burn_from() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), tokens(190_000)); - assert_eq!(ctx.balances().total_supply(), tokens(190_000)); + assert_eq!(ctx.balances().account_balance(&from), U64::from(190_000)); + assert_eq!(ctx.balances().total_supply(), U64::from(190_000)); let tr = Transaction { operation: Operation::Burn { from, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); + tr.apply(&mut ctx, now, U64::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); - assert_eq!(ctx.balances().total_supply().get_e8s(), 90_000); + assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); + assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); + assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(50_000), + amount: U64::from(50_000), expires_at: None, - arrived_at: now, + arrived_at: ts(0), }, ); @@ -591,27 +587,27 @@ fn test_approval_burn_from() { operation: Operation::Burn { from, spender: Some(spender), - amount: tokens(100_000), + amount: U64::from(100_000), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: tokens(50_000) + allowance: U64::from(50_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: tokens(50_000), + amount: U64::from(50_000), expires_at: None, - arrived_at: now, + arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); - assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().total_supply().get_e8s(), 90_000); + assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); + assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); } diff --git a/rs/rosetta-api/ledger_core/src/approvals.rs b/rs/rosetta-api/ledger_core/src/approvals.rs index 8f4764d0b64..bcbc14ca5d8 100644 --- a/rs/rosetta-api/ledger_core/src/approvals.rs +++ b/rs/rosetta-api/ledger_core/src/approvals.rs @@ -267,12 +267,6 @@ where self.allowances_data.len_expirations(), self.allowances_data.len_allowances() ); - debug_assert!( - self.allowances_data.len_arrivals() == self.allowances_data.len_allowances(), - "arrival_queue length ({}) should be equal to allowances length ({})", - self.allowances_data.len_arrivals(), - self.allowances_data.len_allowances() - ); } fn with_postconditions_check(&mut self, f: impl FnOnce(&mut Self) -> R) -> R { From 2b64a035ca9180fb33689cd0e0ab1d46bf47ec44 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 12:44:35 +0000 Subject: [PATCH 018/124] clippy --- rs/rosetta-api/icrc1/ledger/src/lib.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/lib.rs b/rs/rosetta-api/icrc1/ledger/src/lib.rs index cc77993f19c..da701b5981e 100644 --- a/rs/rosetta-api/icrc1/ledger/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/src/lib.rs @@ -348,10 +348,12 @@ thread_local! { RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); // (from, spender) -> allowance - map storing ledger allowances. + #[allow(clippy::type_complexity)] pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_MEMORY_ID)))); // (timestamp, (from, spender)) - expiration set used for removing expired allowances. + #[allow(clippy::type_complexity)] pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } @@ -526,17 +528,11 @@ impl Ledger { } pub fn is_migrating(&self) -> bool { - match self.state { - LedgerState::Ready => false, - _ => true, - } + !matches!(self.state, LedgerState::Ready) } pub fn is_ready(&self) -> bool { - match self.state { - LedgerState::Ready => true, - _ => false, - } + matches!(self.state, LedgerState::Ready) } } @@ -638,7 +634,7 @@ impl Ledger { } pub fn transfer_fee(&self) -> Tokens { - self.transfer_fee.clone() + self.transfer_fee } pub fn max_memo_length(&self) -> u16 { From 9f545dfb6e6eff70e454c155e7257000837dc423 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 13:28:18 +0000 Subject: [PATCH 019/124] fix tests --- rs/rosetta-api/icrc1/ledger/src/tests.rs | 205 +++++++++++++---------- 1 file changed, 113 insertions(+), 92 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/tests.rs b/rs/rosetta-api/icrc1/ledger/src/tests.rs index c1c8ff249fc..1e2595c16c1 100644 --- a/rs/rosetta-api/icrc1/ledger/src/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/src/tests.rs @@ -2,7 +2,6 @@ use crate::{InitArgs, Ledger}; use ic_base_types::PrincipalId; use ic_canister_log::Sink; use ic_icrc1::{Operation, Transaction}; -use ic_icrc1_tokens_u64::U64; use ic_ledger_canister_core::archive::ArchiveOptions; use ic_ledger_canister_core::ledger::{LedgerContext, LedgerTransaction, TxApplyError}; use ic_ledger_core::approvals::Allowance; @@ -36,6 +35,12 @@ fn ts(n: u64) -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(n) } +#[cfg(not(feature = "u256-tokens"))] +pub type Tokens = ic_icrc1_tokens_u64::U64; + +#[cfg(feature = "u256-tokens")] +pub type Tokens = ic_icrc1_tokens_u256::U256; + fn default_init_args() -> InitArgs { InitArgs { minting_account: MINTER, @@ -77,10 +82,12 @@ fn test_approvals_are_not_cumulative() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, U64::from(100_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(100_000)) + .unwrap(); - let approved_amount = U64::from(150_000); - let fee = U64::from(10_000); + let approved_amount = Tokens::from(150_000); + let fee = Tokens::from(10_000); let tr = Transaction { operation: Operation::Approve { @@ -94,10 +101,10 @@ fn test_approvals_are_not_cumulative() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); - assert_eq!(ctx.balances().account_balance(&spender), U64::from(0)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), Tokens::from(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), @@ -108,7 +115,7 @@ fn test_approvals_are_not_cumulative() { }, ); - let new_allowance = U64::from(200_000); + let new_allowance = Tokens::from(200_000); let expiration = now + Duration::from_secs(300); let tr = Transaction { @@ -123,10 +130,10 @@ fn test_approvals_are_not_cumulative() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000)); - assert_eq!(ctx.balances().account_balance(&spender), U64::from(0)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000)); + assert_eq!(ctx.balances().account_balance(&spender), Tokens::from(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { @@ -147,24 +154,26 @@ fn test_approval_transfer_from() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); - let fee = U64::from(10_000); + ctx.balances_mut() + .mint(&from, Tokens::from(200_000)) + .unwrap(); + let fee = Tokens::from(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: U64::from(0) + allowance: Tokens::from(0) } ); @@ -172,7 +181,7 @@ fn test_approval_transfer_from() { operation: Operation::Approve { from, spender, - amount: U64::from(150_000), + amount: Tokens::from(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -180,31 +189,31 @@ fn test_approval_transfer_from() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(190_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(190_000)); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&to), U64::from(100_000)); - assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); - assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000)); + assert_eq!(ctx.balances().account_balance(&to), Tokens::from(100_000)); + assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(40_000), + amount: Tokens::from(40_000), expires_at: None, arrived_at: ts(0), }, @@ -215,29 +224,29 @@ fn test_approval_transfer_from() { from, to, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: U64::from(40_000) + allowance: Tokens::from(40_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(40_000), + amount: Tokens::from(40_000), expires_at: None, arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), U64::from(80_000),); - assert_eq!(ctx.balances().account_balance(&to), U64::from(100_000),); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000),); + assert_eq!(ctx.balances().account_balance(&to), Tokens::from(100_000),); } #[test] @@ -249,27 +258,29 @@ fn test_approval_expiration_override() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(200_000)) + .unwrap(); let approve = |amount: u64, expires_at: Option| Operation::Approve { from, spender, - amount: U64::from(amount), + amount: Tokens::from(amount), expected_allowance: None, expires_at: expires_at.map(|e| e.as_nanos_since_unix_epoch()), - fee: Some(U64::from(10_000)), + fee: Some(Tokens::from(10_000)), }; let tr = Transaction { operation: approve(100_000, Some(ts(2000))), created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(100_000), + amount: Tokens::from(100_000), expires_at: Some(ts(2000)), arrived_at: ts(0), }, @@ -280,12 +291,12 @@ fn test_approval_expiration_override() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(200_000), + amount: Tokens::from(200_000), expires_at: Some(ts(1500)), arrived_at: ts(0), }, @@ -296,12 +307,12 @@ fn test_approval_expiration_override() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(300_000), + amount: Tokens::from(300_000), expires_at: Some(ts(2500)), arrived_at: ts(0), }, @@ -314,14 +325,14 @@ fn test_approval_expiration_override() { memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::ExpiredApproval { now } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(300_000), + amount: Tokens::from(300_000), expires_at: Some(ts(2500)), arrived_at: ts(0), }, @@ -337,23 +348,25 @@ fn test_approval_no_fee_on_reject() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, U64::from(20_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(20_000)) + .unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: U64::from(1_000), + amount: Tokens::from(1_000), expected_allowance: None, expires_at: Some(1), - fee: Some(U64::from(10_000)), + fee: Some(Tokens::from(10_000)), }, created_at_time: Some(1000), memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::ExpiredApproval { now } ); @@ -362,7 +375,7 @@ fn test_approval_no_fee_on_reject() { Allowance::default(), ); - assert_eq!(ctx.balances().account_balance(&from), U64::from(20_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(20_000)); } #[test] @@ -374,7 +387,9 @@ fn test_self_transfer_from() { let from = test_account_id(1); let to = test_account_id(2); - ctx.balances_mut().mint(&from, U64::from(100_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(100_000)) + .unwrap(); assert_eq!( ctx.approvals().allowance(&from, &from, now), @@ -386,16 +401,16 @@ fn test_self_transfer_from() { from, to, spender: Some(from), - amount: U64::from(20_000), - fee: Some(U64::from(10_000)), + amount: Tokens::from(20_000), + fee: Some(Tokens::from(10_000)), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(70_000)); - assert_eq!(ctx.balances().account_balance(&to), U64::from(20_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(70_000)); + assert_eq!(ctx.balances().account_balance(&to), Tokens::from(20_000)); } #[test] @@ -408,13 +423,15 @@ fn test_approval_allowance_covers_fee() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut().mint(&from, U64::from(20_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(20_000)) + .unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: U64::from(10_000), + amount: Tokens::from(10_000), expected_allowance: None, expires_at: None, fee: None, @@ -422,24 +439,24 @@ fn test_approval_allowance_covers_fee() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - let fee = U64::from(10_000); + let fee = Tokens::from(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: U64::from(10_000), + amount: Tokens::from(10_000), fee: Some(fee), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: U64::from(10_000) + allowance: Tokens::from(10_000) } ); @@ -447,7 +464,7 @@ fn test_approval_allowance_covers_fee() { operation: Operation::Approve { from, spender, - amount: U64::from(20_000), + amount: Tokens::from(20_000), expected_allowance: None, expires_at: None, fee: None, @@ -455,28 +472,28 @@ fn test_approval_allowance_covers_fee() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: U64::from(10_000), + amount: Tokens::from(10_000), fee: Some(fee), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(0)); - assert_eq!(ctx.balances().account_balance(&to), U64::from(10_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(0)); + assert_eq!(ctx.balances().account_balance(&to), Tokens::from(10_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(0), + amount: Tokens::from(0), expires_at: None, arrived_at: ts(0), }, @@ -491,7 +508,9 @@ fn test_burn_smoke() { let from = test_account_id(1); - ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); + ctx.balances_mut() + .mint(&from, Tokens::from(200_000)) + .unwrap(); assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); @@ -499,15 +518,15 @@ fn test_burn_smoke() { operation: Operation::Burn { from, spender: None, - amount: U64::from(100_000), + amount: Tokens::from(100_000), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(100_000)); - assert_eq!(ctx.balances().total_supply(), U64::from(100_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(100_000)); + assert_eq!(ctx.balances().total_supply(), Tokens::from(100_000)); } #[test] @@ -519,8 +538,10 @@ fn test_approval_burn_from() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut().mint(&from, U64::from(200_000)).unwrap(); - let fee = U64::from(10_000); + ctx.balances_mut() + .mint(&from, Tokens::from(200_000)) + .unwrap(); + let fee = Tokens::from(10_000); assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); @@ -528,15 +549,15 @@ fn test_approval_burn_from() { operation: Operation::Burn { from, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: U64::ZERO + allowance: Tokens::ZERO } ); @@ -546,7 +567,7 @@ fn test_approval_burn_from() { operation: Operation::Approve { from, spender, - amount: U64::from(150_000), + amount: Tokens::from(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -554,30 +575,30 @@ fn test_approval_burn_from() { created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), U64::from(190_000)); - assert_eq!(ctx.balances().total_supply(), U64::from(190_000)); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(190_000)); + assert_eq!(ctx.balances().total_supply(), Tokens::from(190_000)); let tr = Transaction { operation: Operation::Burn { from, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), }, created_at_time: None, memo: None, }; - tr.apply(&mut ctx, now, U64::ZERO).unwrap(); + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); - assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(50_000), + amount: Tokens::from(50_000), expires_at: None, arrived_at: ts(0), }, @@ -587,27 +608,27 @@ fn test_approval_burn_from() { operation: Operation::Burn { from, spender: Some(spender), - amount: U64::from(100_000), + amount: Tokens::from(100_000), }, created_at_time: None, memo: None, }; assert_eq!( - tr.apply(&mut ctx, now, U64::ZERO).unwrap_err(), + tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: U64::from(50_000) + allowance: Tokens::from(50_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: U64::from(50_000), + amount: Tokens::from(50_000), expires_at: None, arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), U64::from(90_000)); - assert_eq!(ctx.balances().account_balance(&spender), U64::ZERO); + assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); } From 82ff2830324ae054860f9963934524ff87e38439 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 14:02:14 +0000 Subject: [PATCH 020/124] clippy --- rs/rosetta-api/icrc1/ledger/src/tests.rs | 180 +++++++++++------------ 1 file changed, 89 insertions(+), 91 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/tests.rs b/rs/rosetta-api/icrc1/ledger/src/tests.rs index 1e2595c16c1..0375ef33317 100644 --- a/rs/rosetta-api/icrc1/ledger/src/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/src/tests.rs @@ -31,16 +31,30 @@ fn test_account_id(n: u64) -> Account { } } -fn ts(n: u64) -> TimeStamp { - TimeStamp::from_nanos_since_unix_epoch(n) -} - #[cfg(not(feature = "u256-tokens"))] pub type Tokens = ic_icrc1_tokens_u64::U64; #[cfg(feature = "u256-tokens")] pub type Tokens = ic_icrc1_tokens_u256::U256; +fn tokens(n: u64) -> Tokens { + Tokens::from(n) +} + +#[cfg(not(feature = "u256-tokens"))] +fn tokens_to_u64(n: ic_icrc1_tokens_u64::U64) -> u64 { + n.to_u64() +} + +#[cfg(feature = "u256-tokens")] +fn tokens_to_u64(n: ic_icrc1_tokens_u256::U256) -> u64 { + n.try_as_u64().expect("failed to convert to u64") +} + +fn ts(n: u64) -> TimeStamp { + TimeStamp::from_nanos_since_unix_epoch(n) +} + fn default_init_args() -> InitArgs { InitArgs { minting_account: MINTER, @@ -82,12 +96,10 @@ fn test_approvals_are_not_cumulative() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut() - .mint(&from, Tokens::from(100_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(100_000)).unwrap(); - let approved_amount = Tokens::from(150_000); - let fee = Tokens::from(10_000); + let approved_amount = tokens(150_000); + let fee = tokens(10_000); let tr = Transaction { operation: Operation::Approve { @@ -103,8 +115,8 @@ fn test_approvals_are_not_cumulative() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); - assert_eq!(ctx.balances().account_balance(&spender), Tokens::from(0)); + assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); + assert_eq!(ctx.balances().account_balance(&spender), tokens(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), @@ -115,7 +127,7 @@ fn test_approvals_are_not_cumulative() { }, ); - let new_allowance = Tokens::from(200_000); + let new_allowance = tokens(200_000); let expiration = now + Duration::from_secs(300); let tr = Transaction { @@ -132,8 +144,8 @@ fn test_approvals_are_not_cumulative() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000)); - assert_eq!(ctx.balances().account_balance(&spender), Tokens::from(0)); + assert_eq!(ctx.balances().account_balance(&from), tokens(80_000)); + assert_eq!(ctx.balances().account_balance(&spender), tokens(0)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { @@ -154,17 +166,15 @@ fn test_approval_transfer_from() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut() - .mint(&from, Tokens::from(200_000)) - .unwrap(); - let fee = Tokens::from(10_000); + ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); + let fee = tokens(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), fee: Some(fee), }, created_at_time: None, @@ -173,7 +183,7 @@ fn test_approval_transfer_from() { assert_eq!( tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: Tokens::from(0) + allowance: tokens(0) } ); @@ -181,7 +191,7 @@ fn test_approval_transfer_from() { operation: Operation::Approve { from, spender, - amount: Tokens::from(150_000), + amount: tokens(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -191,14 +201,14 @@ fn test_approval_transfer_from() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(190_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(190_000)); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), fee: Some(fee), }, created_at_time: None, @@ -206,14 +216,14 @@ fn test_approval_transfer_from() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&to), Tokens::from(100_000)); + assert_eq!(ctx.balances().account_balance(&to), tokens(100_000)); assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(80_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(40_000), + amount: tokens(40_000), expires_at: None, arrived_at: ts(0), }, @@ -224,7 +234,7 @@ fn test_approval_transfer_from() { from, to, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), fee: Some(fee), }, created_at_time: None, @@ -233,20 +243,20 @@ fn test_approval_transfer_from() { assert_eq!( tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: Tokens::from(40_000) + allowance: tokens(40_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(40_000), + amount: tokens(40_000), expires_at: None, arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(80_000),); - assert_eq!(ctx.balances().account_balance(&to), Tokens::from(100_000),); + assert_eq!(ctx.balances().account_balance(&from), tokens(80_000),); + assert_eq!(ctx.balances().account_balance(&to), tokens(100_000),); } #[test] @@ -258,17 +268,15 @@ fn test_approval_expiration_override() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut() - .mint(&from, Tokens::from(200_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); let approve = |amount: u64, expires_at: Option| Operation::Approve { from, spender, - amount: Tokens::from(amount), + amount: tokens(amount), expected_allowance: None, expires_at: expires_at.map(|e| e.as_nanos_since_unix_epoch()), - fee: Some(Tokens::from(10_000)), + fee: Some(tokens(10_000)), }; let tr = Transaction { operation: approve(100_000, Some(ts(2000))), @@ -280,7 +288,7 @@ fn test_approval_expiration_override() { assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(100_000), + amount: tokens(100_000), expires_at: Some(ts(2000)), arrived_at: ts(0), }, @@ -296,7 +304,7 @@ fn test_approval_expiration_override() { assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(200_000), + amount: tokens(200_000), expires_at: Some(ts(1500)), arrived_at: ts(0), }, @@ -312,7 +320,7 @@ fn test_approval_expiration_override() { assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(300_000), + amount: tokens(300_000), expires_at: Some(ts(2500)), arrived_at: ts(0), }, @@ -332,7 +340,7 @@ fn test_approval_expiration_override() { assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(300_000), + amount: tokens(300_000), expires_at: Some(ts(2500)), arrived_at: ts(0), }, @@ -348,18 +356,16 @@ fn test_approval_no_fee_on_reject() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut() - .mint(&from, Tokens::from(20_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(20_000)).unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: Tokens::from(1_000), + amount: tokens(1_000), expected_allowance: None, expires_at: Some(1), - fee: Some(Tokens::from(10_000)), + fee: Some(tokens(10_000)), }, created_at_time: Some(1000), memo: None, @@ -375,7 +381,7 @@ fn test_approval_no_fee_on_reject() { Allowance::default(), ); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(20_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(20_000)); } #[test] @@ -387,9 +393,7 @@ fn test_self_transfer_from() { let from = test_account_id(1); let to = test_account_id(2); - ctx.balances_mut() - .mint(&from, Tokens::from(100_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(100_000)).unwrap(); assert_eq!( ctx.approvals().allowance(&from, &from, now), @@ -401,16 +405,16 @@ fn test_self_transfer_from() { from, to, spender: Some(from), - amount: Tokens::from(20_000), - fee: Some(Tokens::from(10_000)), + amount: tokens(20_000), + fee: Some(tokens(10_000)), }, created_at_time: None, memo: None, }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(70_000)); - assert_eq!(ctx.balances().account_balance(&to), Tokens::from(20_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(70_000)); + assert_eq!(ctx.balances().account_balance(&to), tokens(20_000)); } #[test] @@ -423,15 +427,13 @@ fn test_approval_allowance_covers_fee() { let spender = test_account_id(2); let to = test_account_id(3); - ctx.balances_mut() - .mint(&from, Tokens::from(20_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(20_000)).unwrap(); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: Tokens::from(10_000), + amount: tokens(10_000), expected_allowance: None, expires_at: None, fee: None, @@ -441,13 +443,13 @@ fn test_approval_allowance_covers_fee() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - let fee = Tokens::from(10_000); + let fee = tokens(10_000); let tr = Transaction { operation: Operation::Transfer { from, to, spender: Some(spender), - amount: Tokens::from(10_000), + amount: tokens(10_000), fee: Some(fee), }, created_at_time: None, @@ -456,7 +458,7 @@ fn test_approval_allowance_covers_fee() { assert_eq!( tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: Tokens::from(10_000) + allowance: tokens(10_000) } ); @@ -464,7 +466,7 @@ fn test_approval_allowance_covers_fee() { operation: Operation::Approve { from, spender, - amount: Tokens::from(20_000), + amount: tokens(20_000), expected_allowance: None, expires_at: None, fee: None, @@ -479,7 +481,7 @@ fn test_approval_allowance_covers_fee() { from, to, spender: Some(spender), - amount: Tokens::from(10_000), + amount: tokens(10_000), fee: Some(fee), }, created_at_time: None, @@ -487,13 +489,13 @@ fn test_approval_allowance_covers_fee() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(0)); - assert_eq!(ctx.balances().account_balance(&to), Tokens::from(10_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(0)); + assert_eq!(ctx.balances().account_balance(&to), tokens(10_000)); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(0), + amount: tokens(0), expires_at: None, arrived_at: ts(0), }, @@ -508,25 +510,23 @@ fn test_burn_smoke() { let from = test_account_id(1); - ctx.balances_mut() - .mint(&from, Tokens::from(200_000)) - .unwrap(); + ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); - assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); + assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 200_000); let tr = Transaction { operation: Operation::Burn { from, spender: None, - amount: Tokens::from(100_000), + amount: tokens(100_000), }, created_at_time: None, memo: None, }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(100_000)); - assert_eq!(ctx.balances().total_supply(), Tokens::from(100_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(100_000)); + assert_eq!(ctx.balances().total_supply(), tokens(100_000)); } #[test] @@ -538,18 +538,16 @@ fn test_approval_burn_from() { let from = test_account_id(1); let spender = test_account_id(2); - ctx.balances_mut() - .mint(&from, Tokens::from(200_000)) - .unwrap(); - let fee = Tokens::from(10_000); + ctx.balances_mut().mint(&from, tokens(200_000)).unwrap(); + let fee = tokens(10_000); - assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); + assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 200_000); let tr = Transaction { operation: Operation::Burn { from, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), }, created_at_time: None, memo: None, @@ -561,13 +559,13 @@ fn test_approval_burn_from() { } ); - assert_eq!(ctx.balances().total_supply().to_u64(), 200_000); + assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 200_000); let tr = Transaction { operation: Operation::Approve { from, spender, - amount: Tokens::from(150_000), + amount: tokens(150_000), expected_allowance: None, expires_at: None, fee: Some(fee), @@ -577,14 +575,14 @@ fn test_approval_burn_from() { }; tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(190_000)); - assert_eq!(ctx.balances().total_supply(), Tokens::from(190_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(190_000)); + assert_eq!(ctx.balances().total_supply(), tokens(190_000)); let tr = Transaction { operation: Operation::Burn { from, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), }, created_at_time: None, memo: None, @@ -592,13 +590,13 @@ fn test_approval_burn_from() { tr.apply(&mut ctx, now, Tokens::ZERO).unwrap(); assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); - assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); + assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); + assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 90_000); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(50_000), + amount: tokens(50_000), expires_at: None, arrived_at: ts(0), }, @@ -608,7 +606,7 @@ fn test_approval_burn_from() { operation: Operation::Burn { from, spender: Some(spender), - amount: Tokens::from(100_000), + amount: tokens(100_000), }, created_at_time: None, memo: None, @@ -616,19 +614,19 @@ fn test_approval_burn_from() { assert_eq!( tr.apply(&mut ctx, now, Tokens::ZERO).unwrap_err(), TxApplyError::InsufficientAllowance { - allowance: Tokens::from(50_000) + allowance: tokens(50_000) } ); assert_eq!( ctx.approvals().allowance(&from, &spender, now), Allowance { - amount: Tokens::from(50_000), + amount: tokens(50_000), expires_at: None, arrived_at: ts(0), }, ); - assert_eq!(ctx.balances().account_balance(&from), Tokens::from(90_000)); + assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); - assert_eq!(ctx.balances().total_supply().to_u64(), 90_000); + assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 90_000); } From 3171aa03c7104b48947e43481743844ba367dde4 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 15:04:04 +0000 Subject: [PATCH 021/124] test account maxserialization len --- .../icrc-ledger-types/src/icrc1/account.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 9f0649f7fdc..bc3129fbed4 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -170,6 +170,8 @@ impl FromStr for Account { } } +const MAX_SERIALIZATION_LEN: u32 = 62; + impl Storable for Account { fn to_bytes(&self) -> Cow<[u8]> { let mut buffer: Vec = vec![]; @@ -212,7 +214,7 @@ impl Storable for Account { } const BOUND: Bound = Bound::Bounded { - max_size: 62, + max_size: MAX_SERIALIZATION_LEN, is_fixed_size: false, }; } @@ -220,11 +222,14 @@ impl Storable for Account { #[cfg(test)] mod tests { use assert_matches::assert_matches; + use ic_stable_structures::Storable; use std::str::FromStr; use candid::Principal; - use crate::icrc1::account::{Account, ICRC1TextReprError}; + use crate::icrc1::account::{ + Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT, MAX_SERIALIZATION_LEN, + }; #[test] fn test_account_display_default_subaccount() { @@ -358,4 +363,24 @@ mod tests { Err(ICRC1TextReprError::InvalidChecksum { expected: _ }) ); } + + #[test] + fn test_account_max_serialization_length() { + let owner = + Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + .unwrap() + .try_into() + .unwrap(), + ); + let account = Account { owner, subaccount }; + let serialized_len = account.to_bytes().len(); + assert_eq!( + serialized_len, + 1 + DEFAULT_SUBACCOUNT.len() + Principal::MAX_LENGTH_IN_BYTES + ); + assert_eq!(serialized_len as u32, MAX_SERIALIZATION_LEN); + } } From ed7731d51906b12d6c84d2ce90e1d697c3bc22bd Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 15:21:15 +0000 Subject: [PATCH 022/124] test account serialization --- .../icrc-ledger-types/src/icrc1/account.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index bc3129fbed4..5aa62a4d49e 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -383,4 +383,34 @@ mod tests { ); assert_eq!(serialized_len as u32, MAX_SERIALIZATION_LEN); } + + #[test] + fn test_account_serialization() { + let owner = + Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + .unwrap() + .try_into() + .unwrap(), + ); + let account1 = Account { owner, subaccount }; + let account2 = Account { + owner, + subaccount: None, + }; + let account3 = Account { + owner: Principal::anonymous(), + subaccount, + }; + let account4 = Account { + owner: Principal::anonymous(), + subaccount: None, + }; + let accounts = vec![account1, account2, account3, account4]; + for account in accounts { + assert_eq!(Account::from_bytes(account.to_bytes()), account); + } + } } From 55855945e413485de52a982f33a627821ac88e25 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 24 Sep 2024 15:30:44 +0000 Subject: [PATCH 023/124] test allowance serialization --- .../ledger_core/src/approvals/tests.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/rs/rosetta-api/ledger_core/src/approvals/tests.rs b/rs/rosetta-api/ledger_core/src/approvals/tests.rs index c6ea7acb0b4..7ef9775936b 100644 --- a/rs/rosetta-api/ledger_core/src/approvals/tests.rs +++ b/rs/rosetta-api/ledger_core/src/approvals/tests.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use super::*; use crate::timestamp::TimeStamp; use crate::tokens::Tokens; +use ic_stable_structures::Storable; use std::cmp; fn ts(n: u64) -> TimeStamp { @@ -619,3 +620,28 @@ fn expected_allowance_if_zero_no_approval() { } ); } + +#[test] +fn allowance_serialization() { + let allowance1 = Allowance { + amount: tokens(100), + expires_at: None, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), + }; + let allowance2 = Allowance { + amount: tokens(300), + expires_at: Some(ts(5)), + arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), + }; + + let allowances = vec![allowance1, allowance2]; + for allowance in allowances { + let new_allowance: Allowance = Allowance::from_bytes(allowance.to_bytes()); + assert_eq!(new_allowance.amount, allowance.amount); + assert_eq!(new_allowance.expires_at, allowance.expires_at); + assert_eq!( + new_allowance.arrived_at, + TimeStamp::from_nanos_since_unix_epoch(0) + ); + } +} From eb830dbcb43547af0eca6be063cb78d20eabea28 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 25 Sep 2024 11:13:41 +0000 Subject: [PATCH 024/124] account for pre_upgrade instructions --- rs/rosetta-api/icrc1/ledger/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index a310a7e43bc..38d03ef6616 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -223,7 +223,7 @@ fn post_upgrade(args: Option) { ledger.approvals.allowances_data.clear_arrivals(); }); const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 20_000_000_000; - migrate_next_part(MAX_INSTRUCTIONS_PER_UPGRADE); + migrate_next_part(MAX_INSTRUCTIONS_PER_UPGRADE - pre_upgrade_instructions_consumed); let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; From 247d2e32bb7ba94ae7b74388b28c555a01a1c564 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 25 Sep 2024 13:04:43 +0000 Subject: [PATCH 025/124] additional ready checks --- rs/rosetta-api/icrc1/ledger/src/lib.rs | 14 +++++++++++++- rs/rosetta-api/icrc1/ledger/src/main.rs | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/lib.rs b/rs/rosetta-api/icrc1/ledger/src/lib.rs index da701b5981e..097facef1d9 100644 --- a/rs/rosetta-api/icrc1/ledger/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/src/lib.rs @@ -383,7 +383,7 @@ pub struct Ledger { #[serde(default)] pub approvals: LedgerAllowances, #[serde(default)] - stable_approvals: AllowanceTable, + pub stable_approvals: AllowanceTable, blockchain: Blockchain, minting_account: Account, @@ -543,18 +543,30 @@ impl LedgerContext for Ledger { type Tokens = Tokens; fn balances(&self) -> &Balances { + if !self.is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } &self.balances } fn balances_mut(&mut self) -> &mut Balances { + if !self.is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } &mut self.balances } fn approvals(&self) -> &AllowanceTable { + if !self.is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } &self.stable_approvals } fn approvals_mut(&mut self) -> &mut AllowanceTable { + if !self.is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } &mut self.stable_approvals } diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 38d03ef6616..6d408c342ec 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -248,7 +248,7 @@ fn migrate_next_part(instruction_limit: u64) { match ledger.approvals.allowances_data.pop_first_allowance() { Some((account_spender, allowance)) => { ledger - .approvals_mut() + .stable_approvals .allowances_data .set_allowance(account_spender, allowance); migrated_allowances += 1; @@ -263,7 +263,7 @@ fn migrate_next_part(instruction_limit: u64) { match ledger.approvals.allowances_data.pop_first_expiry() { Some((timestamp, account_spender)) => { ledger - .approvals_mut() + .stable_approvals .allowances_data .insert_expiry(timestamp, account_spender); migrated_expirations += 1; From dbc7f32156931a8790244988519fa0137ca9f35e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 25 Sep 2024 13:06:58 +0000 Subject: [PATCH 026/124] remove is_migrating --- rs/rosetta-api/icrc1/ledger/src/lib.rs | 4 ---- rs/rosetta-api/icrc1/ledger/src/main.rs | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/lib.rs b/rs/rosetta-api/icrc1/ledger/src/lib.rs index 097facef1d9..27374172066 100644 --- a/rs/rosetta-api/icrc1/ledger/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/src/lib.rs @@ -527,10 +527,6 @@ impl Ledger { ledger } - pub fn is_migrating(&self) -> bool { - !matches!(self.state, LedgerState::Ready) - } - pub fn is_ready(&self) -> bool { matches!(self.state, LedgerState::Ready) } diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index 6d408c342ec..e288bd1577b 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -277,7 +277,7 @@ fn migrate_next_part(instruction_limit: u64) { } let msg = format!("Number of elements migrated: allowances:{migrated_allowances} expirations:{migrated_expirations}. Instructions used {}.", instruction_counter()); - if ledger.is_migrating() { + if !ledger.is_ready() { ic_cdk::println!("Migration partially done. Scheduling the next part. {msg}"); log!( &LOG, From 9559981c0778e11c6db3811bb71b93d3b6dd4dc7 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 25 Sep 2024 15:26:58 +0000 Subject: [PATCH 027/124] migration steps metric --- rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs | 6 +++++- rs/rosetta-api/icrc1/ledger/src/main.rs | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index e47300f638d..9a7b59d4bc6 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -1,4 +1,5 @@ use crate::in_memory_ledger::{verify_ledger_state, InMemoryLedger}; +use crate::metrics::parse_metric; use candid::{CandidType, Decode, Encode, Int, Nat, Principal}; use ic_agent::identity::{BasicIdentity, Identity}; use ic_base_types::PrincipalId; @@ -2654,7 +2655,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( const APPROVE_AMOUNT: u64 = 150_000; let expiration = - system_time_to_nanos(env.time()) + Duration::from_secs(5 * 3600).as_nanos() as u64; + system_time_to_nanos(env.time()) + Duration::from_secs(5000 * 3600).as_nanos() as u64; let mut expected_allowances = vec![]; @@ -2710,6 +2711,9 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( // Test if the old serialized approvals and balances are correctly deserialized test_upgrade(ledger_wasm_current.clone(), balances.clone()); + let stable_upgrade_migration_steps = parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + assert_eq!(stable_upgrade_migration_steps, 1); + // Add some more approvals for a1 in &accounts { for a2 in &additional_accounts { diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index e288bd1577b..e4f25048dbd 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -79,6 +79,7 @@ thread_local! { static LEDGER: RefCell> = const { RefCell::new(None) }; static PRE_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; static POST_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; + static STABLE_UPGRADE_MIGRATION_STEPS: RefCell = const { RefCell::new(0) }; } declare_log_buffer!(name = LOG, capacity = 1000); @@ -231,6 +232,7 @@ fn post_upgrade(args: Option) { } fn migrate_next_part(instruction_limit: u64) { + STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow_mut() += 1); let mut migrated_allowances = 0; let mut migrated_expirations = 0; @@ -321,6 +323,7 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i .value(&[("canister", "icrc1-ledger")], cycle_balance)?; let pre_upgrade_instructions = PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow()); let post_upgrade_instructions = POST_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow()); + let stable_upgrade_migration_steps = STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow()); w.encode_gauge( "ledger_pre_upgrade_instructions_consumed", pre_upgrade_instructions as f64, @@ -336,6 +339,11 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i pre_upgrade_instructions.saturating_add(post_upgrade_instructions) as f64, "Total number of instructions consumed during the last upgrade.", )?; + w.encode_gauge( + "ledger_stable_upgrade_migration_steps", + stable_upgrade_migration_steps as f64, + "Number of steps used to migrate data to stable structures.", + )?; Access::with_ledger(|ledger| { w.encode_gauge( From 35924ed8fbb1b3db5fa28e10a84fdf93254dc59a Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 25 Sep 2024 15:35:43 +0000 Subject: [PATCH 028/124] clippy --- rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index 9a7b59d4bc6..4f7e7207603 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2711,7 +2711,8 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( // Test if the old serialized approvals and balances are correctly deserialized test_upgrade(ledger_wasm_current.clone(), balances.clone()); - let stable_upgrade_migration_steps = parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + let stable_upgrade_migration_steps = + parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); assert_eq!(stable_upgrade_migration_steps, 1); // Add some more approvals From 5b7c9a3722862d623bc9414b73158c4e52e48112 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 26 Sep 2024 12:45:44 +0000 Subject: [PATCH 029/124] update limits, do not write in pre_upgrade if not ready --- rs/rosetta-api/icrc1/ledger/src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index e4f25048dbd..ef5774b5ca5 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -133,6 +133,14 @@ fn pre_upgrade() { #[cfg(feature = "canbench-rs")] let _p = canbench_rs::bench_scope("pre_upgrade"); + Access::with_ledger(|ledger| { + if !ledger.is_ready() { + // This means that migration did not complete and the correct state + // of the ledger is still in UPGRADES_MEMORY. + return + } + }); + let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { Access::with_ledger(|ledger| { @@ -223,7 +231,7 @@ fn post_upgrade(args: Option) { ledger.state = LedgerState::Migrating(LedgerField::Allowances); ledger.approvals.allowances_data.clear_arrivals(); }); - const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 20_000_000_000; + const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; migrate_next_part(MAX_INSTRUCTIONS_PER_UPGRADE - pre_upgrade_instructions_consumed); let end = ic_cdk::api::instruction_counter(); @@ -285,7 +293,7 @@ fn migrate_next_part(instruction_limit: u64) { &LOG, "Migration partially done. Scheduling the next part. {msg}" ); - const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 2_000_000_000; + const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; ic_cdk_timers::set_timer(Duration::from_secs(0), || { migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) }); From 11324d73d00433e3c2c44fbbe57c4c3251faa8d9 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 27 Sep 2024 08:37:48 +0000 Subject: [PATCH 030/124] clippy --- rs/rosetta-api/icrc1/ledger/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index ef5774b5ca5..a8c15f54c30 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -137,9 +137,9 @@ fn pre_upgrade() { if !ledger.is_ready() { // This means that migration did not complete and the correct state // of the ledger is still in UPGRADES_MEMORY. - return + return; } - }); + }); let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { From 74343044f039d37bdde9fec515e8c05540879560 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 30 Sep 2024 08:18:38 +0000 Subject: [PATCH 031/124] require memory manager --- rs/rosetta-api/icrc1/ledger/src/main.rs | 61 ++++++++++--------------- 1 file changed, 24 insertions(+), 37 deletions(-) diff --git a/rs/rosetta-api/icrc1/ledger/src/main.rs b/rs/rosetta-api/icrc1/ledger/src/main.rs index a8c15f54c30..02dfc00c014 100644 --- a/rs/rosetta-api/icrc1/ledger/src/main.rs +++ b/rs/rosetta-api/icrc1/ledger/src/main.rs @@ -133,17 +133,16 @@ fn pre_upgrade() { #[cfg(feature = "canbench-rs")] let _p = canbench_rs::bench_scope("pre_upgrade"); - Access::with_ledger(|ledger| { - if !ledger.is_ready() { - // This means that migration did not complete and the correct state - // of the ledger is still in UPGRADES_MEMORY. - return; - } - }); - let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { Access::with_ledger(|ledger| { + if !ledger.is_ready() { + // This means that migration did not complete and the correct state + // of the ledger is still in UPGRADES_MEMORY. + ic_cdk::println!("Ledger not ready, skipping write to UPGRADES_MEMORY."); + log!(&LOG, "Ledger not ready, skipping write to UPGRADES_MEMORY."); + return; + } let writer = Writer::new(bs, 0); let mut buffered_writer = BufferedWriter::new(BUFFER_SIZE, writer); ciborium::ser::into_writer(ledger, &mut buffered_writer) @@ -176,43 +175,31 @@ fn post_upgrade(args: Option) { let mut pre_upgrade_instructions_consumed = 0; if !memory_manager_found { - let mut stable_reader = StableReader::default(); - LEDGER.with(|cell| { - *cell.borrow_mut() = Some( - ciborium::de::from_reader(&mut stable_reader) - .expect("failed to decode ledger state"), - ); - }); + let msg = + "Cannot upgrade from scratch stable memory, please upgrade to memory manager first."; + ic_cdk::println!("{msg}"); + panic!("{msg}"); + } + + let state: Ledger = UPGRADES_MEMORY.with_borrow(|bs| { + let reader = Reader::new(bs, 0); + let mut buffered_reader = BufferedReader::new(BUFFER_SIZE, reader); + let state = ciborium::de::from_reader(&mut buffered_reader).expect( + "Failed to read the Ledger state from memory manager managed stable structures", + ); let mut pre_upgrade_instructions_counter_bytes = [0u8; 8]; pre_upgrade_instructions_consumed = - match stable_reader.read_exact(&mut pre_upgrade_instructions_counter_bytes) { + match buffered_reader.read_exact(&mut pre_upgrade_instructions_counter_bytes) { Ok(_) => u64::from_le_bytes(pre_upgrade_instructions_counter_bytes), Err(_) => { // If upgrading from a version that didn't write the instructions counter to stable memory 0u64 } }; - } else { - let state: Ledger = UPGRADES_MEMORY.with_borrow(|bs| { - let reader = Reader::new(bs, 0); - let mut buffered_reader = BufferedReader::new(BUFFER_SIZE, reader); - let state = ciborium::de::from_reader(&mut buffered_reader).expect( - "Failed to read the Ledger state from memory manager managed stable structures", - ); - let mut pre_upgrade_instructions_counter_bytes = [0u8; 8]; - pre_upgrade_instructions_consumed = - match buffered_reader.read_exact(&mut pre_upgrade_instructions_counter_bytes) { - Ok(_) => u64::from_le_bytes(pre_upgrade_instructions_counter_bytes), - Err(_) => { - // If upgrading from a version that didn't write the instructions counter to stable memory - 0u64 - } - }; - state - }); - ic_cdk::println!("Successfully read state from memory manager managed stable structures"); - LEDGER.with_borrow_mut(|ledger| *ledger = Some(state)); - } + state + }); + ic_cdk::println!("Successfully read state from memory manager managed stable structures"); + LEDGER.with_borrow_mut(|ledger| *ledger = Some(state)); if let Some(args) = args { match args { From 1ae73235aa2dd4bb934c29d95dd25cef125cdbd7 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 30 Sep 2024 09:54:58 +0000 Subject: [PATCH 032/124] test endpoints disabled during migration and test failed migration --- .../icrc1/ledger/sm-tests/src/lib.rs | 217 ++++++++++++++++++ rs/rosetta-api/icrc1/ledger/tests/tests.rs | 18 ++ 2 files changed, 235 insertions(+) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index 4f7e7207603..df7df508dcd 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -35,6 +35,7 @@ use icrc_ledger_types::icrc21::requests::{ use icrc_ledger_types::icrc21::responses::{ConsentInfo, ConsentMessage}; use icrc_ledger_types::icrc3; use icrc_ledger_types::icrc3::archive::ArchiveInfo; +use icrc_ledger_types::icrc3::archive::GetArchivesArgs; use icrc_ledger_types::icrc3::blocks::{ BlockRange, GenericBlock as IcrcBlock, GetBlocksRequest, GetBlocksResponse, }; @@ -2748,6 +2749,222 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } } +pub fn icrc1_test_stable_migration_endpoints_disabled( + ledger_wasm_mainnet: Vec, + ledger_wasm_current: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + let account = Account::from(PrincipalId::new_user_test_id(1).0); + let initial_balances = vec![(account, 100_000_000u64)]; + + // Setup ledger as it is deployed on the mainnet. + let (env, canister_id) = setup(ledger_wasm_mainnet, encode_init_args, initial_balances); + + const APPROVE_AMOUNT: u64 = 150_000; + + for i in 2..40 { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, APPROVE_AMOUNT); + send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); + } + + env.upgrade_canister( + canister_id, + ledger_wasm_current, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + let transfer_args = TransferArg { + from_subaccount: None, + to: Account::from(PrincipalId::new_user_test_id(2).0), + fee: None, + created_at_time: None, + amount: Nat::from(1u64), + memo: None, + }; + let approve_args = default_approve_args( + Account::from(PrincipalId::new_user_test_id(200).0), + APPROVE_AMOUNT, + ); + let transfer_from_args = TransferFromArgs { + spender_subaccount: None, + from: account.into(), + to: Account::from(PrincipalId::new_user_test_id(2).0), + amount: Nat::from(1u64), + fee: None, + memo: None, + created_at_time: None, + }; + let allowance_args = AllowanceArgs { + account, + spender: account, + }; + let get_transactions_args = GetTransactionsRequest { + start: Nat::from(1u64), + length: Nat::from(1u64), + }; + let get_archives_args = GetArchivesArgs { from: None }; + let blocks_req_vec: Vec = vec![]; + let consent_msg_args = ConsentMessageRequest { + method: "icrc1_transfer".to_owned(), + arg: Encode!(&transfer_args).unwrap(), + user_preferences: ConsentMessageSpec { + metadata: ConsentMessageMetadata { + language: "en".to_string(), + utc_offset_minutes: Some(60), + }, + device_spec: Some(DisplayMessageType::GenericDisplay), + }, + }; + + let test_endpoint = |endpoint_name: &str, args: Vec, expect_error: bool| { + let result = env.execute_ingress_as(account.owner.into(), canister_id, endpoint_name, args); + if expect_error { + result + .unwrap_err() + .assert_contains(ErrorCode::CanisterCalledTrap, "The Ledger is not ready."); + } else { + assert!(result.is_ok()); + } + }; + + test_endpoint("icrc1_transfer", Encode!(&transfer_args).unwrap(), true); + test_endpoint("icrc2_approve", Encode!(&approve_args).unwrap(), true); + test_endpoint( + "icrc2_transfer_from", + Encode!(&transfer_from_args).unwrap(), + true, + ); + test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), true); + test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), true); + test_endpoint("icrc1_total_supply", Encode!().unwrap(), true); + test_endpoint("archives", Encode!().unwrap(), true); + test_endpoint( + "get_transactions", + Encode!(&get_transactions_args).unwrap(), + true, + ); + test_endpoint("get_blocks", Encode!(&get_transactions_args).unwrap(), true); + test_endpoint("get_data_certificate", Encode!().unwrap(), true); + test_endpoint( + "icrc3_get_archives", + Encode!(&get_archives_args).unwrap(), + true, + ); + test_endpoint("icrc3_get_tip_certificate", Encode!().unwrap(), true); + test_endpoint("icrc3_get_blocks", Encode!(&blocks_req_vec).unwrap(), true); + test_endpoint( + "icrc21_canister_call_consent_message", + Encode!(&consent_msg_args).unwrap(), + true, + ); + + wait_ledger_ready(&env, canister_id, 10); + + test_endpoint("icrc1_transfer", Encode!(&transfer_args).unwrap(), false); + test_endpoint("icrc2_approve", Encode!(&approve_args).unwrap(), false); + test_endpoint( + "icrc2_transfer_from", + Encode!(&transfer_from_args).unwrap(), + false, + ); + test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), false); + test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), false); + test_endpoint("icrc1_total_supply", Encode!().unwrap(), false); + test_endpoint("archives", Encode!().unwrap(), false); + test_endpoint( + "get_transactions", + Encode!(&get_transactions_args).unwrap(), + false, + ); + test_endpoint( + "get_blocks", + Encode!(&get_transactions_args).unwrap(), + false, + ); + test_endpoint("get_data_certificate", Encode!().unwrap(), false); + test_endpoint( + "icrc3_get_archives", + Encode!(&get_archives_args).unwrap(), + false, + ); + test_endpoint("icrc3_get_tip_certificate", Encode!().unwrap(), false); + test_endpoint("icrc3_get_blocks", Encode!(&blocks_req_vec).unwrap(), false); + test_endpoint( + "icrc21_canister_call_consent_message", + Encode!(&consent_msg_args).unwrap(), + false, + ); +} + +pub fn test_incomplete_migration( + ledger_wasm_mainnet: Vec, + ledger_wasm_current: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + let account = Account::from(PrincipalId::new_user_test_id(1).0); + let initial_balances = vec![(account, 100_000_000u64)]; + + // Setup ledger as it is deployed on the mainnet. + let (env, canister_id) = setup( + ledger_wasm_mainnet.clone(), + encode_init_args, + initial_balances, + ); + + const APPROVE_AMOUNT: u64 = 150_000; + + const NUM_APPROVALS: u64 = 20; + + let send_approvals = || { + for i in 2..2 + NUM_APPROVALS { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, APPROVE_AMOUNT); + send_approval(&env, canister_id, account.owner, &approve_args) + .expect("approval failed"); + } + }; + + send_approvals(); + + let check_approvals = |expected_allowance: u64| { + for i in 2..2 + NUM_APPROVALS { + let allowance = get_allowance( + &env, + canister_id, + account, + Account::from(PrincipalId::new_user_test_id(i).0), + ); + assert_eq!(allowance.allowance, Nat::from(expected_allowance)); + } + }; + + check_approvals(APPROVE_AMOUNT); + + env.upgrade_canister( + canister_id, + ledger_wasm_current, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + // Downgrade to mainnet without waiting for the migration to complete. + env.upgrade_canister( + canister_id, + ledger_wasm_mainnet, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + // All approvals should still be in UPGRADES_MEMORY and downgrade should succeed. + check_approvals(APPROVE_AMOUNT); +} + pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { ApproveArgs { from_subaccount: None, diff --git a/rs/rosetta-api/icrc1/ledger/tests/tests.rs b/rs/rosetta-api/icrc1/ledger/tests/tests.rs index b419148af9d..dbd249aeee0 100644 --- a/rs/rosetta-api/icrc1/ledger/tests/tests.rs +++ b/rs/rosetta-api/icrc1/ledger/tests/tests.rs @@ -411,6 +411,24 @@ fn icrc1_test_upgrade_serialization_fixed_tx() { ); } +#[test] +fn icrc1_test_stable_migration_endpoints_disabled() { + ic_icrc1_ledger_sm_tests::icrc1_test_stable_migration_endpoints_disabled( + ledger_mainnet_wasm(), + ledger_wasm(), + encode_init_args, + ); +} + +#[test] +fn icrc1_test_incomplete_migration() { + ic_icrc1_ledger_sm_tests::test_incomplete_migration( + ledger_mainnet_wasm(), + ledger_wasm(), + encode_init_args, + ); +} + mod metrics { use crate::{encode_init_args, encode_upgrade_args, ledger_wasm}; use ic_icrc1_ledger_sm_tests::metrics::LedgerSuiteType; From 9e50e99e06736f45dde668cb54e5d1c86fe15c7d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 30 Sep 2024 10:30:37 +0000 Subject: [PATCH 033/124] clippy --- rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs index df7df508dcd..0ce09f5c491 100644 --- a/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs +++ b/rs/rosetta-api/icrc1/ledger/sm-tests/src/lib.rs @@ -2791,7 +2791,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( ); let transfer_from_args = TransferFromArgs { spender_subaccount: None, - from: account.into(), + from: account, to: Account::from(PrincipalId::new_user_test_id(2).0), amount: Nat::from(1u64), fee: None, From 1086b5ae44f14f4e528f1a31816e5df2cdaa4c9e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 7 Oct 2024 14:47:51 +0000 Subject: [PATCH 034/124] build fix --- mainnet-canisters.bzl | 4 +-- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 8 +++--- rs/ledger_suite/tests/sm-tests/src/lib.rs | 32 ----------------------- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/mainnet-canisters.bzl b/mainnet-canisters.bzl index 7bd9f2df6c8..3b3a88c6a60 100644 --- a/mainnet-canisters.bzl +++ b/mainnet-canisters.bzl @@ -23,9 +23,9 @@ CANISTER_NAME_TO_WASM_METADATA = { "sns_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"), "sns_archive": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "9476aa71bcee621aba93a3d7c115c543f42c543de840da3224c5f70a32dbfe4d"), "ck_btc_index": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61"), - "ck_btc_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"), + "ck_btc_ledger": ("d1db89ed78716b9258a8ec75bd6f53aa067a5be7", "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577"), "ck_eth_index": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "de250f08dc7e699144b73514f55fbbb3a3f8cd97abf0f7ae31d9fb7494f55234"), - "ck_eth_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "e6072806ae22868ee09c07923d093b1b0b687dba540d22cfc1e1a5392bfcca46"), + "ck_eth_ledger": ("d1db89ed78716b9258a8ec75bd6f53aa067a5be7", "566f1c1ae68cca5618e74cdb2239d989084d92237b04a31f7898182ad0547a93"), } def canister_url(git_commit_id, filename): diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index 695f90ef05e..d07e4a41833 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -48,7 +48,7 @@ package(default_visibility = ["//visibility:public"]) "", [], [ - "//rs/rosetta-api/icrc1/tokens_u64", + "//rs/ledger_suite/icrc1/tokens_u64", ], ), ( @@ -56,7 +56,7 @@ package(default_visibility = ["//visibility:public"]) "_u256", ["u256-tokens"], [ - "//rs/rosetta-api/icrc1/tokens_u256", + "//rs/ledger_suite/icrc1/tokens_u256", ], ), ( @@ -64,7 +64,7 @@ package(default_visibility = ["//visibility:public"]) "", ["canbench-rs"], [ - "//rs/rosetta-api/icrc1/tokens_u64", + "//rs/ledger_suite/icrc1/tokens_u64", "@crate_index//:canbench-rs", ], ), @@ -76,7 +76,7 @@ package(default_visibility = ["//visibility:public"]) "u256-tokens", ], [ - "//rs/rosetta-api/icrc1/tokens_u256", + "//rs/ledger_suite/icrc1/tokens_u256", "@crate_index//:canbench-rs", ], ), diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 78e60e7daad..f3fbf613850 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2678,38 +2678,6 @@ pub fn test_upgrade_serialization( test_upgrade(ledger_wasm_nextmigrationversionmemorymanager.clone()); // Test upgrade to memory manager again test_upgrade(ledger_wasm_nextmigrationversionmemorymanager.clone()); - - // Current mainnet ICP wasm (V0) cannot deserialize from memory manager, but ICRC (V1) can - match env.upgrade_canister( - ledger_id, - ledger_wasm_mainnet.clone(), - upgrade_args.clone(), - ) { - Ok(_) => { - if !downgrade_to_mainnet_should_succeed { - panic!("Downgrade from memory manager directly to mainnet should fail (since mainnet is V0)!") - } else { - // In case this succeeded, we need to upgrade the ledger back to - // the next version (via the current version), so that the - // subsequent upgrade is from - // `ledger_wasm_nextmigrationversionmemorymanager` to - // `ledger_wasm_current`, rather than from `ledger_wasm_mainnet` to - // `ledger_wasm_current` (currently, from V2 -> V1, rather than from - // V0 -> V1). - test_upgrade(ledger_wasm_current.clone()); - test_upgrade(ledger_wasm_nextmigrationversionmemorymanager); - } - } - Err(e) => { - if downgrade_to_mainnet_should_succeed { - panic!("Downgrade from memory manager to mainnet should succeed (since mainnet is V1), but failed with error: {}", e) - } - assert!( - e.description().contains("failed to decode ledger state") - || e.description().contains("Decoding stable memory failed") - ) - } - }; } // Test deserializing from memory manager test_upgrade(ledger_wasm_current.clone()); From 8811688dda32a94d27bf5adab41da3d802fc2974 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 7 Oct 2024 16:45:37 +0000 Subject: [PATCH 035/124] clippy --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index f3fbf613850..6bdf03ac564 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2622,12 +2622,7 @@ pub fn test_upgrade_serialization( const TOTAL_TX_COUNT: usize = INITIAL_TX_BATCH_SIZE + 8 * ADDITIONAL_TX_BATCH_SIZE; runner .run( - &(valid_transactions_strategy( - minter, - FEE, - TOTAL_TX_COUNT, - now, - ).no_shrink(),), + &(valid_transactions_strategy(minter, FEE, TOTAL_TX_COUNT, now).no_shrink(),), |(transactions,)| { let env = StateMachine::new(); env.set_time(now); From ac7ec29293b5d458c609e78e8dec96b54fbe6a39 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 7 Oct 2024 18:19:36 +0000 Subject: [PATCH 036/124] remove test upgrade from first version --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 57 --------------------- 1 file changed, 57 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 5bdc2f21385..2ad3a644d7f 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -526,63 +526,6 @@ fn balance_of(env: &StateMachine, ledger_id: CanisterId, account: Account) -> u6 Decode!(&res, Nat).unwrap().0.to_u64().unwrap() } -#[cfg_attr(feature = "u256-tokens", ignore)] -#[test] -fn test_upgrade_from_first_version() { - let env = StateMachine::new(); - - let ledger_wasm_first_version = - std::fs::read(std::env::var("IC_ICRC1_LEDGER_FIRST_VERSION_WASM_PATH").unwrap()).unwrap(); - let init_args = Encode!(&LegacyInitArgs { - minting_account: MINTER, - fee_collector_account: None, - initial_balances: vec![], - transfer_fee: FEE, - token_name: TOKEN_NAME.to_string(), - token_symbol: TOKEN_SYMBOL.to_string(), - metadata: vec![ - MetadataValue::entry(NAT_META_KEY, NAT_META_VALUE), - MetadataValue::entry(INT_META_KEY, INT_META_VALUE), - MetadataValue::entry(TEXT_META_KEY, TEXT_META_VALUE), - MetadataValue::entry(BLOB_META_KEY, BLOB_META_VALUE), - ], - archive_options: ArchiveOptions { - trigger_threshold: ARCHIVE_TRIGGER_THRESHOLD as usize, - num_blocks_to_archive: NUM_BLOCKS_TO_ARCHIVE as usize, - node_max_memory_size_bytes: None, - max_message_size_bytes: None, - controller_id: PrincipalId::new_user_test_id(100), - more_controller_ids: None, - cycles_for_archive_creation: None, - max_transactions_per_response: None, - }, - }) - .unwrap(); - let ledger_id = env - .install_canister(ledger_wasm_first_version, init_args, None) - .unwrap(); - transfer(&env, ledger_id, MINTER, account(1), 1_000_000); - transfer(&env, ledger_id, MINTER, account(1), 2_000_000); - transfer(&env, ledger_id, MINTER, account(2), 3_000_000); - transfer(&env, ledger_id, account(1), account(3), 1_000_000); - let balance_1 = balance_of(&env, ledger_id, account(1)); - let balance_2 = balance_of(&env, ledger_id, account(2)); - let balance_3 = balance_of(&env, ledger_id, account(3)); - - let upgrade_args = Encode!(&LedgerArgument::Upgrade(None)).unwrap(); - env.upgrade_canister(ledger_id, ledger_wasm(), upgrade_args) - .expect("Unable to upgrade the ledger canister"); - assert_eq!(balance_1, balance_of(&env, ledger_id, account(1))); - assert_eq!(balance_2, balance_of(&env, ledger_id, account(2))); - assert_eq!(balance_3, balance_of(&env, ledger_id, account(3))); - - // check that transfer works - transfer(&env, ledger_id, MINTER, account(1), 1_000_000); - transfer(&env, ledger_id, MINTER, account(1), 2_000_000); - transfer(&env, ledger_id, MINTER, account(2), 3_000_000); - transfer(&env, ledger_id, account(1), account(3), 1_000_000); -} - #[test] fn test_icrc2_feature_flag_doesnt_disable_icrc2_endpoints() { // Disable ICRC-2 and check the endpoints still work From c8b94d1bcef05f907d0ebc848c35061b534816bf Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 8 Oct 2024 14:28:47 +0000 Subject: [PATCH 037/124] feature flag with lower instuction limits --- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 21 +++++++++++++++++++++ rs/ledger_suite/icrc1/ledger/Cargo.toml | 1 + rs/ledger_suite/icrc1/ledger/src/main.rs | 16 +++++++++++++--- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 8 ++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index d07e4a41833..59d70491251 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -173,6 +173,25 @@ package(default_visibility = ["//visibility:public"]) "//rs/ledger_suite/icrc1/tokens_u256", ], ), + ( + "_lowupgradeinstructionlimits", + ["low-upgrade-instruction-limits"], + [ + ":ledger", + "//rs/ledger_suite/icrc1/tokens_u64", + ], + ), + ( + "_u256_lowupgradeinstructionlimits", + [ + "low-upgrade-instruction-limits", + "u256-tokens", + ], + [ + ":ledger_u256", + "//rs/ledger_suite/icrc1/tokens_u256", + ], + ), ] ] @@ -196,6 +215,7 @@ rust_test( data = [ ":block.cddl", ":ledger_canister" + name_suffix + ".wasm", + ":ledger_canister" + name_suffix + "_lowupgradeinstructionlimits.wasm", "//rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz", "@ic-icrc1-ledger-first-version.wasm.gz//file", "@mainnet_ckbtc_ic-icrc1-ledger//file", @@ -210,6 +230,7 @@ rust_test( "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)", "IC_ICRC1_LEDGER_FIRST_VERSION_WASM_PATH": "$(rootpath @ic-icrc1-ledger-first-version.wasm.gz//file)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath :ledger_canister" + name_suffix + ".wasm)", + "IC_ICRC1_LEDGER_WASM_INSTR_LIMITS_PATH": "$(rootpath :ledger_canister" + name_suffix + "_lowupgradeinstructionlimits.wasm)", }, deps = [ # Keep sorted. diff --git a/rs/ledger_suite/icrc1/ledger/Cargo.toml b/rs/ledger_suite/icrc1/ledger/Cargo.toml index a12d611ab0c..4913352156e 100644 --- a/rs/ledger_suite/icrc1/ledger/Cargo.toml +++ b/rs/ledger_suite/icrc1/ledger/Cargo.toml @@ -56,3 +56,4 @@ default = [] get-blocks-disabled = [] u256-tokens = ["dep:ic-icrc1-tokens-u256"] canbench-rs = ["dep:canbench-rs", "dep:assert_matches"] +low-upgrade-instruction-limits = [] diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 02dfc00c014..faa794451d0 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -157,6 +157,16 @@ fn pre_upgrade() { }); } +#[cfg(not(feature = "low-upgrade-instruction-limits"))] +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; +#[cfg(not(feature = "low-upgrade-instruction-limits"))] +const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; + +#[cfg(feature = "low-upgrade-instruction-limits")] +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 0; +#[cfg(feature = "low-upgrade-instruction-limits")] +const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 500_000; + #[post_upgrade] fn post_upgrade(args: Option) { #[cfg(feature = "canbench-rs")] @@ -218,8 +228,9 @@ fn post_upgrade(args: Option) { ledger.state = LedgerState::Migrating(LedgerField::Allowances); ledger.approvals.allowances_data.clear_arrivals(); }); - const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; - migrate_next_part(MAX_INSTRUCTIONS_PER_UPGRADE - pre_upgrade_instructions_consumed); + migrate_next_part( + MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), + ); let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; @@ -280,7 +291,6 @@ fn migrate_next_part(instruction_limit: u64) { &LOG, "Migration partially done. Scheduling the next part. {msg}" ); - const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; ic_cdk_timers::set_timer(Duration::from_secs(0), || { migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) }); diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 2ad3a644d7f..497e59d3315 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -88,6 +88,10 @@ fn ledger_wasm() -> Vec { ) } +fn ledger_wasm_lowupgradeinstructionlimits() -> Vec { + std::fs::read(std::env::var("IC_ICRC1_LEDGER_WASM_INSTR_LIMITS_PATH").unwrap()).unwrap() +} + fn archive_wasm() -> Vec { ic_test_utilities_load_wasm::load_wasm( PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) @@ -426,7 +430,7 @@ fn icrc1_test_upgrade_serialization_fixed_tx() { fn icrc1_test_stable_migration_endpoints_disabled() { ic_icrc1_ledger_sm_tests::icrc1_test_stable_migration_endpoints_disabled( ledger_mainnet_wasm(), - ledger_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), encode_init_args, ); } @@ -435,7 +439,7 @@ fn icrc1_test_stable_migration_endpoints_disabled() { fn icrc1_test_incomplete_migration() { ic_icrc1_ledger_sm_tests::test_incomplete_migration( ledger_mainnet_wasm(), - ledger_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), encode_init_args, ); } From 9e67753c38e93e90b525a02b3fe2ad6317375d24 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 11 Oct 2024 11:48:57 +0000 Subject: [PATCH 038/124] restore mainnet canisters --- mainnet-canisters.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mainnet-canisters.bzl b/mainnet-canisters.bzl index 3b3a88c6a60..7bd9f2df6c8 100644 --- a/mainnet-canisters.bzl +++ b/mainnet-canisters.bzl @@ -23,9 +23,9 @@ CANISTER_NAME_TO_WASM_METADATA = { "sns_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"), "sns_archive": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "9476aa71bcee621aba93a3d7c115c543f42c543de840da3224c5f70a32dbfe4d"), "ck_btc_index": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61"), - "ck_btc_ledger": ("d1db89ed78716b9258a8ec75bd6f53aa067a5be7", "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577"), + "ck_btc_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"), "ck_eth_index": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "de250f08dc7e699144b73514f55fbbb3a3f8cd97abf0f7ae31d9fb7494f55234"), - "ck_eth_ledger": ("d1db89ed78716b9258a8ec75bd6f53aa067a5be7", "566f1c1ae68cca5618e74cdb2239d989084d92237b04a31f7898182ad0547a93"), + "ck_eth_ledger": ("d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "e6072806ae22868ee09c07923d093b1b0b687dba540d22cfc1e1a5392bfcca46"), } def canister_url(git_commit_id, filename): From d8ca322d3a6f3426ce8922564acad5efe8ddbf29 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 11 Oct 2024 12:06:13 +0000 Subject: [PATCH 039/124] update mainnet canisters json --- mainnet-canisters.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mainnet-canisters.json b/mainnet-canisters.json index 22eb6531a76..43f84abfd82 100644 --- a/mainnet-canisters.json +++ b/mainnet-canisters.json @@ -8,16 +8,16 @@ "sha256": "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61" }, "ck_btc_ledger": { - "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", - "sha256": "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c" + "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", + "sha256": "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577" }, "ck_eth_index": { "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "sha256": "de250f08dc7e699144b73514f55fbbb3a3f8cd97abf0f7ae31d9fb7494f55234" }, "ck_eth_ledger": { - "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", - "sha256": "e6072806ae22868ee09c07923d093b1b0b687dba540d22cfc1e1a5392bfcca46" + "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", + "sha256": "566f1c1ae68cca5618e74cdb2239d989084d92237b04a31f7898182ad0547a93" }, "cycles-minting": { "rev": "77f48ae63af09b6538b1bf33d3accc3bc74d14f8", From 43efb21429072e6d3b468171cfedde997192fa69 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 11 Oct 2024 13:33:42 +0000 Subject: [PATCH 040/124] implement stable structures storable for account --- packages/icrc-ledger-types/BUILD.bazel | 1 + packages/icrc-ledger-types/Cargo.toml | 1 + .../icrc-ledger-types/src/icrc1/account.rs | 107 +++++++++++++++++- 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/packages/icrc-ledger-types/BUILD.bazel b/packages/icrc-ledger-types/BUILD.bazel index 5861b827794..37acab199b5 100644 --- a/packages/icrc-ledger-types/BUILD.bazel +++ b/packages/icrc-ledger-types/BUILD.bazel @@ -22,6 +22,7 @@ rust_library( "@crate_index//:crc32fast", "@crate_index//:hex", "@crate_index//:ic-cdk", + "@crate_index//:ic-stable-structures", "@crate_index//:itertools", "@crate_index//:num-bigint", "@crate_index//:num-traits", diff --git a/packages/icrc-ledger-types/Cargo.toml b/packages/icrc-ledger-types/Cargo.toml index 91a51ec82fc..91b96ea0e65 100644 --- a/packages/icrc-ledger-types/Cargo.toml +++ b/packages/icrc-ledger-types/Cargo.toml @@ -15,6 +15,7 @@ base32 = "0.4.0" candid = { workspace = true } crc32fast = "1.2.0" hex = { workspace = true } +ic-stable-structures = { workspace = true } itertools = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 32f23b5838b..ceeca41d2e0 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -6,7 +6,10 @@ use std::{ use base32::Alphabet; use candid::{types::principal::PrincipalError, CandidType, Deserialize, Principal}; +use ic_stable_structures::{storable::Bound, Storable}; use serde::Serialize; +use std::borrow::Cow; +use std::io::{Cursor, Read}; pub type Subaccount = [u8; 32]; @@ -167,14 +170,66 @@ impl FromStr for Account { } } +const MAX_SERIALIZATION_LEN: u32 = 62; + +impl Storable for Account { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buffer: Vec = vec![]; + let mut buffer0: Vec = vec![]; + + if let Some(subaccount) = self.subaccount { + buffer0.extend(subaccount.as_slice()); + } + buffer0.extend(self.owner.as_slice()); + buffer.extend((buffer0.len() as u8).to_le_bytes()); + buffer.append(&mut buffer0); + + Cow::Owned(buffer) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + let mut cursor = Cursor::new(bytes); + + let mut len_bytes = [0u8; 1]; + cursor + .read_exact(&mut len_bytes) + .expect("Unable to read the len of the account"); + let mut len = u8::from_le_bytes(len_bytes); + let subaccount = if len >= 32 { + let mut subaccount_bytes = [0u8; 32]; + cursor + .read_exact(&mut subaccount_bytes) + .expect("Unable to read the bytes of the account's subaccount"); + len -= 32; + Some(subaccount_bytes) + } else { + None + }; + let mut owner_bytes = vec![0; len as usize]; + cursor + .read_exact(&mut owner_bytes) + .expect("Unable to read the bytes of the account's owners"); + let owner = Principal::from_slice(&owner_bytes); + Account { owner, subaccount } + } + + const BOUND: Bound = Bound::Bounded { + max_size: MAX_SERIALIZATION_LEN, + is_fixed_size: false, + }; +} + #[cfg(test)] mod tests { use assert_matches::assert_matches; + use ic_stable_structures::Storable; use std::str::FromStr; use candid::Principal; - use crate::icrc1::account::{Account, ICRC1TextReprError}; + use crate::icrc1::account::{ + Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT, MAX_SERIALIZATION_LEN, + }; #[test] fn test_account_display_default_subaccount() { @@ -308,4 +363,54 @@ mod tests { Err(ICRC1TextReprError::InvalidChecksum { expected: _ }) ); } + + #[test] + fn test_account_max_serialization_length() { + let owner = + Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + .unwrap() + .try_into() + .unwrap(), + ); + let account = Account { owner, subaccount }; + let serialized_len = account.to_bytes().len(); + assert_eq!( + serialized_len, + 1 + DEFAULT_SUBACCOUNT.len() + Principal::MAX_LENGTH_IN_BYTES + ); + assert_eq!(serialized_len as u32, MAX_SERIALIZATION_LEN); + } + + #[test] + fn test_account_serialization() { + let owner = + Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + .unwrap() + .try_into() + .unwrap(), + ); + let account1 = Account { owner, subaccount }; + let account2 = Account { + owner, + subaccount: None, + }; + let account3 = Account { + owner: Principal::anonymous(), + subaccount, + }; + let account4 = Account { + owner: Principal::anonymous(), + subaccount: None, + }; + let accounts = vec![account1, account2, account3, account4]; + for account in accounts { + assert_eq!(Account::from_bytes(account.to_bytes()), account); + } + } } From 1a03ada78dc03c097c4c7b610f5f077f347216d4 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 11 Oct 2024 13:35:52 +0000 Subject: [PATCH 041/124] clippy --- packages/icrc-ledger-types/src/icrc1/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index ceeca41d2e0..5aa62a4d49e 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -412,5 +412,5 @@ mod tests { for account in accounts { assert_eq!(Account::from_bytes(account.to_bytes()), account); } - } + } } From cef0b739e42a5d0a520ffc38528f2a1650f203a0 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Fri, 11 Oct 2024 13:37:14 +0000 Subject: [PATCH 042/124] Automatically updated Cargo*.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 90d31be3398..ae7ad2d0679 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13551,6 +13551,7 @@ dependencies = [ "candid", "crc32fast", "hex", + "ic-stable-structures", "itertools 0.12.1", "num-bigint 0.4.6", "num-traits", From 168afff063ac524c15b877b96e92598b079e7fdc Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 11 Oct 2024 13:42:19 +0000 Subject: [PATCH 043/124] implement storable for allowance --- .../common/ledger_core/BUILD.bazel | 1 + rs/ledger_suite/common/ledger_core/Cargo.toml | 1 + .../common/ledger_core/src/approvals.rs | 43 +++++++++++++++++++ .../common/ledger_core/src/approvals/tests.rs | 26 +++++++++++ 4 files changed, 71 insertions(+) diff --git a/rs/ledger_suite/common/ledger_core/BUILD.bazel b/rs/ledger_suite/common/ledger_core/BUILD.bazel index 797c654a270..94f48aa34f0 100644 --- a/rs/ledger_suite/common/ledger_core/BUILD.bazel +++ b/rs/ledger_suite/common/ledger_core/BUILD.bazel @@ -11,6 +11,7 @@ rust_library( # Keep sorted. "//packages/ic-ledger-hash-of:ic_ledger_hash_of", "@crate_index//:candid", + "@crate_index//:ic-stable-structures", "@crate_index//:num-traits", "@crate_index//:serde", "@crate_index//:serde_bytes", diff --git a/rs/ledger_suite/common/ledger_core/Cargo.toml b/rs/ledger_suite/common/ledger_core/Cargo.toml index fa577268bfe..7f5279d3fc4 100644 --- a/rs/ledger_suite/common/ledger_core/Cargo.toml +++ b/rs/ledger_suite/common/ledger_core/Cargo.toml @@ -9,6 +9,7 @@ documentation.workspace = true [dependencies] candid = { workspace = true } ic-ledger-hash-of = { path = "../../../../packages/ic-ledger-hash-of" } +ic-stable-structures = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 5939d2a791e..6d8000629c4 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -1,7 +1,13 @@ use crate::timestamp::TimeStamp; use crate::tokens::{CheckedSub, TokensType, Zero}; +use candid::Nat; +use ic_stable_structures::{storable::Bound, Storable}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; +use std::{ + borrow::Cow, + io::{Cursor, Read}, +}; #[cfg(test)] mod tests; @@ -474,3 +480,40 @@ where fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } + +impl + TryFrom> Storable for Allowance { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buffer = vec![]; + let amount: Nat = self.amount.clone().into(); + amount + .encode(&mut buffer) + .expect("Unable to serialize amount"); + if let Some(expires_at) = self.expires_at { + buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); + } + // We don't seriazlize arrived_at - it is not used after stable structures migration. + Cow::Owned(buffer) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + let mut cursor = Cursor::new(bytes.into_owned()); + let amount = Nat::decode(&mut cursor).expect("Unable to deserialize amount"); + let amount = Tokens::try_from(amount).expect("Unable to convert Nat to Tokens"); + // arrived_at was not serialized, use a default value. + let arrived_at = TimeStamp::from_nanos_since_unix_epoch(0); + let mut expires_at_bytes = [0u8; 8]; + let expires_at = match cursor.read_exact(&mut expires_at_bytes) { + Ok(()) => Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( + expires_at_bytes, + ))), + _ => None, + }; + Self { + amount, + arrived_at, + expires_at, + } + } + + const BOUND: Bound = Bound::Unbounded; +} diff --git a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs index c6ea7acb0b4..7ef9775936b 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use super::*; use crate::timestamp::TimeStamp; use crate::tokens::Tokens; +use ic_stable_structures::Storable; use std::cmp; fn ts(n: u64) -> TimeStamp { @@ -619,3 +620,28 @@ fn expected_allowance_if_zero_no_approval() { } ); } + +#[test] +fn allowance_serialization() { + let allowance1 = Allowance { + amount: tokens(100), + expires_at: None, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), + }; + let allowance2 = Allowance { + amount: tokens(300), + expires_at: Some(ts(5)), + arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), + }; + + let allowances = vec![allowance1, allowance2]; + for allowance in allowances { + let new_allowance: Allowance = Allowance::from_bytes(allowance.to_bytes()); + assert_eq!(new_allowance.amount, allowance.amount); + assert_eq!(new_allowance.expires_at, allowance.expires_at); + assert_eq!( + new_allowance.arrived_at, + TimeStamp::from_nanos_since_unix_epoch(0) + ); + } +} From aff2fcd46f341cffbb88af7397fb518e0af9bfae Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Fri, 11 Oct 2024 13:44:22 +0000 Subject: [PATCH 044/124] Automatically updated Cargo*.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 90d31be3398..7f3098da5dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9156,6 +9156,7 @@ version = "0.9.0" dependencies = [ "candid", "ic-ledger-hash-of", + "ic-stable-structures", "num-traits", "serde", "serde_bytes", From 193b5660ee2a3f05744957571952d32f0d79d008 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 12:44:22 +0000 Subject: [PATCH 045/124] increase max allowed orchestrator wasm size --- publish/canisters/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/canisters/BUILD.bazel b/publish/canisters/BUILD.bazel index fd916583e3f..34b24deae66 100644 --- a/publish/canisters/BUILD.bazel +++ b/publish/canisters/BUILD.bazel @@ -69,7 +69,7 @@ CANISTERS_MAX_SIZE_COMPRESSED_E5_BYTES = { # The orchestrator needs to embed 3 wasms at compile time # (ICRC1 index, ICRC1 ledger, and ICRC1 archive) and size is # therefore strictly controlled. - "ic-ledger-suite-orchestrator-canister.wasm.gz": "16", + "ic-ledger-suite-orchestrator-canister.wasm.gz": "17", } # How these limits were chosen: From bbd77adab3513f8ed8f7de4d54fbba095c4592af Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 13:29:17 +0000 Subject: [PATCH 046/124] fix test upgrade from 256 to 64 --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 497e59d3315..fa0b6234974 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -1698,7 +1698,7 @@ mod incompatible_token_type_upgrade { } #[test] - #[should_panic(expected = "failed to decode ledger state")] + #[should_panic(expected = "invalid type: enum, expected u64 or { e8s: u64 }")] fn should_trap_when_upgrading_a_ledger_installed_as_u256_to_u64_wasm() { let env = StateMachine::new(); let ledger_id = env From f72145b3011d0a7f3cc4ced3da31048b17982429 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 13:38:16 +0000 Subject: [PATCH 047/124] update canbench, pre_upgrade improved --- .../ledger/canbench_results/canbench_u256.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml index c2f7aec9183..738580e81a8 100644 --- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml +++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml @@ -1,42 +1,42 @@ benches: bench_icrc1_transfers: total: - instructions: 6411999531 + instructions: 6398020912 heap_increase: 227 - stable_memory_increase: 129 + stable_memory_increase: 128 scopes: before_upgrade: - instructions: 5777186328 + instructions: 5769079066 heap_increase: 71 stable_memory_increase: 0 post_upgrade: - instructions: 434316493 + instructions: 440269297 heap_increase: 27 stable_memory_increase: 0 pre_upgrade: - instructions: 200493625 + instructions: 188669464 heap_increase: 129 - stable_memory_increase: 129 + stable_memory_increase: 128 upgrade: - instructions: 634811735 + instructions: 628940378 heap_increase: 156 - stable_memory_increase: 129 + stable_memory_increase: 128 bench_upgrade_baseline: total: - instructions: 8724806 + instructions: 8749289 heap_increase: 258 stable_memory_increase: 129 scopes: post_upgrade: - instructions: 8595659 + instructions: 8618994 heap_increase: 129 stable_memory_increase: 0 pre_upgrade: - instructions: 126706 + instructions: 127854 heap_increase: 129 stable_memory_increase: 129 upgrade: - instructions: 8724099 + instructions: 8748582 heap_increase: 258 stable_memory_increase: 129 version: 0.1.7 From 356f8b87be68a2630624d166d5e529810f522adc Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 13:40:09 +0000 Subject: [PATCH 048/124] refactor --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 4fe409400fe..662aced31db 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2818,7 +2818,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( pub fn icrc1_test_stable_migration_endpoints_disabled( ledger_wasm_mainnet: Vec, - ledger_wasm_current: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, encode_init_args: fn(InitArgs) -> T, ) where T: CandidType, @@ -2839,7 +2839,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( env.upgrade_canister( canister_id, - ledger_wasm_current, + ledger_wasm_current_lowinstructionlimits, Encode!(&LedgerArgument::Upgrade(None)).unwrap(), ) .unwrap(); @@ -2969,7 +2969,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( pub fn test_incomplete_migration( ledger_wasm_mainnet: Vec, - ledger_wasm_current: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, encode_init_args: fn(InitArgs) -> T, ) where T: CandidType, @@ -3015,7 +3015,7 @@ pub fn test_incomplete_migration( env.upgrade_canister( canister_id, - ledger_wasm_current, + ledger_wasm_current_lowinstructionlimits, Encode!(&LedgerArgument::Upgrade(None)).unwrap(), ) .unwrap(); From e1eccd41c18437ed655b518f72f0dbb8e03691dc Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 13:48:26 +0000 Subject: [PATCH 049/124] test multiple round upgrade --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 2 +- rs/ledger_suite/tests/sm-tests/src/lib.rs | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index fa0b6234974..0d5c1bfa155 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -421,7 +421,7 @@ fn icrc1_test_upgrade_serialization() { fn icrc1_test_upgrade_serialization_fixed_tx() { ic_icrc1_ledger_sm_tests::icrc1_test_upgrade_serialization_fixed_tx( ledger_mainnet_wasm(), - ledger_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), encode_init_args, ); } diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 662aced31db..59854e4a963 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2651,6 +2651,9 @@ pub fn test_upgrade_serialization( .unwrap(); if migration_to_stable_structures { wait_ledger_ready(&env, ledger_id, 10); + let stable_upgrade_migration_steps = + parse_metric(&env, ledger_id, "ledger_stable_upgrade_migration_steps"); + assert_eq!(stable_upgrade_migration_steps, 1); } add_tx_and_verify(); }; @@ -2688,7 +2691,7 @@ pub fn test_upgrade_serialization( pub fn icrc1_test_upgrade_serialization_fixed_tx( ledger_wasm_mainnet: Vec, - ledger_wasm_current: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, encode_init_args: fn(InitArgs) -> T, ) where T: CandidType, @@ -2758,6 +2761,10 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( wait_ledger_ready(&env, canister_id, 10); + let stable_upgrade_migration_steps = + parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + assert!(stable_upgrade_migration_steps > 1); + let mut allowances = vec![]; for i in 0..accounts.len() { for j in i + 1..accounts.len() { @@ -2777,11 +2784,10 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( }; // Test if the old serialized approvals and balances are correctly deserialized - test_upgrade(ledger_wasm_current.clone(), balances.clone()); - - let stable_upgrade_migration_steps = - parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); - assert_eq!(stable_upgrade_migration_steps, 1); + test_upgrade( + ledger_wasm_current_lowinstructionlimits.clone(), + balances.clone(), + ); // Add some more approvals for a1 in &accounts { @@ -2800,7 +2806,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } // Test the new wasm serialization - test_upgrade(ledger_wasm_current, balances); + test_upgrade(ledger_wasm_current_lowinstructionlimits, balances); // See if the additional approvals are there for a1 in &accounts { From 73f624091a49e9d390a9588a6cd4f2a2ac42480d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 14 Oct 2024 15:34:10 +0000 Subject: [PATCH 050/124] panic if clearing stable allowance is called --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 27374172066..e82ade2fbd4 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -1002,7 +1002,7 @@ impl AllowancesData for StableAllowancesData { fn pop_first_allowance( &mut self, ) -> Option<((Self::AccountId, Self::AccountId), Allowance)> { - todo!() + panic!("The method should not be called for StableAllowancesData") } fn len_allowances(&self) -> usize { @@ -1024,6 +1024,6 @@ impl AllowancesData for StableAllowancesData { } fn clear_arrivals(&mut self) { - todo!() + panic!("The method should not be called for StableAllowancesData") } } From 6308d5feedd2fb2f2b46060d0e4b66ed839ec88b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 15 Oct 2024 08:39:58 +0000 Subject: [PATCH 051/124] update ledger and archive wasms --- mainnet-canisters.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mainnet-canisters.json b/mainnet-canisters.json index 43f84abfd82..3b9307bb6fc 100644 --- a/mainnet-canisters.json +++ b/mainnet-canisters.json @@ -56,8 +56,8 @@ "sha256": "6dd00ebe425ba360be161c880ce3a3b3cda5a3738d6b323a9fd0366debf590ce" }, "sns_archive": { - "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", - "sha256": "9476aa71bcee621aba93a3d7c115c543f42c543de840da3224c5f70a32dbfe4d" + "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", + "sha256": "4f002f50679af4cc2d5a91b842eeff47bc1aff22286a42cf4c605a28b46b6689" }, "sns_governance": { "rev": "db5901a6e90b718918978ae5167b9c98d5aa7ab6", @@ -68,8 +68,8 @@ "sha256": "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61" }, "sns_ledger": { - "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", - "sha256": "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c" + "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", + "sha256": "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577" }, "sns_root": { "rev": "db5901a6e90b718918978ae5167b9c98d5aa7ab6", From 3109c06b0de08745764902cbd3c75846d5cab967 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 15 Oct 2024 09:34:36 +0000 Subject: [PATCH 052/124] readd approval trimming test --- rs/ledger_suite/icp/ledger/tests/tests.rs | 2 +- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 5 +++++ rs/ledger_suite/tests/sm-tests/src/lib.rs | 16 +++++++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index 8b6589bcd83..af1e4cde137 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -1494,7 +1494,7 @@ fn test_balances_overflow() { #[test] fn test_approval_trimming() { - ic_icrc1_ledger_sm_tests::test_approval_trimming(ledger_wasm(), encode_init_args); + ic_icrc1_ledger_sm_tests::test_approval_trimming(ledger_wasm(), encode_init_args, true); } #[test] diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 0d5c1bfa155..62e8e9e330c 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -355,6 +355,11 @@ fn test_balances_overflow() { ic_icrc1_ledger_sm_tests::test_balances_overflow(ledger_wasm(), encode_init_args); } +#[test] +fn test_approval_trimming() { + ic_icrc1_ledger_sm_tests::test_approval_trimming(ledger_wasm(), encode_init_args, false); +} + #[test] fn test_archive_controllers() { ic_icrc1_ledger_sm_tests::test_archive_controllers(ledger_wasm()); diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 59854e4a963..3e68ae51069 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -3767,8 +3767,11 @@ where assert_eq!(total_supply(&env, canister_id), credited - 1 - 2); } -pub fn test_approval_trimming(ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T) -where +pub fn test_approval_trimming( + ledger_wasm: Vec, + encode_init_args: fn(InitArgs) -> T, + trimming_enabled: bool, +) where T: CandidType, { let env = StateMachine::new(); @@ -3851,7 +3854,14 @@ where .expect("failed to mint tokens"); new_accounts += 1; - let remaining_approvals = cmp::max(num_approvals as i64 - (new_accounts + 1) / 2, 0) as u64; + let remaining_approvals = if trimming_enabled { + cmp::max(num_approvals as i64 - (new_accounts + 1) / 2, 0) as u64 + } else { + // The ICRC ledger does not trim approvals. We still want to run + // this test to make sure the trimming code does not cause panic, etc. + // Once ICP ledger approvals are not trimmed, this test will be removed entirely. + num_approvals + }; assert_eq!( total_allowance(&env, canister_id, num_approvals), Nat::from(10_000 * remaining_approvals) From d854423c8a4da41382b0de6e81c803f8f369f390 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 15 Oct 2024 09:40:27 +0000 Subject: [PATCH 053/124] remove unused param --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 3e68ae51069..c331070ba80 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -3005,7 +3005,7 @@ pub fn test_incomplete_migration( send_approvals(); - let check_approvals = |expected_allowance: u64| { + let check_approvals = || { for i in 2..2 + NUM_APPROVALS { let allowance = get_allowance( &env, @@ -3013,11 +3013,11 @@ pub fn test_incomplete_migration( account, Account::from(PrincipalId::new_user_test_id(i).0), ); - assert_eq!(allowance.allowance, Nat::from(expected_allowance)); + assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); } }; - check_approvals(APPROVE_AMOUNT); + check_approvals(); env.upgrade_canister( canister_id, @@ -3035,7 +3035,7 @@ pub fn test_incomplete_migration( .unwrap(); // All approvals should still be in UPGRADES_MEMORY and downgrade should succeed. - check_approvals(APPROVE_AMOUNT); + check_approvals(); } pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { From 1de5b1d5442d4778ef6ab3ef305950ad3a8c2b39 Mon Sep 17 00:00:00 2001 From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:13:56 +0200 Subject: [PATCH 054/124] Update rs/ledger_suite/common/ledger_core/src/approvals.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mathias Björkqvist --- rs/ledger_suite/common/ledger_core/src/approvals.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 6d8000629c4..9cf9006e7f7 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -491,7 +491,7 @@ impl + TryFrom> Storable for Allo if let Some(expires_at) = self.expires_at { buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); } - // We don't seriazlize arrived_at - it is not used after stable structures migration. + // We don't serialize arrived_at - it is not used after stable structures migration. Cow::Owned(buffer) } From b3f88e03cbf0edcdebda97a832bbf8ed7bc51286 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 17:08:40 +0000 Subject: [PATCH 055/124] only disable endpoints that change ledger state --- rs/ledger_suite/icrc1/ledger/src/main.rs | 42 --------------- rs/ledger_suite/tests/sm-tests/src/lib.rs | 62 +---------------------- 2 files changed, 1 insertion(+), 103 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 0e2058ef38c..a8f6929abd2 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -505,18 +505,12 @@ fn icrc1_minting_account() -> Option { #[query(name = "icrc1_balance_of")] #[candid_method(query, rename = "icrc1_balance_of")] fn icrc1_balance_of(account: Account) -> Nat { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| ledger.balances().account_balance(&account).into()) } #[query(name = "icrc1_total_supply")] #[candid_method(query, rename = "icrc1_total_supply")] fn icrc1_total_supply() -> Nat { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| ledger.balances().total_supply().into()) } @@ -653,9 +647,6 @@ fn execute_transfer_not_async( #[update] #[candid_method(update)] async fn icrc1_transfer(arg: TransferArg) -> Result { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let from_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.from_subaccount, @@ -683,9 +674,6 @@ async fn icrc1_transfer(arg: TransferArg) -> Result { #[update] #[candid_method(update)] async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let spender_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.spender_subaccount, @@ -712,9 +700,6 @@ async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result Vec { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| { ledger .blockchain() @@ -764,9 +749,6 @@ fn supported_standards() -> Vec { #[query] #[candid_method(query)] fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let (start, length) = req .as_start_and_length() .unwrap_or_else(|msg| ic_cdk::api::trap(&msg)); @@ -777,9 +759,6 @@ fn get_transactions(req: GetTransactionsRequest) -> GetTransactionsResponse { #[query] #[candid_method(query)] fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let (start, length) = req .as_start_and_length() .unwrap_or_else(|msg| ic_cdk::api::trap(&msg)); @@ -789,9 +768,6 @@ fn get_blocks(req: GetBlocksRequest) -> GetBlocksResponse { #[query] #[candid_method(query)] fn get_data_certificate() -> DataCertificate { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree()); let mut tree_buf = vec![]; ciborium::ser::into_writer(&hash_tree, &mut tree_buf).unwrap(); @@ -804,9 +780,6 @@ fn get_data_certificate() -> DataCertificate { #[update] #[candid_method(update)] async fn icrc2_approve(arg: ApproveArgs) -> Result { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let block_idx = Access::with_ledger_mut(|ledger| { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); @@ -885,9 +858,6 @@ async fn icrc2_approve(arg: ApproveArgs) -> Result { #[query] #[candid_method(query)] fn icrc2_allowance(arg: AllowanceArgs) -> Allowance { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); let allowance = ledger @@ -903,18 +873,12 @@ fn icrc2_allowance(arg: AllowanceArgs) -> Allowance { #[query] #[candid_method(query)] fn icrc3_get_archives(args: GetArchivesArgs) -> GetArchivesResult { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| ledger.icrc3_get_archives(args)) } #[query] #[candid_method(query)] fn icrc3_get_tip_certificate() -> Option { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let certificate = ByteBuf::from(ic_cdk::api::data_certificate()?); let hash_tree = Access::with_ledger(|ledger| ledger.construct_hash_tree()); let mut tree_buf = vec![]; @@ -957,9 +921,6 @@ fn icrc3_supported_block_types() -> Vec) -> GetBlocksResult { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } Access::with_ledger(|ledger| ledger.icrc3_get_blocks(args)) } @@ -974,9 +935,6 @@ fn icrc10_supported_standards() -> Vec { fn icrc21_canister_call_consent_message( consent_msg_request: ConsentMessageRequest, ) -> Result { - if !is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } let caller_principal = ic_cdk::api::caller(); let ledger_fee = icrc1_fee(); let token_symbol = icrc1_symbol(); diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index c331070ba80..85175eb5375 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2875,25 +2875,9 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( account, spender: account, }; - let get_transactions_args = GetTransactionsRequest { - start: Nat::from(1u64), - length: Nat::from(1u64), - }; - let get_archives_args = GetArchivesArgs { from: None }; - let blocks_req_vec: Vec = vec![]; - let consent_msg_args = ConsentMessageRequest { - method: "icrc1_transfer".to_owned(), - arg: Encode!(&transfer_args).unwrap(), - user_preferences: ConsentMessageSpec { - metadata: ConsentMessageMetadata { - language: "en".to_string(), - utc_offset_minutes: Some(60), - }, - device_spec: Some(DisplayMessageType::GenericDisplay), - }, - }; let test_endpoint = |endpoint_name: &str, args: Vec, expect_error: bool| { + println!("testing endpoint {endpoint_name}"); let result = env.execute_ingress_as(account.owner.into(), canister_id, endpoint_name, args); if expect_error { result @@ -2914,26 +2898,6 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), true); test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), true); test_endpoint("icrc1_total_supply", Encode!().unwrap(), true); - test_endpoint("archives", Encode!().unwrap(), true); - test_endpoint( - "get_transactions", - Encode!(&get_transactions_args).unwrap(), - true, - ); - test_endpoint("get_blocks", Encode!(&get_transactions_args).unwrap(), true); - test_endpoint("get_data_certificate", Encode!().unwrap(), true); - test_endpoint( - "icrc3_get_archives", - Encode!(&get_archives_args).unwrap(), - true, - ); - test_endpoint("icrc3_get_tip_certificate", Encode!().unwrap(), true); - test_endpoint("icrc3_get_blocks", Encode!(&blocks_req_vec).unwrap(), true); - test_endpoint( - "icrc21_canister_call_consent_message", - Encode!(&consent_msg_args).unwrap(), - true, - ); wait_ledger_ready(&env, canister_id, 10); @@ -2947,30 +2911,6 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), false); test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), false); test_endpoint("icrc1_total_supply", Encode!().unwrap(), false); - test_endpoint("archives", Encode!().unwrap(), false); - test_endpoint( - "get_transactions", - Encode!(&get_transactions_args).unwrap(), - false, - ); - test_endpoint( - "get_blocks", - Encode!(&get_transactions_args).unwrap(), - false, - ); - test_endpoint("get_data_certificate", Encode!().unwrap(), false); - test_endpoint( - "icrc3_get_archives", - Encode!(&get_archives_args).unwrap(), - false, - ); - test_endpoint("icrc3_get_tip_certificate", Encode!().unwrap(), false); - test_endpoint("icrc3_get_blocks", Encode!(&blocks_req_vec).unwrap(), false); - test_endpoint( - "icrc21_canister_call_consent_message", - Encode!(&consent_msg_args).unwrap(), - false, - ); } pub fn test_incomplete_migration( From ccb5710ade8121fe591050494770e47384c6ac79 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 17:35:29 +0000 Subject: [PATCH 056/124] proptest account serialization --- .../icrc-ledger-types/src/icrc1/account.rs | 46 ++++++++----------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 5aa62a4d49e..b9894123c55 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -223,6 +223,8 @@ impl Storable for Account { mod tests { use assert_matches::assert_matches; use ic_stable_structures::Storable; + use proptest::prelude::prop; + use proptest::strategy::Strategy; use std::str::FromStr; use candid::Principal; @@ -231,6 +233,20 @@ mod tests { Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT, MAX_SERIALIZATION_LEN, }; + pub fn principal_strategy() -> impl Strategy { + let bytes_strategy = prop::collection::vec(0..=255u8, 29); + bytes_strategy.prop_map(|bytes| Principal::from_slice(bytes.as_slice())) + } + + pub fn account_strategy() -> impl Strategy { + let bytes_strategy = prop::option::of(prop::collection::vec(0..=255u8, 32)); + let principal_strategy = principal_strategy(); + (bytes_strategy, principal_strategy).prop_map(|(bytes, principal)| Account { + owner: principal, + subaccount: bytes.map(|x| x.as_slice().try_into().unwrap()), + }) + } + #[test] fn test_account_display_default_subaccount() { let owner = Principal::anonymous(); @@ -386,31 +402,9 @@ mod tests { #[test] fn test_account_serialization() { - let owner = - Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") - .unwrap(); - let subaccount = Some( - hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") - .unwrap() - .try_into() - .unwrap(), - ); - let account1 = Account { owner, subaccount }; - let account2 = Account { - owner, - subaccount: None, - }; - let account3 = Account { - owner: Principal::anonymous(), - subaccount, - }; - let account4 = Account { - owner: Principal::anonymous(), - subaccount: None, - }; - let accounts = vec![account1, account2, account3, account4]; - for account in accounts { - assert_eq!(Account::from_bytes(account.to_bytes()), account); - } + use proptest::{prop_assert_eq, proptest}; + proptest!(|(account in account_strategy())| { + prop_assert_eq!(Account::from_bytes(account.to_bytes()), account); + }) } } From b69eb989fa340ffb32395d3ca8dd3c92a0355a2b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 19:09:37 +0000 Subject: [PATCH 057/124] proptest allowance serialization --- .../common/ledger_core/BUILD.bazel | 1 + rs/ledger_suite/common/ledger_core/Cargo.toml | 1 + .../common/ledger_core/src/approvals/tests.rs | 43 +++++++++++-------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/rs/ledger_suite/common/ledger_core/BUILD.bazel b/rs/ledger_suite/common/ledger_core/BUILD.bazel index 94f48aa34f0..ba8f054c248 100644 --- a/rs/ledger_suite/common/ledger_core/BUILD.bazel +++ b/rs/ledger_suite/common/ledger_core/BUILD.bazel @@ -22,6 +22,7 @@ rust_test( name = "ledger_core_test", crate = ":ledger_core", deps = [ + "@crate_index//:proptest", ], ) diff --git a/rs/ledger_suite/common/ledger_core/Cargo.toml b/rs/ledger_suite/common/ledger_core/Cargo.toml index 7f5279d3fc4..469dc7b61ed 100644 --- a/rs/ledger_suite/common/ledger_core/Cargo.toml +++ b/rs/ledger_suite/common/ledger_core/Cargo.toml @@ -15,3 +15,4 @@ serde = { workspace = true } serde_bytes = { workspace = true } [dev-dependencies] +proptest = { workspace = true } diff --git a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs index 7ef9775936b..c112741de90 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs @@ -621,27 +621,36 @@ fn expected_allowance_if_zero_no_approval() { ); } +use proptest::prelude::{any, prop_assert_eq, proptest}; +use proptest::strategy::Strategy; + #[test] fn allowance_serialization() { - let allowance1 = Allowance { - amount: tokens(100), - expires_at: None, - arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), - }; - let allowance2 = Allowance { - amount: tokens(300), - expires_at: Some(ts(5)), - arrived_at: TimeStamp::from_nanos_since_unix_epoch(6), - }; - - let allowances = vec![allowance1, allowance2]; - for allowance in allowances { + fn arb_token() -> impl Strategy { + any::().prop_map(Tokens::from_e8s) + } + fn arb_timestamp() -> impl Strategy { + any::().prop_map(TimeStamp::from_nanos_since_unix_epoch) + } + fn arb_opt_expiration() -> impl Strategy> { + proptest::option::of(any::().prop_map(TimeStamp::from_nanos_since_unix_epoch)) + } + fn arb_allowance() -> impl Strategy> { + (arb_token(), arb_opt_expiration(), arb_timestamp()).prop_map( + |(amount, expires_at, arrived_at)| Allowance { + amount, + expires_at, + arrived_at, + }, + ) + } + proptest!(|(allowance in arb_allowance())| { let new_allowance: Allowance = Allowance::from_bytes(allowance.to_bytes()); - assert_eq!(new_allowance.amount, allowance.amount); - assert_eq!(new_allowance.expires_at, allowance.expires_at); - assert_eq!( + prop_assert_eq!(new_allowance.amount, allowance.amount); + prop_assert_eq!(new_allowance.expires_at, allowance.expires_at); + prop_assert_eq!( new_allowance.arrived_at, TimeStamp::from_nanos_since_unix_epoch(0) ); - } + }) } From 8873fb83e968efd90d7c27c70ce6ebda0dd2ae71 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Thu, 17 Oct 2024 19:11:55 +0000 Subject: [PATCH 058/124] Automatically updated Cargo*.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 7f3098da5dc..eef5cf65a91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9158,6 +9158,7 @@ dependencies = [ "ic-ledger-hash-of", "ic-stable-structures", "num-traits", + "proptest", "serde", "serde_bytes", ] From 9a28bc413059b0a67cf691ac6d530c9b70e1a2b9 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 20:55:34 +0000 Subject: [PATCH 059/124] migrate code in ledger --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 32 ++++++++- rs/ledger_suite/icrc1/ledger/src/main.rs | 86 ++++++++++------------- rs/ledger_suite/tests/sm-tests/src/lib.rs | 1 - 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index e82ade2fbd4..dcb21115754 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -381,9 +381,9 @@ impl Default for LedgerState { pub struct Ledger { balances: LedgerBalances, #[serde(default)] - pub approvals: LedgerAllowances, + approvals: LedgerAllowances, #[serde(default)] - pub stable_approvals: AllowanceTable, + stable_approvals: AllowanceTable, blockchain: Blockchain, minting_account: Account, @@ -530,6 +530,34 @@ impl Ledger { pub fn is_ready(&self) -> bool { matches!(self.state, LedgerState::Ready) } + + pub fn migrate_one_allowance(&mut self) -> bool { + match self.approvals.allowances_data.pop_first_allowance() { + Some((account_spender, allowance)) => { + self.stable_approvals + .allowances_data + .set_allowance(account_spender, allowance); + true + } + None => false, + } + } + + pub fn migrate_one_expiration(&mut self) -> bool { + match self.approvals.allowances_data.pop_first_expiry() { + Some((timestamp, account_spender)) => { + self.stable_approvals + .allowances_data + .insert_expiry(timestamp, account_spender); + true + } + None => false, + } + } + + pub fn clear_arrivals(&mut self) { + self.approvals.allowances_data.clear_arrivals(); + } } impl LedgerContext for Ledger { diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index a8f6929abd2..08dd49cac0e 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -22,7 +22,6 @@ use ic_ledger_canister_core::ledger::{ TransferError as CoreTransferError, }; use ic_ledger_canister_core::runtime::total_memory_size_bytes; -use ic_ledger_core::approvals::AllowancesData; use ic_ledger_core::block::BlockIndex; use ic_ledger_core::timestamp::TimeStamp; use ic_ledger_core::tokens::Zero; @@ -226,7 +225,7 @@ fn post_upgrade(args: Option) { Access::with_ledger_mut(|ledger| { ledger.state = LedgerState::Migrating(LedgerField::Allowances); - ledger.approvals.allowances_data.clear_arrivals(); + ledger.clear_arrivals(); }); migrate_next_part( MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), @@ -253,33 +252,18 @@ fn migrate_next_part(instruction_limit: u64) { }; match field { LedgerField::Allowances => { - match ledger.approvals.allowances_data.pop_first_allowance() { - Some((account_spender, allowance)) => { - ledger - .stable_approvals - .allowances_data - .set_allowance(account_spender, allowance); - migrated_allowances += 1; - } - None => { - ledger.state = - LedgerState::Migrating(LedgerField::AllowancesExpirations); - } - }; + if ledger.migrate_one_allowance() { + migrated_allowances += 1; + } else { + ledger.state = LedgerState::Migrating(LedgerField::AllowancesExpirations); + } } LedgerField::AllowancesExpirations => { - match ledger.approvals.allowances_data.pop_first_expiry() { - Some((timestamp, account_spender)) => { - ledger - .stable_approvals - .allowances_data - .insert_expiry(timestamp, account_spender); - migrated_expirations += 1; - } - None => { - ledger.state = LedgerState::Ready; - } - }; + if ledger.migrate_one_expiration() { + migrated_expirations += 1; + } else { + ledger.state = LedgerState::Ready; + } } } } @@ -378,23 +362,25 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i ledger.blockchain().num_archived_blocks.saturating_add(ledger.blockchain().blocks.len() as u64) as f64, "Total number of transactions stored in the main memory, plus total number of transactions sent to the archive.", )?; - let token_pool: Nat = ledger.balances().token_pool.into(); - w.encode_gauge( - "ledger_balances_token_pool", - token_pool.0.to_f64().unwrap_or(f64::INFINITY), - "Total number of Tokens in the pool.", - )?; - let total_supply: Nat = ledger.balances().total_supply().into(); - w.encode_gauge( - "ledger_total_supply", - total_supply.0.to_f64().unwrap_or(f64::INFINITY), - "Total number of tokens in circulation.", - )?; - w.encode_gauge( - "ledger_balance_store_entries", - ledger.balances().store.len() as f64, - "Total number of accounts in the balance store.", - )?; + if is_ready() { + let token_pool: Nat = ledger.balances().token_pool.into(); + w.encode_gauge( + "ledger_balances_token_pool", + token_pool.0.to_f64().unwrap_or(f64::INFINITY), + "Total number of Tokens in the pool.", + )?; + let total_supply: Nat = ledger.balances().total_supply().into(); + w.encode_gauge( + "ledger_total_supply", + total_supply.0.to_f64().unwrap_or(f64::INFINITY), + "Total number of tokens in circulation.", + )?; + w.encode_gauge( + "ledger_balance_store_entries", + ledger.balances().store.len() as f64, + "Total number of accounts in the balance store.", + )?; + } w.encode_gauge( "ledger_most_recent_block_time_seconds", (ledger @@ -421,11 +407,13 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i format!("Failed to read number of archives: {}", err), ))?, } - w.encode_gauge( - "ledger_num_approvals", - ledger.approvals().get_num_approvals() as f64, - "Total number of approvals.", - )?; + if is_ready() { + w.encode_gauge( + "ledger_num_approvals", + ledger.approvals().get_num_approvals() as f64, + "Total number of approvals.", + )?; + } Ok(()) }) } diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 85175eb5375..21ac138633a 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -35,7 +35,6 @@ use icrc_ledger_types::icrc21::requests::{ use icrc_ledger_types::icrc21::responses::{ConsentInfo, ConsentMessage}; use icrc_ledger_types::icrc3; use icrc_ledger_types::icrc3::archive::ArchiveInfo; -use icrc_ledger_types::icrc3::archive::GetArchivesArgs; use icrc_ledger_types::icrc3::blocks::{ BlockRange, GenericBlock as IcrcBlock, GetBlocksRequest, GetBlocksResponse, }; From a24fe450d88b48a0daebdb8f14dba66742075ee7 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 21:50:26 +0000 Subject: [PATCH 060/124] clippy --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 028f65a9b12..a58d825bafb 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -76,10 +76,10 @@ pub type Tokens = ic_icrc1_tokens_u256::U256; /// We have the following ledger versions: /// * 0 - the whole ledger state is stored on the heap. #[cfg(not(feature = "next-ledger-version"))] -pub const LEDGER_VERSION: u64 = 0; +pub const LEDGER_VERSION: u64 = 1; #[cfg(feature = "next-ledger-version")] -pub const LEDGER_VERSION: u64 = 1; +pub const LEDGER_VERSION: u64 = 2; #[derive(Clone, Debug)] pub struct Icrc1ArchiveWasm; @@ -423,7 +423,7 @@ pub struct Ledger { #[serde(default)] pub state: LedgerState, - + #[serde(default = "default_ledger_version")] pub ledger_version: u64, } @@ -576,7 +576,7 @@ impl Ledger { pub fn clear_arrivals(&mut self) { self.approvals.allowances_data.clear_arrivals(); - } + } } impl LedgerContext for Ledger { From accaa54af42e42ee4405985a7b4101db1a20755e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 22:10:42 +0000 Subject: [PATCH 061/124] fix migration to prev version tests --- mainnet-canisters.json | 16 +++++------ rs/ledger_suite/tests/sm-tests/src/lib.rs | 34 +++++++++++++---------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/mainnet-canisters.json b/mainnet-canisters.json index 8fc308e7218..3d41dd12e1f 100644 --- a/mainnet-canisters.json +++ b/mainnet-canisters.json @@ -8,16 +8,16 @@ "sha256": "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61" }, "ck_btc_ledger": { - "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", - "sha256": "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577" + "rev": "b98f0feed23702b954e61912e81c1613c075cffd", + "sha256": "0f194ee27c3a046423b5d36725df71b62156cec173a6c52890fcc2ac2308bbbf" }, "ck_eth_index": { "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31", "sha256": "de250f08dc7e699144b73514f55fbbb3a3f8cd97abf0f7ae31d9fb7494f55234" }, "ck_eth_ledger": { - "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", - "sha256": "566f1c1ae68cca5618e74cdb2239d989084d92237b04a31f7898182ad0547a93" + "rev": "b98f0feed23702b954e61912e81c1613c075cffd", + "sha256": "50959283b0e1f130eedbe8fb326ce0c65d757227d21ddc9153bcc6420e922986" }, "cycles-minting": { "rev": "ca8847547d327ce8a3bd81d25a590e01da1a3af5", @@ -56,8 +56,8 @@ "sha256": "18fa2612dd51837d8f54769761b421627826fa1e19bb3a788ea6ffa8bd59f7b8" }, "sns_archive": { - "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", - "sha256": "4f002f50679af4cc2d5a91b842eeff47bc1aff22286a42cf4c605a28b46b6689" + "rev": "b98f0feed23702b954e61912e81c1613c075cffd", + "sha256": "d8c712ad8beeb7c88556b7da1cb7bae28ba983c369fea4d221f281b899a6af76" }, "sns_governance": { "rev": "c494c2af8bfc70a6501448dc73bf806477388738", @@ -68,8 +68,8 @@ "sha256": "612410c71e893bb64772ab8131d77264740398f3932d873cb4f640fc257f9e61" }, "sns_ledger": { - "rev": "d1db89ed78716b9258a8ec75bd6f53aa067a5be7", - "sha256": "10265a7d88461317953030fb0e6da20b9035260a11d077a1b249d4a18a5b8577" + "rev": "b98f0feed23702b954e61912e81c1613c075cffd", + "sha256": "0f194ee27c3a046423b5d36725df71b62156cec173a6c52890fcc2ac2308bbbf" }, "sns_root": { "rev": "db5901a6e90b718918978ae5167b9c98d5aa7ab6", diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index eeb3caba13b..5feda792ea2 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2985,20 +2985,8 @@ pub fn test_downgrade_from_incompatible_version( ) where T: CandidType, { - // Setup ledger with unsupported future version. - let (env, canister_id) = setup( - ledger_wasm_nextledgerversion.clone(), - encode_init_args, - vec![], - ); - - // For now the mainnet ledger does not perform the check and downgrade is possible. - env.upgrade_canister( - canister_id, - ledger_wasm_mainnet, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .expect("failed to downgrade to mainnet"); + // Setup ledger with mainnet version. + let (env, canister_id) = setup(ledger_wasm_mainnet.clone(), encode_init_args, vec![]); // Upgrade to current version. env.upgrade_canister( @@ -3016,6 +3004,22 @@ pub fn test_downgrade_from_incompatible_version( ) .expect("failed to upgrade to current version"); + // Downgrade to mainnet not possible. + match env.upgrade_canister( + canister_id, + ledger_wasm_mainnet, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade from future ledger version should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; + // Upgrade to the next version. env.upgrade_canister( canister_id, @@ -3024,7 +3028,7 @@ pub fn test_downgrade_from_incompatible_version( ) .expect("failed to upgrade to next version"); - // Downgrade not possible. + // Downgrade to current not possible. match env.upgrade_canister( canister_id, ledger_wasm, From a8035e3c528dbfb02b1248dfe721d57de8ae6f2e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 22:16:42 +0000 Subject: [PATCH 062/124] build fix --- rs/ledger_suite/icp/ledger/tests/tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index e63e4aa9187..7de2c8bffcc 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -1505,7 +1505,11 @@ fn test_balances_overflow() { #[test] fn test_approval_trimming() { - ic_ledger_suite_state_machine_tests::test_approval_trimming(ledger_wasm(), encode_init_args); + ic_ledger_suite_state_machine_tests::test_approval_trimming( + ledger_wasm(), + encode_init_args, + true, + ); } #[test] From c07c8aceb8f43c7607c66a4d39540f40c349ac0b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 22:54:34 +0000 Subject: [PATCH 063/124] disable downgrade to mainnet test --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 46 +++++++++++---------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index eb2b200d329..4d14279fee2 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -167,29 +167,31 @@ fn test_upgrade() { ic_ledger_suite_state_machine_tests::test_upgrade(ledger_wasm(), encode_init_args) } -#[test] -fn test_install_mainnet_ledger_then_upgrade_then_downgrade() { - ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( - ledger_mainnet_wasm(), - encode_init_args, - ledger_wasm(), - encode_upgrade_args, - ledger_mainnet_wasm(), - encode_upgrade_args, - ) -} +// It is not possible to downgrade to mainnet for now, disable the tests. +// Similar testing is done by icrc1_test_downgrade_from_incompatible_version +// #[test] +// fn test_install_mainnet_ledger_then_upgrade_then_downgrade() { +// ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( +// ledger_mainnet_wasm(), +// encode_init_args, +// ledger_wasm(), +// encode_upgrade_args, +// ledger_mainnet_wasm(), +// encode_upgrade_args, +// ) +// } -#[test] -fn test_install_current_ledger_then_upgrade_then_downgrade_to_mainnet_version() { - ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( - ledger_wasm(), - encode_init_args, - ledger_wasm(), - encode_upgrade_args, - ledger_mainnet_wasm(), - encode_upgrade_args, - ) -} +// #[test] +// fn test_install_current_ledger_then_upgrade_then_downgrade_to_mainnet_version() { +// ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( +// ledger_wasm(), +// encode_init_args, +// ledger_wasm(), +// encode_upgrade_args, +// ledger_mainnet_wasm(), +// encode_upgrade_args, +// ) +// } #[test] fn test_upgrade_archive_options() { From 56940aa35595153cdc4ab44517b6b17bfc5f8ebb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 23:26:03 +0000 Subject: [PATCH 064/124] fix test --- rs/ledger_suite/icp/ledger/tests/tests.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index 7de2c8bffcc..2c004a59ca6 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -1245,10 +1245,7 @@ fn test_upgrade_serialization() { upgrade_args, minter, false, - // With the ICP mainnet canister being at V1, and the tip-of-master also being V1, - // downgrading the ledger canister to the mainnet version from the tip-of-master version - // should succeed. - true, + false, ); } From 48e0fe451049aed38a377aa33c1aec6caa97a0c3 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 23:43:24 +0000 Subject: [PATCH 065/124] fix test --- .../icrc1/tests/upgrade_downgrade.rs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs index aff617dbfc2..9013b225e01 100644 --- a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs +++ b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs @@ -67,22 +67,23 @@ fn should_upgrade_and_downgrade_ledger_canister_suite() { ) .unwrap(); - env.advance_time(Duration::from_secs(60)); - env.tick(); + // Downgrade to mainnet is not possible for this version. + // env.advance_time(Duration::from_secs(60)); + // env.tick(); - env.upgrade_canister( - index_id, - index_ng_mainnet_wasm(), - Encode!(&index_upgrade_arg).unwrap(), - ) - .unwrap(); + // env.upgrade_canister( + // index_id, + // index_ng_mainnet_wasm(), + // Encode!(&index_upgrade_arg).unwrap(), + // ) + // .unwrap(); - env.upgrade_canister( - ledger_id, - ledger_mainnet_wasm(), - Encode!(&ledger_upgrade_arg).unwrap(), - ) - .unwrap(); + // env.upgrade_canister( + // ledger_id, + // ledger_mainnet_wasm(), + // Encode!(&ledger_upgrade_arg).unwrap(), + // ) + // .unwrap(); } fn default_archive_options() -> ArchiveOptions { From 7f3e0249c3d03e2cfd0c87b3929de8dabbef5f1b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 17 Oct 2024 23:47:55 +0000 Subject: [PATCH 066/124] update canbench results --- .../ledger/canbench_results/canbench_u256.yml | 18 +++++++------- .../ledger/canbench_results/canbench_u64.yml | 24 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml index 738580e81a8..c37d9397ecd 100644 --- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml +++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml @@ -1,42 +1,42 @@ benches: bench_icrc1_transfers: total: - instructions: 6398020912 + instructions: 6390210524 heap_increase: 227 stable_memory_increase: 128 scopes: before_upgrade: - instructions: 5769079066 + instructions: 5769009086 heap_increase: 71 stable_memory_increase: 0 post_upgrade: - instructions: 440269297 + instructions: 434388599 heap_increase: 27 stable_memory_increase: 0 pre_upgrade: - instructions: 188669464 + instructions: 186809754 heap_increase: 129 stable_memory_increase: 128 upgrade: - instructions: 628940378 + instructions: 621199970 heap_increase: 156 stable_memory_increase: 128 bench_upgrade_baseline: total: - instructions: 8749289 + instructions: 8750052 heap_increase: 258 stable_memory_increase: 129 scopes: post_upgrade: - instructions: 8618994 + instructions: 8619094 heap_increase: 129 stable_memory_increase: 0 pre_upgrade: - instructions: 127854 + instructions: 128517 heap_increase: 129 stable_memory_increase: 129 upgrade: - instructions: 8748582 + instructions: 8749345 heap_increase: 258 stable_memory_increase: 129 version: 0.1.7 diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml index aa947c6db67..264cac42052 100644 --- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml +++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml @@ -1,42 +1,42 @@ benches: bench_icrc1_transfers: total: - instructions: 5925418308 + instructions: 5911654561 heap_increase: 214 - stable_memory_increase: 129 + stable_memory_increase: 128 scopes: before_upgrade: - instructions: 5322891171 + instructions: 5315972434 heap_increase: 64 stable_memory_increase: 0 post_upgrade: - instructions: 421019190 + instructions: 415243938 heap_increase: 21 stable_memory_increase: 0 pre_upgrade: - instructions: 181504754 + instructions: 180434821 heap_increase: 129 - stable_memory_increase: 129 + stable_memory_increase: 128 upgrade: - instructions: 602525561 + instructions: 595680376 heap_increase: 150 - stable_memory_increase: 129 + stable_memory_increase: 128 bench_upgrade_baseline: total: - instructions: 8728677 + instructions: 8753711 heap_increase: 258 stable_memory_increase: 129 scopes: post_upgrade: - instructions: 8599099 + instructions: 8621012 heap_increase: 129 stable_memory_increase: 0 pre_upgrade: - instructions: 127137 + instructions: 130258 heap_increase: 129 stable_memory_increase: 129 upgrade: - instructions: 8727970 + instructions: 8753004 heap_increase: 258 stable_memory_increase: 129 version: 0.1.7 From 85c0d9de4617ba31fa7e28d15fc58357a96f62ef Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 18 Oct 2024 00:25:48 +0000 Subject: [PATCH 067/124] larger limits --- rs/ledger_suite/icrc1/ledger/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index e76b0bd9973..709464f89c8 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -157,9 +157,9 @@ fn pre_upgrade() { } #[cfg(not(feature = "low-upgrade-instruction-limits"))] -const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 199_950_000_000; #[cfg(not(feature = "low-upgrade-instruction-limits"))] -const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; +const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_950_000_000; #[cfg(feature = "low-upgrade-instruction-limits")] const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 0; From 095509e29c6543fad055b3a727510c520f9734fe Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 18 Oct 2024 08:33:11 -0700 Subject: [PATCH 068/124] move state below version --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index a58d825bafb..e03164b213b 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -421,11 +421,11 @@ pub struct Ledger { #[serde(default = "default_accounts_overflow_trim_quantity")] accounts_overflow_trim_quantity: usize, - #[serde(default)] - pub state: LedgerState, - #[serde(default = "default_ledger_version")] pub ledger_version: u64, + + #[serde(default)] + pub state: LedgerState, } fn default_maximum_number_of_accounts() -> usize { @@ -523,8 +523,8 @@ impl Ledger { .unwrap_or_else(|| ACCOUNTS_OVERFLOW_TRIM_QUANTITY.try_into().unwrap()) .try_into() .unwrap(), - state: LedgerState::Ready, ledger_version: LEDGER_VERSION, + state: LedgerState::Ready, }; for (account, balance) in initial_balances.into_iter() { From 0a29200b2505ce91be98b0607d81787694056926 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 18 Oct 2024 11:40:26 -0700 Subject: [PATCH 069/124] move test up in file --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 136 +++++++++++----------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 5feda792ea2..1e98173ac59 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2821,6 +2821,74 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } } +pub fn test_downgrade_from_incompatible_version( + ledger_wasm_mainnet: Vec, + ledger_wasm_nextledgerversion: Vec, + ledger_wasm: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + // Setup ledger with mainnet version. + let (env, canister_id) = setup(ledger_wasm_mainnet.clone(), encode_init_args, vec![]); + + // Upgrade to current version. + env.upgrade_canister( + canister_id, + ledger_wasm.clone(), + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .expect("failed to upgrade to current version"); + + // Upgrade to the same verison. + env.upgrade_canister( + canister_id, + ledger_wasm.clone(), + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .expect("failed to upgrade to current version"); + + // Downgrade to mainnet not possible. + match env.upgrade_canister( + canister_id, + ledger_wasm_mainnet, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade from future ledger version should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; + + // Upgrade to the next version. + env.upgrade_canister( + canister_id, + ledger_wasm_nextledgerversion, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .expect("failed to upgrade to next version"); + + // Downgrade to current not possible. + match env.upgrade_canister( + canister_id, + ledger_wasm, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade from future ledger version should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; +} + pub fn icrc1_test_stable_migration_endpoints_disabled( ledger_wasm_mainnet: Vec, ledger_wasm_current_lowinstructionlimits: Vec, @@ -2977,74 +3045,6 @@ pub fn test_incomplete_migration( check_approvals(); } -pub fn test_downgrade_from_incompatible_version( - ledger_wasm_mainnet: Vec, - ledger_wasm_nextledgerversion: Vec, - ledger_wasm: Vec, - encode_init_args: fn(InitArgs) -> T, -) where - T: CandidType, -{ - // Setup ledger with mainnet version. - let (env, canister_id) = setup(ledger_wasm_mainnet.clone(), encode_init_args, vec![]); - - // Upgrade to current version. - env.upgrade_canister( - canister_id, - ledger_wasm.clone(), - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .expect("failed to upgrade to current version"); - - // Upgrade to the same verison. - env.upgrade_canister( - canister_id, - ledger_wasm.clone(), - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .expect("failed to upgrade to current version"); - - // Downgrade to mainnet not possible. - match env.upgrade_canister( - canister_id, - ledger_wasm_mainnet, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) { - Ok(_) => { - panic!("Upgrade from future ledger version should fail!") - } - Err(e) => { - assert!(e - .description() - .contains("Trying to downgrade from incompatible version")) - } - }; - - // Upgrade to the next version. - env.upgrade_canister( - canister_id, - ledger_wasm_nextledgerversion, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .expect("failed to upgrade to next version"); - - // Downgrade to current not possible. - match env.upgrade_canister( - canister_id, - ledger_wasm, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) { - Ok(_) => { - panic!("Upgrade from future ledger version should fail!") - } - Err(e) => { - assert!(e - .description() - .contains("Trying to downgrade from incompatible version")) - } - }; -} - pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { ApproveArgs { from_subaccount: None, From 692f89649a17952ff3459cded1ab9e79c7ff8c91 Mon Sep 17 00:00:00 2001 From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:57:19 -0800 Subject: [PATCH 070/124] Update rs/ledger_suite/icrc1/ledger/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mathias Björkqvist --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index e03164b213b..9e410c5bebd 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -710,8 +710,7 @@ impl Ledger { records.push(Value::entry("icrc1:decimals", self.decimals() as u64)); records.push(Value::entry("icrc1:name", self.token_name())); records.push(Value::entry("icrc1:symbol", self.token_symbol())); - let nat_fee: Nat = self.transfer_fee().into(); - records.push(Value::entry("icrc1:fee", nat_fee)); + records.push(Value::entry("icrc1:fee", Nat::from(self.transfer_fee()))); records.push(Value::entry( "icrc1:max_memo_length", self.max_memo_length() as u64, From 633653f868a7fcf252f80a1955411a51b64de528 Mon Sep 17 00:00:00 2001 From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:58:03 -0800 Subject: [PATCH 071/124] Update rs/ledger_suite/icrc1/ledger/src/lib.rs Co-authored-by: mraszyk <31483726+mraszyk@users.noreply.github.com> --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 9e410c5bebd..7dd64842cbe 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -1048,7 +1048,7 @@ impl AllowancesData for StableAllowancesData { fn pop_first_allowance( &mut self, ) -> Option<((Self::AccountId, Self::AccountId), Allowance)> { - panic!("The method should not be called for StableAllowancesData") + panic!("The method `pop_first_allowance` should not be called for StableAllowancesData") } fn len_allowances(&self) -> usize { From fc0b00d16d94f01f797324294637a4d8d976037d Mon Sep 17 00:00:00 2001 From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:58:54 -0800 Subject: [PATCH 072/124] Update rs/ledger_suite/icrc1/ledger/src/lib.rs Co-authored-by: mraszyk <31483726+mraszyk@users.noreply.github.com> --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 7dd64842cbe..646e9654911 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -1070,6 +1070,6 @@ impl AllowancesData for StableAllowancesData { } fn clear_arrivals(&mut self) { - panic!("The method should not be called for StableAllowancesData") + panic!("The method `clear_arrivals` should not be called for StableAllowancesData") } } From 1d06a25299ecc95d89096f4e6adf09d6eeaf3d4d Mon Sep 17 00:00:00 2001 From: maciejdfinity <122265298+maciejdfinity@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:59:33 -0800 Subject: [PATCH 073/124] Update rs/ledger_suite/tests/sm-tests/src/lib.rs Co-authored-by: mraszyk <31483726+mraszyk@users.noreply.github.com> --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 1e98173ac59..131fd23bf89 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -553,7 +553,7 @@ pub fn wait_ledger_ready(env: &StateMachine, ledger: CanisterId, num_waits: u16) .bytes(), bool ) - .expect("failed to decode balance_of response") + .expect("failed to decode is_ledger_ready response") }; for i in 0..num_waits { if is_ledger_ready() { From 0c2e9c30b7fc80d892516ade86c6c6abfca62e08 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 19:05:15 +0000 Subject: [PATCH 074/124] extend the comment for ledger_version --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 646e9654911..f22f502f09b 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -75,6 +75,7 @@ pub type Tokens = ic_icrc1_tokens_u256::U256; /// Upgrading from version N to version N+1 should always be possible. /// We have the following ledger versions: /// * 0 - the whole ledger state is stored on the heap. +/// * 1 - the allowances are stored in stable structures. #[cfg(not(feature = "next-ledger-version"))] pub const LEDGER_VERSION: u64 = 1; From aba940887100452416f478c6a886528b620977ea Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 19:11:43 +0000 Subject: [PATCH 075/124] panic if not ready fn --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index f22f502f09b..018e5c9cc46 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -551,6 +551,12 @@ impl Ledger { matches!(self.state, LedgerState::Ready) } + pub fn panic_if_not_ready(&self) { + if !self.is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } + } + pub fn migrate_one_allowance(&mut self) -> bool { match self.approvals.allowances_data.pop_first_allowance() { Some((account_spender, allowance)) => { @@ -587,30 +593,22 @@ impl LedgerContext for Ledger { type Tokens = Tokens; fn balances(&self) -> &Balances { - if !self.is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } + self.panic_if_not_ready(); &self.balances } fn balances_mut(&mut self) -> &mut Balances { - if !self.is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } + self.panic_if_not_ready(); &mut self.balances } fn approvals(&self) -> &AllowanceTable { - if !self.is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } + self.panic_if_not_ready(); &self.stable_approvals } fn approvals_mut(&mut self) -> &mut AllowanceTable { - if !self.is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } + self.panic_if_not_ready(); &mut self.stable_approvals } From 0cd743fc4493d21f586274ddbf42f3c0e2c9ea79 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 19:41:57 +0000 Subject: [PATCH 076/124] build fix --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 8bcb67a1905..237c7598556 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -454,6 +454,7 @@ fn icrc1_test_upgrade_serialization() { upgrade_args, minter, true, + true, ); } From d644f2f0f30514180bedbf7d7f5f481aa68ec511 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 20:34:04 +0000 Subject: [PATCH 077/124] comment about not storing arrivals --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 018e5c9cc46..03be1ce6f2d 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -1021,6 +1021,7 @@ impl AllowancesData for StableAllowancesData { _timestamp: TimeStamp, _account_spender: (Self::AccountId, Self::AccountId), ) { + // We do not store arrivals in stable structures. } fn remove_arrival( @@ -1028,6 +1029,7 @@ impl AllowancesData for StableAllowancesData { _timestamp: TimeStamp, _account_spender: (Self::AccountId, Self::AccountId), ) { + // We do not store arrivals in stable structures. } fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { @@ -1036,6 +1038,7 @@ impl AllowancesData for StableAllowancesData { } fn oldest_arrivals(&self, _n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { + // We do not store arrivals in stable structures. vec![] } From ebf3c2a7da69768931d8691007128072e3fe71eb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 22:12:48 +0000 Subject: [PATCH 078/124] improve logging --- rs/ledger_suite/icrc1/ledger/src/main.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 709464f89c8..67438e40b3c 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -186,7 +186,7 @@ fn post_upgrade(args: Option) { if !memory_manager_found { let msg = "Cannot upgrade from scratch stable memory, please upgrade to memory manager first."; - ic_cdk::println!("{msg}"); + log_message(msg); panic!("{msg}"); } @@ -237,6 +237,7 @@ fn post_upgrade(args: Option) { ledger.state = LedgerState::Migrating(LedgerField::Allowances); ledger.clear_arrivals(); }); + log_message("Migration started."); migrate_next_part( MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), ); @@ -251,8 +252,7 @@ fn migrate_next_part(instruction_limit: u64) { let mut migrated_allowances = 0; let mut migrated_expirations = 0; - ic_cdk::println!("Migration started."); - log!(&LOG, "Migration started."); + log_message("Migrating part of the ledger state."); Access::with_ledger_mut(|ledger| { while instruction_counter() < instruction_limit { @@ -280,21 +280,23 @@ fn migrate_next_part(instruction_limit: u64) { let msg = format!("Number of elements migrated: allowances:{migrated_allowances} expirations:{migrated_expirations}. Instructions used {}.", instruction_counter()); if !ledger.is_ready() { - ic_cdk::println!("Migration partially done. Scheduling the next part. {msg}"); - log!( - &LOG, - "Migration partially done. Scheduling the next part. {msg}" + log_message( + format!("Migration partially done. Scheduling the next part. {msg}").as_str(), ); ic_cdk_timers::set_timer(Duration::from_secs(0), || { migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) }); } else { - ic_cdk::println!("Migration completed! {msg}"); - log!(&LOG, "Migration completed! {msg}"); + log_message(format!("Migration completed! {msg}").as_str()); } }); } +fn log_message(msg: &str) { + ic_cdk::println!("{msg}"); + log!(&LOG, "{msg}"); +} + fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()> { w.encode_gauge( "ledger_stable_memory_pages", From 5ad210d69ad8740393753bb59f9e4bc6d90c9b1d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 22:28:30 +0000 Subject: [PATCH 079/124] update comment --- rs/ledger_suite/icrc1/ledger/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 67438e40b3c..37a8e69dbec 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -277,7 +277,7 @@ fn migrate_next_part(instruction_limit: u64) { } } } - let msg = format!("Number of elements migrated: allowances:{migrated_allowances} expirations:{migrated_expirations}. Instructions used {}.", + let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Total instructions used in message: {}." , instruction_counter()); if !ledger.is_ready() { log_message( From 27b06e75d0270bf3dfe542f2879178cb606571f5 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 22:41:21 +0000 Subject: [PATCH 080/124] test failure to downgrade from stable structures --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index cd9f7dad66c..cb688326104 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2427,6 +2427,22 @@ pub fn test_upgrade_serialization( if !migration_to_stable_structures { // Test downgrade to mainnet wasm test_upgrade(ledger_wasm_mainnet.clone()); + } else { + // Downgrade from stable structures to mainnet not possible. + match env.upgrade_canister( + ledger_id, + ledger_wasm_mainnet.clone(), + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade from future ledger version should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; } if verify_blocks { // This will also verify the ledger blocks. From 5fe793c1e1ddab0ae837decb644b29693ce98df7 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 22:49:48 +0000 Subject: [PATCH 081/124] assert ledger not ready --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index cb688326104..76a1f2ceb7d 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2802,6 +2802,15 @@ pub fn test_incomplete_migration( ) .unwrap(); + let is_ledger_ready = Decode!( + &env.query(canister_id, "is_ledger_ready", Encode!().unwrap()) + .expect("failed to call is_ledger_ready") + .bytes(), + bool + ) + .expect("failed to decode is_ledger_ready response"); + assert!(!is_ledger_ready); + // Downgrade to mainnet without waiting for the migration to complete. env.upgrade_canister( canister_id, From 5c2757a5ed1fbd0a7267a4763f9a62c4c3038ea6 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 23:08:40 +0000 Subject: [PATCH 082/124] remove unused tests --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 26 ---------------- rs/ledger_suite/tests/sm-tests/src/lib.rs | 33 --------------------- 2 files changed, 59 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 237c7598556..628541fe1da 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -168,32 +168,6 @@ fn test_upgrade() { ic_ledger_suite_state_machine_tests::test_upgrade(ledger_wasm(), encode_init_args) } -// It is not possible to downgrade to mainnet for now, disable the tests. -// Similar testing is done by icrc1_test_downgrade_from_incompatible_version -// #[test] -// fn test_install_mainnet_ledger_then_upgrade_then_downgrade() { -// ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( -// ledger_mainnet_wasm(), -// encode_init_args, -// ledger_wasm(), -// encode_upgrade_args, -// ledger_mainnet_wasm(), -// encode_upgrade_args, -// ) -// } - -// #[test] -// fn test_install_current_ledger_then_upgrade_then_downgrade_to_mainnet_version() { -// ic_ledger_suite_state_machine_tests::test_install_upgrade_downgrade( -// ledger_wasm(), -// encode_init_args, -// ledger_wasm(), -// encode_upgrade_args, -// ledger_mainnet_wasm(), -// encode_upgrade_args, -// ) -// } - #[test] fn test_upgrade_archive_options() { ic_ledger_suite_state_machine_tests::test_upgrade_archive_options( diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 76a1f2ceb7d..97b0aec5974 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2096,39 +2096,6 @@ where assert_eq!(token_fee_after_upgrade, NEW_FEE); } -pub fn test_install_upgrade_downgrade( - install_ledger_wasm: Vec, - encode_init_args: fn(InitArgs) -> T, - upgrade_ledger_wasm: Vec, - encode_upgrade_args: fn() -> U, - downgrade_ledger_wasm: Vec, - encode_downgrade_args: fn() -> D, -) where - T: CandidType, - U: CandidType, - D: CandidType, -{ - let (env, canister_id) = setup(install_ledger_wasm.clone(), encode_init_args, vec![]); - - let args = encode_upgrade_args(); - let encoded_upgrade_args = Encode!(&args).unwrap(); - env.upgrade_canister( - canister_id, - upgrade_ledger_wasm, - encoded_upgrade_args.clone(), - ) - .expect("should successfully upgrade ledger canister"); - - let args = encode_downgrade_args(); - let encoded_downgrade_args = Encode!(&args).unwrap(); - env.upgrade_canister( - canister_id, - downgrade_ledger_wasm, - encoded_downgrade_args.clone(), - ) - .expect("should successfully downgrade ledger canister"); -} - pub fn test_memo_max_len(ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T) where T: CandidType, From 752266cca879a112e0d94e3ca5bddc891de9f744 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 23:24:22 +0000 Subject: [PATCH 083/124] remove first version icrc wasm --- MODULE.bazel | 7 ------- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 2 -- rs/ledger_suite/icrc1/ledger/src/main.rs | 2 +- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 098fec780f4..947a3469be6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -612,13 +612,6 @@ http_file( # Financial Integration artifacts for upgrade testing -# first ic-icrc1-ledger release (see https://dashboard.internetcomputer.org/proposal/104499) -http_file( - name = "ic-icrc1-ledger-first-version.wasm.gz", - sha256 = "7af4f7308c883c286d4a5c0448b6bd03bd4d8bffebd58c4fc7114761ad652932", - url = "https://download.dfinity.systems/ic/0456f740295aabdd287811f1ed51829082b3df01/canisters/ic-icrc1-ledger.wasm.gz", -) - # ic-icrc1-ledger releases without ICRC-3 http_file( name = "ic-icrc1-ledger-wo-icrc-3.wasm.gz", diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index a8d8c20ba44..b1b0dd14df9 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -255,7 +255,6 @@ rust_test( ":ledger_canister" + name_suffix + "_lowupgradeinstructionlimits.wasm", ":ledger_canister" + name_suffix + "_nextledgerversion.wasm", "//rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz", - "@ic-icrc1-ledger-first-version.wasm.gz//file", "@mainnet_ckbtc_ic-icrc1-ledger//file", "@mainnet_cketh_ic-icrc1-ledger-u256//file", "@mainnet_ic-icrc1-ledger//file", @@ -266,7 +265,6 @@ rust_test( "CKETH_IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256//file)", "IC_ICRC1_ARCHIVE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz)", "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)", - "IC_ICRC1_LEDGER_FIRST_VERSION_WASM_PATH": "$(rootpath @ic-icrc1-ledger-first-version.wasm.gz//file)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath :ledger_canister" + name_suffix + ".wasm)", "IC_ICRC1_LEDGER_WASM_INSTR_LIMITS_PATH": "$(rootpath :ledger_canister" + name_suffix + "_lowupgradeinstructionlimits.wasm)", "IC_ICRC1_LEDGER_NEXT_VERSION_WASM_PATH": "$(rootpath :ledger_canister" + name_suffix + "_nextledgerversion.wasm)", diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 37a8e69dbec..7fc40ffac32 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -298,7 +298,7 @@ fn log_message(msg: &str) { } fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()> { - w.encode_gauge( + w.encode_gauge( "ledger_stable_memory_pages", ic_cdk::api::stable::stable_size() as f64, "Size of the stable memory allocated by this canister measured in 64K Wasm pages.", From 4386237954a5ba392e89f38eb4c1256e70cdc7ed Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 4 Nov 2024 23:58:34 +0000 Subject: [PATCH 084/124] clippy --- rs/ledger_suite/icrc1/ledger/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 7fc40ffac32..37a8e69dbec 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -298,7 +298,7 @@ fn log_message(msg: &str) { } fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::io::Result<()> { - w.encode_gauge( + w.encode_gauge( "ledger_stable_memory_pages", ic_cdk::api::stable::stable_size() as f64, "Size of the stable memory allocated by this canister measured in 64K Wasm pages.", From 3f4e28738f113dfb89139fac9d389eb448aa13b0 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 5 Nov 2024 20:02:52 +0000 Subject: [PATCH 085/124] check for ready in endpoints --- rs/ledger_suite/icrc1/ledger/src/main.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 37a8e69dbec..e192b1978d6 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -647,6 +647,7 @@ fn execute_transfer_not_async( #[update] #[candid_method(update)] async fn icrc1_transfer(arg: TransferArg) -> Result { + panic_if_not_ready(); let from_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.from_subaccount, @@ -674,6 +675,7 @@ async fn icrc1_transfer(arg: TransferArg) -> Result { #[update] #[candid_method(update)] async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result { + panic_if_not_ready(); let spender_account = Account { owner: ic_cdk::api::caller(), subaccount: arg.spender_subaccount, @@ -780,6 +782,7 @@ fn get_data_certificate() -> DataCertificate { #[update] #[candid_method(update)] async fn icrc2_approve(arg: ApproveArgs) -> Result { + panic_if_not_ready(); let block_idx = Access::with_ledger_mut(|ledger| { let now = TimeStamp::from_nanos_since_unix_epoch(ic_cdk::api::time()); @@ -953,6 +956,10 @@ fn is_ready() -> bool { Access::with_ledger(|ledger| ledger.is_ready()) } +fn panic_if_not_ready() { + Access::with_ledger(|ledger| ledger.panic_if_not_ready()); +} + #[query] #[candid_method(query)] fn is_ledger_ready() -> bool { From 6d48c28a16d9bc9881c78b2e7bc4a86ae6daf86b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 5 Nov 2024 23:26:38 +0000 Subject: [PATCH 086/124] fix test --- rs/ledger_suite/icp/ledger/tests/tests.rs | 1 + rs/ledger_suite/icrc1/ledger/tests/tests.rs | 1 + rs/ledger_suite/tests/sm-tests/src/lib.rs | 15 +++++++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index a9c2c4dc1d9..0ca41ccbba3 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -1335,6 +1335,7 @@ fn test_downgrade_from_incompatible_version() { ledger_wasm_next_version(), ledger_wasm(), encode_init_args, + true, ); } diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index 628541fe1da..fdb2120bf8e 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -448,6 +448,7 @@ fn icrc1_test_downgrade_from_incompatible_version() { ledger_wasm_nextledgerversion(), ledger_wasm(), encode_init_args, + false, ); } diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 97b0aec5974..2848eb731dc 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2562,6 +2562,7 @@ pub fn test_downgrade_from_incompatible_version( ledger_wasm_nextledgerversion: Vec, ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T, + downgrade_to_mainnet_possible: bool, ) where T: CandidType, { @@ -2591,12 +2592,18 @@ pub fn test_downgrade_from_incompatible_version( Encode!(&LedgerArgument::Upgrade(None)).unwrap(), ) { Ok(_) => { - panic!("Upgrade from future ledger version should fail!") + if !downgrade_to_mainnet_possible { + panic!("Upgrade from future ledger version should fail!") + } } Err(e) => { - assert!(e - .description() - .contains("Trying to downgrade from incompatible version")) + if downgrade_to_mainnet_possible { + panic!("Downgrade to mainnet should be possible!") + } else { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } } }; From 9127d6e8725db976131a56f5b59737d61e945d40 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 5 Nov 2024 23:44:40 +0000 Subject: [PATCH 087/124] update canbench results --- .../ledger/canbench_results/canbench_u64.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml index 264cac42052..22fe755925c 100644 --- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml +++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml @@ -1,42 +1,42 @@ benches: bench_icrc1_transfers: total: - instructions: 5911654561 + instructions: 5926738616 heap_increase: 214 stable_memory_increase: 128 scopes: before_upgrade: - instructions: 5315972434 + instructions: 5341274673 heap_increase: 64 stable_memory_increase: 0 post_upgrade: - instructions: 415243938 + instructions: 406888378 heap_increase: 21 stable_memory_increase: 0 pre_upgrade: - instructions: 180434821 + instructions: 178572258 heap_increase: 129 stable_memory_increase: 128 upgrade: - instructions: 595680376 + instructions: 585462226 heap_increase: 150 stable_memory_increase: 128 bench_upgrade_baseline: total: - instructions: 8753711 + instructions: 8758206 heap_increase: 258 stable_memory_increase: 129 scopes: post_upgrade: - instructions: 8621012 + instructions: 8627876 heap_increase: 129 stable_memory_increase: 0 pre_upgrade: - instructions: 130258 + instructions: 127923 heap_increase: 129 stable_memory_increase: 129 upgrade: - instructions: 8753004 + instructions: 8757517 heap_increase: 258 stable_memory_increase: 129 version: 0.1.7 From 6fed6f302c8382fb42a106e6883dca3bd72ce67d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 6 Nov 2024 16:24:13 +0000 Subject: [PATCH 088/124] test metrics while updating --- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 9 +++ rs/ledger_suite/tests/sm-tests/src/lib.rs | 72 ++++++++++++++++++- rs/ledger_suite/tests/sm-tests/src/metrics.rs | 2 +- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index fdb2120bf8e..ad0efdd5bcf 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -470,6 +470,15 @@ fn icrc1_test_incomplete_migration() { ); } +#[test] +fn icrc1_test_metrics_while_migrating() { + ic_ledger_suite_state_machine_tests::test_metrics_while_migrating( + ledger_mainnet_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), + encode_init_args, + ); +} + mod metrics { use crate::{encode_init_args, encode_upgrade_args, ledger_wasm}; use ic_ledger_suite_state_machine_tests::metrics::LedgerSuiteType; diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 532aaa64295..25c0d0dceb1 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -1,5 +1,5 @@ use crate::in_memory_ledger::{verify_ledger_state, InMemoryLedger}; -use crate::metrics::parse_metric; +use crate::metrics::{parse_metric, retrieve_metrics}; use candid::{CandidType, Decode, Encode, Int, Nat, Principal}; use ic_agent::identity::{BasicIdentity, Identity}; use ic_base_types::CanisterId; @@ -2801,6 +2801,76 @@ pub fn test_incomplete_migration( check_approvals(); } +pub fn test_metrics_while_migrating( + ledger_wasm_mainnet: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + let account = Account::from(PrincipalId::new_user_test_id(1).0); + let initial_balances = vec![(account, 100_000_000u64)]; + + // Setup ledger as it is deployed on the mainnet. + let (env, canister_id) = setup( + ledger_wasm_mainnet.clone(), + encode_init_args, + initial_balances, + ); + + for i in 2..22 { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, 150_000); + send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); + } + + env.upgrade_canister( + canister_id, + ledger_wasm_current_lowinstructionlimits, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + let metrics = retrieve_metrics(&env, canister_id); + assert!( + metrics + .iter() + .any(|line| line.contains("ledger_transactions")), + "Did not find ledger_transactions metric" + ); + assert!( + !metrics + .iter() + .any(|line| line.contains("ledger_total_supply")), + "ledger_total_supply should not be in metrics" + ); + + let is_ledger_ready = Decode!( + &env.query(canister_id, "is_ledger_ready", Encode!().unwrap()) + .expect("failed to call is_ledger_ready") + .bytes(), + bool + ) + .expect("failed to decode is_ledger_ready response"); + assert!(!is_ledger_ready); + + wait_ledger_ready(&env, canister_id, 10); + + let metrics = retrieve_metrics(&env, canister_id); + assert!( + metrics + .iter() + .any(|line| line.contains("ledger_transactions")), + "Did not find ledger_transactions metric" + ); + assert!( + metrics + .iter() + .any(|line| line.contains("ledger_total_supply")), + "Did not find ledger_total_supply metric" + ); +} + pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { ApproveArgs { from_subaccount: None, diff --git a/rs/ledger_suite/tests/sm-tests/src/metrics.rs b/rs/ledger_suite/tests/sm-tests/src/metrics.rs index 5f6fd4257ee..cfeb34d4405 100644 --- a/rs/ledger_suite/tests/sm-tests/src/metrics.rs +++ b/rs/ledger_suite/tests/sm-tests/src/metrics.rs @@ -187,7 +187,7 @@ pub(crate) fn parse_metric(env: &StateMachine, canister_id: CanisterId, metric: panic!("metric '{}' not found in metrics: {:?}", metric, metrics); } -fn retrieve_metrics(env: &StateMachine, canister_id: CanisterId) -> Vec { +pub(crate) fn retrieve_metrics(env: &StateMachine, canister_id: CanisterId) -> Vec { let request = HttpRequest { method: "GET".to_string(), url: "/metrics".to_string(), From 8afc10d878552ec6930cfbd5fef9da773afd2ffe Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 6 Nov 2024 18:43:07 +0000 Subject: [PATCH 089/124] log migration step instructions --- rs/ledger_suite/icrc1/ledger/src/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index e192b1978d6..0ae40bd5614 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -248,6 +248,7 @@ fn post_upgrade(args: Option) { } fn migrate_next_part(instruction_limit: u64) { + let instructions_mingration_start = instruction_counter(); STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow_mut() += 1); let mut migrated_allowances = 0; let mut migrated_expirations = 0; @@ -277,7 +278,8 @@ fn migrate_next_part(instruction_limit: u64) { } } } - let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Total instructions used in message: {}." , + let instructions_mingration = instruction_counter() - instructions_mingration_start; + let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_mingration}, total instructions used in message: {}." , instruction_counter()); if !ledger.is_ready() { log_message( From 86656951223d47befb76afd5d3c13db566bdc0b6 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 6 Nov 2024 19:40:01 +0000 Subject: [PATCH 090/124] test timestamp serialization --- rs/ledger_suite/common/ledger_core/src/timestamp.rs | 3 +++ .../common/ledger_core/src/timestamp/tests.rs | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 rs/ledger_suite/common/ledger_core/src/timestamp/tests.rs diff --git a/rs/ledger_suite/common/ledger_core/src/timestamp.rs b/rs/ledger_suite/common/ledger_core/src/timestamp.rs index c4bd1fd06d3..17f05835e88 100644 --- a/rs/ledger_suite/common/ledger_core/src/timestamp.rs +++ b/rs/ledger_suite/common/ledger_core/src/timestamp.rs @@ -6,6 +6,9 @@ use std::convert::TryInto; use std::ops::{Add, Sub}; use std::time::{Duration, SystemTime}; +#[cfg(test)] +mod tests; + #[derive( Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, CandidType, Deserialize, Serialize, )] diff --git a/rs/ledger_suite/common/ledger_core/src/timestamp/tests.rs b/rs/ledger_suite/common/ledger_core/src/timestamp/tests.rs new file mode 100644 index 00000000000..0bf4443fcde --- /dev/null +++ b/rs/ledger_suite/common/ledger_core/src/timestamp/tests.rs @@ -0,0 +1,12 @@ +use crate::timestamp::TimeStamp; +use ic_stable_structures::Storable; +use proptest::prelude::{any, prop_assert_eq, proptest}; + +#[test] +fn timestamp_serialization() { + proptest!(|(ts_u64 in any::())| { + let timestamp = TimeStamp::from_nanos_since_unix_epoch(ts_u64); + let new_timestamp = TimeStamp::from_bytes(timestamp.to_bytes()); + prop_assert_eq!(new_timestamp, timestamp); + }) +} From 695394253b13abda70aa3281f7f6332f333bf695 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 7 Nov 2024 00:01:17 +0000 Subject: [PATCH 091/124] use thread local for ledger state --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 46 +++++++++++++----------- rs/ledger_suite/icrc1/ledger/src/main.rs | 26 ++++++-------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 03be1ce6f2d..78db8bc0c83 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -359,6 +359,8 @@ thread_local! { pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); + pub static LEDGER_STATE: RefCell = RefCell::new(LedgerState::Ready); + // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = @@ -370,13 +372,13 @@ thread_local! { MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] pub enum LedgerField { Allowances, AllowancesExpirations, } -#[derive(Clone, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] pub enum LedgerState { Migrating(LedgerField), Ready, @@ -424,9 +426,6 @@ pub struct Ledger { #[serde(default = "default_ledger_version")] pub ledger_version: u64, - - #[serde(default)] - pub state: LedgerState, } fn default_maximum_number_of_accounts() -> usize { @@ -525,7 +524,6 @@ impl Ledger { .try_into() .unwrap(), ledger_version: LEDGER_VERSION, - state: LedgerState::Ready, }; for (account, balance) in initial_balances.into_iter() { @@ -547,16 +545,6 @@ impl Ledger { ledger } - pub fn is_ready(&self) -> bool { - matches!(self.state, LedgerState::Ready) - } - - pub fn panic_if_not_ready(&self) { - if !self.is_ready() { - ic_cdk::trap("The Ledger is not ready"); - } - } - pub fn migrate_one_allowance(&mut self) -> bool { match self.approvals.allowances_data.pop_first_allowance() { Some((account_spender, allowance)) => { @@ -593,22 +581,22 @@ impl LedgerContext for Ledger { type Tokens = Tokens; fn balances(&self) -> &Balances { - self.panic_if_not_ready(); + panic_if_not_ready(); &self.balances } fn balances_mut(&mut self) -> &mut Balances { - self.panic_if_not_ready(); + panic_if_not_ready(); &mut self.balances } fn approvals(&self) -> &AllowanceTable { - self.panic_if_not_ready(); + panic_if_not_ready(); &self.stable_approvals } fn approvals_mut(&mut self) -> &mut AllowanceTable { - self.panic_if_not_ready(); + panic_if_not_ready(); &mut self.stable_approvals } @@ -969,6 +957,24 @@ impl Ledger { } } +pub fn is_ready() -> bool { + LEDGER_STATE.with(|s| matches!(*s.borrow(), LedgerState::Ready)) +} + +pub fn panic_if_not_ready() { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } +} + +pub fn ledger_state() -> LedgerState { + LEDGER_STATE.with(|s| *s.borrow()) +} + +pub fn set_ledger_state(ledger_state: LedgerState) { + LEDGER_STATE.with(|s| *s.borrow_mut() = ledger_state); +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct StableAllowancesData {} diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 0ae40bd5614..ba9d2786b38 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -15,8 +15,10 @@ use ic_icrc1::{ endpoints::{convert_transfer_error, StandardRecord}, Operation, Transaction, }; +use ic_icrc1_ledger::{ + is_ready, ledger_state, panic_if_not_ready, set_ledger_state, LEDGER_VERSION, UPGRADES_MEMORY, +}; use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; -use ic_icrc1_ledger::{LEDGER_VERSION, UPGRADES_MEMORY}; use ic_ledger_canister_core::ledger::{ apply_transaction, archive_blocks, LedgerAccess, LedgerContext, LedgerData, TransferError as CoreTransferError, @@ -135,7 +137,7 @@ fn pre_upgrade() { let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { Access::with_ledger(|ledger| { - if !ledger.is_ready() { + if !is_ready() { // This means that migration did not complete and the correct state // of the ledger is still in UPGRADES_MEMORY. ic_cdk::println!("Ledger not ready, skipping write to UPGRADES_MEMORY."); @@ -233,8 +235,8 @@ fn post_upgrade(args: Option) { PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); + set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); Access::with_ledger_mut(|ledger| { - ledger.state = LedgerState::Migrating(LedgerField::Allowances); ledger.clear_arrivals(); }); log_message("Migration started."); @@ -257,7 +259,7 @@ fn migrate_next_part(instruction_limit: u64) { Access::with_ledger_mut(|ledger| { while instruction_counter() < instruction_limit { - let field = match ledger.state.clone() { + let field = match ledger_state() { LedgerState::Migrating(ledger_field) => ledger_field, LedgerState::Ready => break, }; @@ -266,14 +268,16 @@ fn migrate_next_part(instruction_limit: u64) { if ledger.migrate_one_allowance() { migrated_allowances += 1; } else { - ledger.state = LedgerState::Migrating(LedgerField::AllowancesExpirations); + set_ledger_state(LedgerState::Migrating( + LedgerField::AllowancesExpirations, + )); } } LedgerField::AllowancesExpirations => { if ledger.migrate_one_expiration() { migrated_expirations += 1; } else { - ledger.state = LedgerState::Ready; + set_ledger_state(LedgerState::Ready); } } } @@ -281,7 +285,7 @@ fn migrate_next_part(instruction_limit: u64) { let instructions_mingration = instruction_counter() - instructions_mingration_start; let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_mingration}, total instructions used in message: {}." , instruction_counter()); - if !ledger.is_ready() { + if !is_ready() { log_message( format!("Migration partially done. Scheduling the next part. {msg}").as_str(), ); @@ -954,14 +958,6 @@ fn icrc21_canister_call_consent_message( ) } -fn is_ready() -> bool { - Access::with_ledger(|ledger| ledger.is_ready()) -} - -fn panic_if_not_ready() { - Access::with_ledger(|ledger| ledger.panic_if_not_ready()); -} - #[query] #[candid_method(query)] fn is_ledger_ready() -> bool { From d3f51c4eb839d827db815398da41691f22ab259f Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 7 Nov 2024 00:13:44 +0000 Subject: [PATCH 092/124] clippy --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 78db8bc0c83..8998610d1a7 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -359,7 +359,7 @@ thread_local! { pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); - pub static LEDGER_STATE: RefCell = RefCell::new(LedgerState::Ready); + pub static LEDGER_STATE: RefCell = const { RefCell::new(LedgerState::Ready) }; // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] From a7fd4abcf8f162690bcced43da2515ac813160fd Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 7 Nov 2024 02:31:40 +0000 Subject: [PATCH 093/124] update canbench results --- .../ledger/canbench_results/canbench_u256.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml index c37d9397ecd..77770b0cfb7 100644 --- a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml +++ b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml @@ -1,42 +1,42 @@ benches: bench_icrc1_transfers: total: - instructions: 6390210524 + instructions: 6398923208 heap_increase: 227 stable_memory_increase: 128 scopes: before_upgrade: - instructions: 5769009086 + instructions: 5790054103 heap_increase: 71 stable_memory_increase: 0 post_upgrade: - instructions: 434388599 + instructions: 424320009 heap_increase: 27 stable_memory_increase: 0 pre_upgrade: - instructions: 186809754 + instructions: 184546072 heap_increase: 129 stable_memory_increase: 128 upgrade: - instructions: 621199970 + instructions: 608867671 heap_increase: 156 stable_memory_increase: 128 bench_upgrade_baseline: total: - instructions: 8750052 + instructions: 8755062 heap_increase: 258 stable_memory_increase: 129 scopes: post_upgrade: - instructions: 8619094 + instructions: 8627524 heap_increase: 129 stable_memory_increase: 0 pre_upgrade: - instructions: 128517 + instructions: 125131 heap_increase: 129 stable_memory_increase: 129 upgrade: - instructions: 8749345 + instructions: 8754373 heap_increase: 258 stable_memory_increase: 129 version: 0.1.7 From b33b6a0b7e27964a913b9543e03f412179e3f2dc Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 8 Nov 2024 09:04:12 +0000 Subject: [PATCH 094/124] migrate some allowances in post_upgrade --- rs/ledger_suite/icrc1/ledger/src/main.rs | 2 +- rs/ledger_suite/tests/sm-tests/src/lib.rs | 60 ++++++++++++----------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index ba9d2786b38..17fa824f184 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -164,7 +164,7 @@ const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 199_950_000_000; const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_950_000_000; #[cfg(feature = "low-upgrade-instruction-limits")] -const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 0; +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 13_000_000; #[cfg(feature = "low-upgrade-instruction-limits")] const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 500_000; diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 25c0d0dceb1..e55851aca99 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2490,42 +2490,44 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( balances.insert(account, Nat::from(balance_of(&env, canister_id, *account))); } - let test_upgrade = |ledger_wasm: Vec, balances: BTreeMap<&Account, Nat>| { - env.upgrade_canister( - canister_id, - ledger_wasm, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .unwrap(); + let test_upgrade = + |ledger_wasm: Vec, balances: BTreeMap<&Account, Nat>, min_migration_steps: u64| { + env.upgrade_canister( + canister_id, + ledger_wasm, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); - wait_ledger_ready(&env, canister_id, 10); - - let stable_upgrade_migration_steps = - parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); - assert!(stable_upgrade_migration_steps > 1); - - let mut allowances = vec![]; - for i in 0..accounts.len() { - for j in i + 1..accounts.len() { - let allowance = get_allowance(&env, canister_id, accounts[i], accounts[j]); - assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); - allowances.push(allowance); - let allowance = get_allowance(&env, canister_id, accounts[j], accounts[i]); - assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); - allowances.push(allowance); + wait_ledger_ready(&env, canister_id, 10); + + let stable_upgrade_migration_steps = + parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + assert!(stable_upgrade_migration_steps >= min_migration_steps); + + let mut allowances = vec![]; + for i in 0..accounts.len() { + for j in i + 1..accounts.len() { + let allowance = get_allowance(&env, canister_id, accounts[i], accounts[j]); + assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); + allowances.push(allowance); + let allowance = get_allowance(&env, canister_id, accounts[j], accounts[i]); + assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); + allowances.push(allowance); + } } - } - assert_eq!(expected_allowances, allowances); + assert_eq!(expected_allowances, allowances); - for account in &all_accounts { - assert_eq!(balance_of(&env, canister_id, *account), balances[account]); - } - }; + for account in &all_accounts { + assert_eq!(balance_of(&env, canister_id, *account), balances[account]); + } + }; // Test if the old serialized approvals and balances are correctly deserialized test_upgrade( ledger_wasm_current_lowinstructionlimits.clone(), balances.clone(), + 2, ); // Add some more approvals @@ -2545,7 +2547,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } // Test the new wasm serialization - test_upgrade(ledger_wasm_current_lowinstructionlimits, balances); + test_upgrade(ledger_wasm_current_lowinstructionlimits, balances, 1); // See if the additional approvals are there for a1 in &accounts { From a4fa643b5faf8d55580801682a43f1ad8f9b5c8e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 11 Nov 2024 14:03:26 +0000 Subject: [PATCH 095/124] speed up account serialization and deserialization --- .../icrc-ledger-types/src/icrc1/account.rs | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index b9894123c55..c161ceb7c22 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -9,7 +9,6 @@ use candid::{types::principal::PrincipalError, CandidType, Deserialize, Principa use ic_stable_structures::{storable::Bound, Storable}; use serde::Serialize; use std::borrow::Cow; -use std::io::{Cursor, Read}; pub type Subaccount = [u8; 32]; @@ -170,46 +169,45 @@ impl FromStr for Account { } } -const MAX_SERIALIZATION_LEN: u32 = 62; +const MAX_SERIALIZATION_LEN: u32 = 63; impl Storable for Account { fn to_bytes(&self) -> Cow<[u8]> { - let mut buffer: Vec = vec![]; - let mut buffer0: Vec = vec![]; + let mut buffer: Vec = vec![0]; + // Owner principal + buffer.extend(self.owner.as_slice()); + buffer[0] = (buffer.len() - 1) as u8; + + // Subaccount if let Some(subaccount) = self.subaccount { - buffer0.extend(subaccount.as_slice()); + buffer.extend(32u8.to_le_bytes()); + buffer.extend(subaccount.as_slice()); + } else { + buffer.extend(0u8.to_le_bytes()); } - buffer0.extend(self.owner.as_slice()); - buffer.extend((buffer0.len() as u8).to_le_bytes()); - buffer.append(&mut buffer0); Cow::Owned(buffer) } fn from_bytes(bytes: Cow<[u8]>) -> Self { - let mut cursor = Cursor::new(bytes); - - let mut len_bytes = [0u8; 1]; - cursor - .read_exact(&mut len_bytes) - .expect("Unable to read the len of the account"); - let mut len = u8::from_le_bytes(len_bytes); - let subaccount = if len >= 32 { - let mut subaccount_bytes = [0u8; 32]; - cursor - .read_exact(&mut subaccount_bytes) - .expect("Unable to read the bytes of the account's subaccount"); - len -= 32; + let mut index = 0usize; + let len_owner = bytes[index] as usize; + index += 1; + let owner = Principal::from_slice(&bytes[index..index + len_owner]); + index += len_owner; + + // Subaccount + let len_subaccount = bytes[index] as usize; + index += 1; + let subaccount = if len_subaccount > 0 { + let subaccount_bytes: [u8; 32] = + bytes[index..index + len_subaccount].try_into().unwrap(); Some(subaccount_bytes) } else { None }; - let mut owner_bytes = vec![0; len as usize]; - cursor - .read_exact(&mut owner_bytes) - .expect("Unable to read the bytes of the account's owners"); - let owner = Principal::from_slice(&owner_bytes); + Account { owner, subaccount } } @@ -395,7 +393,7 @@ mod tests { let serialized_len = account.to_bytes().len(); assert_eq!( serialized_len, - 1 + DEFAULT_SUBACCOUNT.len() + Principal::MAX_LENGTH_IN_BYTES + 1 + DEFAULT_SUBACCOUNT.len() + 1 + Principal::MAX_LENGTH_IN_BYTES ); assert_eq!(serialized_len as u32, MAX_SERIALIZATION_LEN); } From 2730a5cfa3e991ffef1101a5ad4e62db8c84c2a6 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 12 Nov 2024 13:40:21 +0000 Subject: [PATCH 096/124] simplify the code a bit --- packages/icrc-ledger-types/src/icrc1/account.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index c161ceb7c22..017c8ff04bd 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -181,10 +181,10 @@ impl Storable for Account { // Subaccount if let Some(subaccount) = self.subaccount { - buffer.extend(32u8.to_le_bytes()); + buffer.extend([32u8]); buffer.extend(subaccount.as_slice()); } else { - buffer.extend(0u8.to_le_bytes()); + buffer.extend([0u8]); } Cow::Owned(buffer) From 186b8928ea2751da5605bb4dbe2e13265ba2806b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 12 Nov 2024 15:53:11 +0000 Subject: [PATCH 097/124] migrating with interval timer --- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 1 + rs/ledger_suite/icrc1/ledger/src/lib.rs | 38 ++++++++++++++--- rs/ledger_suite/icrc1/ledger/src/main.rs | 23 +++++++--- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 9 ++++ rs/ledger_suite/tests/sm-tests/src/lib.rs | 47 ++++++++++++++++++++- 5 files changed, 104 insertions(+), 14 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index b1b0dd14df9..d56840aaa4a 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -36,6 +36,7 @@ package(default_visibility = ["//visibility:public"]) "@crate_index//:ciborium", "@crate_index//:hex", "@crate_index//:ic-cdk", + "@crate_index//:ic-cdk-timers", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", "@crate_index//:serde", diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 8998610d1a7..5bfc1f0b5f8 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -10,6 +10,7 @@ use candid::{ }; use ic_base_types::PrincipalId; use ic_canister_log::{log, Sink}; +use ic_cdk_timers::TimerId; use ic_crypto_tree_hash::{Label, MixedHashTree}; use ic_icrc1::blocks::encoded_block_to_generic_block; use ic_icrc1::{Block, LedgerAllowances, LedgerBalances, Transaction}; @@ -359,7 +360,10 @@ thread_local! { pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); - pub static LEDGER_STATE: RefCell = const { RefCell::new(LedgerState::Ready) }; + pub static MIGRATION_STATE: RefCell = const { RefCell::new(MigrationState { + ledger_state: LedgerState::Ready, + timer_id: None, + }) }; // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] @@ -372,13 +376,13 @@ thread_local! { MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } -#[derive(Copy, Clone, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, Debug)] pub enum LedgerField { Allowances, AllowancesExpirations, } -#[derive(Copy, Clone, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, Debug)] pub enum LedgerState { Migrating(LedgerField), Ready, @@ -390,6 +394,20 @@ impl Default for LedgerState { } } +pub struct MigrationState { + ledger_state: LedgerState, + timer_id: Option, +} + +impl Default for MigrationState { + fn default() -> Self { + Self { + ledger_state: LedgerState::default(), + timer_id: None, + } + } +} + #[derive(Debug, Deserialize, Serialize)] #[serde(bound = "")] pub struct Ledger { @@ -958,7 +976,7 @@ impl Ledger { } pub fn is_ready() -> bool { - LEDGER_STATE.with(|s| matches!(*s.borrow(), LedgerState::Ready)) + MIGRATION_STATE.with(|s| matches!((*s.borrow()).ledger_state, LedgerState::Ready)) } pub fn panic_if_not_ready() { @@ -968,11 +986,19 @@ pub fn panic_if_not_ready() { } pub fn ledger_state() -> LedgerState { - LEDGER_STATE.with(|s| *s.borrow()) + MIGRATION_STATE.with(|s| (*s.borrow()).ledger_state) } pub fn set_ledger_state(ledger_state: LedgerState) { - LEDGER_STATE.with(|s| *s.borrow_mut() = ledger_state); + MIGRATION_STATE.with(|s| (*s.borrow_mut()).ledger_state = ledger_state); +} + +pub fn migration_timer_id() -> Option { + MIGRATION_STATE.with(|s| (*s.borrow()).timer_id) +} + +pub fn set_migration_timer_id(timer_id: Option) { + MIGRATION_STATE.with(|s| (*s.borrow_mut()).timer_id = timer_id); } #[derive(Serialize, Deserialize, Debug, Default)] diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 17fa824f184..dc9de680a87 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -16,7 +16,8 @@ use ic_icrc1::{ Operation, Transaction, }; use ic_icrc1_ledger::{ - is_ready, ledger_state, panic_if_not_ready, set_ledger_state, LEDGER_VERSION, UPGRADES_MEMORY, + is_ready, ledger_state, migration_timer_id, panic_if_not_ready, set_ledger_state, + set_migration_timer_id, LEDGER_VERSION, UPGRADES_MEMORY, }; use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; use ic_ledger_canister_core::ledger::{ @@ -243,6 +244,12 @@ fn post_upgrade(args: Option) { migrate_next_part( MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), ); + if !is_ready() { + let timer_id = ic_cdk_timers::set_timer_interval(Duration::from_secs(1), || { + migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) + }); + set_migration_timer_id(Some(timer_id)); + } let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; @@ -286,14 +293,16 @@ fn migrate_next_part(instruction_limit: u64) { let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_mingration}, total instructions used in message: {}." , instruction_counter()); if !is_ready() { - log_message( - format!("Migration partially done. Scheduling the next part. {msg}").as_str(), - ); - ic_cdk_timers::set_timer(Duration::from_secs(0), || { - migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) - }); + log_message(format!("Migration partially done. {msg}").as_str()); } else { log_message(format!("Migration completed! {msg}").as_str()); + if let Some(timer_id) = migration_timer_id() { + log_message(format!("Canceling timer with timer_id {:?}", timer_id).as_str()); + ic_cdk_timers::clear_timer(timer_id); + set_migration_timer_id(None); + } else { + log_message("Migration timer was not scheduled."); + } } }); } diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index ad0efdd5bcf..eba6bc2ffb9 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -479,6 +479,15 @@ fn icrc1_test_metrics_while_migrating() { ); } +#[test] +fn icrc1_test_migration_timer_canceled() { + ic_ledger_suite_state_machine_tests::test_migration_timer_canceled( + ledger_mainnet_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), + encode_init_args, + ); +} + mod metrics { use crate::{encode_init_args, encode_upgrade_args, ledger_wasm}; use ic_ledger_suite_state_machine_tests::metrics::LedgerSuiteType; diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 41f3b77a1e3..0e7dad7186f 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2719,7 +2719,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), true); test_endpoint("icrc1_total_supply", Encode!().unwrap(), true); - wait_ledger_ready(&env, canister_id, 10); + wait_ledger_ready(&env, canister_id, 20); test_endpoint("icrc1_transfer", Encode!(&transfer_args).unwrap(), false); test_endpoint("icrc2_approve", Encode!(&approve_args).unwrap(), false); @@ -2877,6 +2877,51 @@ pub fn test_metrics_while_migrating( ); } +pub fn test_migration_timer_canceled( + ledger_wasm_mainnet: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + let account = Account::from(PrincipalId::new_user_test_id(1).0); + let initial_balances = vec![(account, 100_000_000u64)]; + + // Setup ledger as it is deployed on the mainnet. + let (env, canister_id) = setup(ledger_wasm_mainnet, encode_init_args, initial_balances); + + const APPROVE_AMOUNT: u64 = 150_000; + + for i in 2..40 { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, APPROVE_AMOUNT); + send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); + } + + env.upgrade_canister( + canister_id, + ledger_wasm_current_lowinstructionlimits, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + wait_ledger_ready(&env, canister_id, 20); + + let stable_upgrade_migration_steps = + parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + assert!(stable_upgrade_migration_steps > 1); + + env.advance_time(Duration::from_secs(1000)); + env.tick(); + + let later_stable_upgrade_migration_steps = + parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); + assert_eq!( + stable_upgrade_migration_steps, + later_stable_upgrade_migration_steps + ); +} + pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { ApproveArgs { from_subaccount: None, From a5ec43d5cb5c5b6f78f1f9aaaa2ba0fc5dea080a Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Fri, 15 Nov 2024 12:27:35 +0000 Subject: [PATCH 098/124] store allowance expiration length while serializing --- .../common/ledger_core/src/approvals.rs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 5068d9b763a..dae3ec969c5 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -500,7 +500,10 @@ impl + TryFrom> Storable for Allo .encode(&mut buffer) .expect("Unable to serialize amount"); if let Some(expires_at) = self.expires_at { + buffer.extend([8u8]); buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); + } else { + buffer.extend([0u8]); } // We don't serialize arrived_at - it is not used after stable structures migration. Cow::Owned(buffer) @@ -512,12 +515,20 @@ impl + TryFrom> Storable for Allo let amount = Tokens::try_from(amount).expect("Unable to convert Nat to Tokens"); // arrived_at was not serialized, use a default value. let arrived_at = TimeStamp::from_nanos_since_unix_epoch(0); - let mut expires_at_bytes = [0u8; 8]; - let expires_at = match cursor.read_exact(&mut expires_at_bytes) { - Ok(()) => Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( + let mut expires_at_length_bytes = [0u8; 1]; + cursor + .read_exact(&mut expires_at_length_bytes) + .expect("could not read expires_at_length_bytes"); + let expires_at = if expires_at_length_bytes[0] > 0 { + let mut expires_at_bytes = [0u8; 8]; + cursor + .read_exact(&mut expires_at_bytes) + .expect("could not read expires_at"); + Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( expires_at_bytes, - ))), - _ => None, + ))) + } else { + None }; Self { amount, From c93ecbb0240a7cc45e128613d29ef33bdc794912 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 18 Nov 2024 11:09:13 +0000 Subject: [PATCH 099/124] test if migration resumes when frozen and unfrozen --- Cargo.lock | 3 + rs/ledger_suite/icrc1/ledger/tests/tests.rs | 9 ++ rs/ledger_suite/tests/sm-tests/BUILD.bazel | 3 + rs/ledger_suite/tests/sm-tests/Cargo.toml | 3 + rs/ledger_suite/tests/sm-tests/src/lib.rs | 104 +++++++++++++++++++- 5 files changed, 121 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 481d96b1b9a..4dd4c6ad6c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9416,6 +9416,7 @@ name = "ic-ledger-suite-state-machine-tests" version = "0.9.0" dependencies = [ "anyhow", + "assert_matches", "async-trait", "candid", "cddl", @@ -9424,6 +9425,7 @@ dependencies = [ "ic-agent", "ic-base-types", "ic-canisters-http-types", + "ic-config", "ic-error-types", "ic-icrc1", "ic-icrc1-ledger", @@ -9434,6 +9436,7 @@ dependencies = [ "ic-ledger-core", "ic-ledger-hash-of", "ic-management-canister-types", + "ic-registry-subnet-type", "ic-rosetta-test-utils", "ic-state-machine-tests", "ic-types", diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index eba6bc2ffb9..da5f3608828 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -470,6 +470,15 @@ fn icrc1_test_incomplete_migration() { ); } +#[test] +fn icrc1_test_migration_resumes_from_frozen() { + ic_ledger_suite_state_machine_tests::test_migration_resumes_from_frozen( + ledger_mainnet_wasm(), + ledger_wasm_lowupgradeinstructionlimits(), + encode_init_args, + ); +} + #[test] fn icrc1_test_metrics_while_migrating() { ic_ledger_suite_state_machine_tests::test_metrics_while_migrating( diff --git a/rs/ledger_suite/tests/sm-tests/BUILD.bazel b/rs/ledger_suite/tests/sm-tests/BUILD.bazel index 381f807caf0..ab3e462c843 100644 --- a/rs/ledger_suite/tests/sm-tests/BUILD.bazel +++ b/rs/ledger_suite/tests/sm-tests/BUILD.bazel @@ -6,11 +6,13 @@ DEPENDENCIES = [ # Keep sorted. "//packages/ic-ledger-hash-of:ic_ledger_hash_of", "//packages/icrc-ledger-types:icrc_ledger_types", + "//rs/config", "//rs/ledger_suite/common/ledger_canister_core", "//rs/ledger_suite/common/ledger_core", "//rs/ledger_suite/icrc1", "//rs/ledger_suite/icrc1/ledger", "//rs/ledger_suite/icrc1/test_utils", + "//rs/registry/subnet_type", "//rs/rosetta-api/icp/test_utils", "//rs/rust_canisters/http_types", "//rs/state_machine_tests", @@ -20,6 +22,7 @@ DEPENDENCIES = [ "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", + "@crate_index//:assert_matches", "@crate_index//:candid", "@crate_index//:cddl", "@crate_index//:futures", diff --git a/rs/ledger_suite/tests/sm-tests/Cargo.toml b/rs/ledger_suite/tests/sm-tests/Cargo.toml index e96cf4d8664..cf559ad81c7 100644 --- a/rs/ledger_suite/tests/sm-tests/Cargo.toml +++ b/rs/ledger_suite/tests/sm-tests/Cargo.toml @@ -8,6 +8,7 @@ documentation.workspace = true [dependencies] anyhow = { workspace = true } +assert_matches = { workspace = true } async-trait = { workspace = true } candid = { workspace = true } cddl = "0.9.4" @@ -16,6 +17,7 @@ hex = { workspace = true } ic-agent = { workspace = true } ic-base-types = { path = "../../../types/base_types" } ic-canisters-http-types = { path = "../../../rust_canisters/http_types" } +ic-config = { path = "../../../config" } ic-error-types = { path = "../../../types/error_types" } ic-icrc1 = { path = "../../icrc1" } ic-icrc1-ledger = { path = "../../icrc1/ledger" } @@ -27,6 +29,7 @@ ic-ledger-core = { path = "../../common/ledger_core" } ic-ledger-hash-of = { path = "../../../../packages/ic-ledger-hash-of" } ic-rosetta-test-utils = { path = "../../../rosetta-api/icp/test_utils" } ic-management-canister-types = { path = "../../../types/management_canister_types" } +ic-registry-subnet-type = { path = "../../../registry/subnet_type" } ic-state-machine-tests = { path = "../../../state_machine_tests" } ic-types = { path = "../../../types/types" } ic-universal-canister = { path = "../../../universal_canister/lib" } diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 0e7dad7186f..22e2df829e6 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -1,9 +1,11 @@ use crate::in_memory_ledger::{verify_ledger_state, InMemoryLedger}; use crate::metrics::{parse_metric, retrieve_metrics}; +use assert_matches::assert_matches; use candid::{CandidType, Decode, Encode, Int, Nat, Principal}; use ic_agent::identity::{BasicIdentity, Identity}; use ic_base_types::CanisterId; use ic_base_types::PrincipalId; +use ic_config::{execution_environment::Config as HypervisorConfig, subnet_config::SubnetConfig}; use ic_error_types::UserError; use ic_icrc1::blocks::encoded_block_to_generic_block; use ic_icrc1::{endpoints::StandardRecord, hash::Hash, Block, Operation, Transaction}; @@ -13,11 +15,13 @@ use ic_ledger_canister_core::archive::ArchiveOptions; use ic_ledger_core::block::{BlockIndex, BlockType}; use ic_ledger_core::timestamp::TimeStamp; use ic_ledger_hash_of::HashOf; +use ic_management_canister_types::CanisterSettingsArgsBuilder; use ic_management_canister_types::{ self as ic00, CanisterInfoRequest, CanisterInfoResponse, Method, Payload, }; +use ic_registry_subnet_type::SubnetType; use ic_rosetta_test_utils::test_http_request_decoding_quota; -use ic_state_machine_tests::{ErrorCode, StateMachine, WasmResult}; +use ic_state_machine_tests::{ErrorCode, StateMachine, StateMachineConfig, WasmResult}; use ic_types::Cycles; use ic_universal_canister::{call_args, wasm, UNIVERSAL_CANISTER_WASM}; use icrc_ledger_types::icrc::generic_metadata_value::MetadataValue as Value; @@ -2807,6 +2811,104 @@ pub fn test_incomplete_migration( check_approvals(); } +pub fn test_migration_resumes_from_frozen( + ledger_wasm_mainnet: Vec, + ledger_wasm_current_lowinstructionlimits: Vec, + encode_init_args: fn(InitArgs) -> T, +) where + T: CandidType, +{ + let account = Account::from(PrincipalId::new_user_test_id(1).0); + let initial_balances = vec![(account, 100_000_000u64)]; + + let subnet_config = SubnetConfig::new(SubnetType::Application); + let env = StateMachine::new_with_config(StateMachineConfig::new( + subnet_config.clone(), + HypervisorConfig::default(), + )); + + let args = encode_init_args(init_args(initial_balances)); + let args = Encode!(&args).unwrap(); + let canister_id = env + .install_canister_with_cycles( + ledger_wasm_mainnet, + args, + None, + Cycles::new(1_000_000_000_000), + ) + .unwrap(); + + const APPROVE_AMOUNT: u64 = 150_000; + const NUM_APPROVALS: u64 = 20; + + let send_approvals = || { + for i in 2..2 + NUM_APPROVALS { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, APPROVE_AMOUNT); + send_approval(&env, canister_id, account.owner, &approve_args) + .expect("approval failed"); + } + }; + + send_approvals(); + + let check_approvals = || { + for i in 2..2 + NUM_APPROVALS { + let allowance = get_allowance( + &env, + canister_id, + account, + Account::from(PrincipalId::new_user_test_id(i).0), + ); + assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); + } + }; + + check_approvals(); + + env.upgrade_canister( + canister_id, + ledger_wasm_current_lowinstructionlimits, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + + let is_ledger_ready = || { + Decode!( + &env.query(canister_id, "is_ledger_ready", Encode!().unwrap()) + .expect("failed to call is_ledger_ready") + .bytes(), + bool + ) + .expect("failed to decode is_ledger_ready response") + }; + assert!(!is_ledger_ready()); + + let freeze = |env: &StateMachine, canister_id: CanisterId| { + let args = CanisterSettingsArgsBuilder::new() + .with_freezing_threshold(1 << 62) + .build(); + let result = env.update_settings(&canister_id, args); + assert_matches!(result, Ok(_)); + }; + let unfreeze = |env: &StateMachine, canister_id: CanisterId| { + let args = CanisterSettingsArgsBuilder::new() + .with_freezing_threshold(0) + .build(); + let result = env.update_settings(&canister_id, args); + assert_matches!(result, Ok(_)); + }; + + freeze(&env, canister_id); + env.advance_time(Duration::from_secs(1000)); + env.tick(); + unfreeze(&env, canister_id); + // even though 1000s passed, the ledger did not migrate when it was frozen + assert!(!is_ledger_ready()); + wait_ledger_ready(&env, canister_id, 20); + check_approvals(); +} + pub fn test_metrics_while_migrating( ledger_wasm_mainnet: Vec, ledger_wasm_current_lowinstructionlimits: Vec, From cfa4744b9942026440dd8a12070fd530f949d08b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Mon, 18 Nov 2024 13:10:33 +0000 Subject: [PATCH 100/124] Revert "migrating with interval timer" This reverts commit 186b8928ea2751da5605bb4dbe2e13265ba2806b. --- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 1 - rs/ledger_suite/icrc1/ledger/src/lib.rs | 38 +++-------------- rs/ledger_suite/icrc1/ledger/src/main.rs | 23 +++------- rs/ledger_suite/icrc1/ledger/tests/tests.rs | 9 ---- rs/ledger_suite/tests/sm-tests/src/lib.rs | 47 +-------------------- 5 files changed, 14 insertions(+), 104 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index d56840aaa4a..b1b0dd14df9 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -36,7 +36,6 @@ package(default_visibility = ["//visibility:public"]) "@crate_index//:ciborium", "@crate_index//:hex", "@crate_index//:ic-cdk", - "@crate_index//:ic-cdk-timers", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", "@crate_index//:serde", diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 5bfc1f0b5f8..8998610d1a7 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -10,7 +10,6 @@ use candid::{ }; use ic_base_types::PrincipalId; use ic_canister_log::{log, Sink}; -use ic_cdk_timers::TimerId; use ic_crypto_tree_hash::{Label, MixedHashTree}; use ic_icrc1::blocks::encoded_block_to_generic_block; use ic_icrc1::{Block, LedgerAllowances, LedgerBalances, Transaction}; @@ -360,10 +359,7 @@ thread_local! { pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); - pub static MIGRATION_STATE: RefCell = const { RefCell::new(MigrationState { - ledger_state: LedgerState::Ready, - timer_id: None, - }) }; + pub static LEDGER_STATE: RefCell = const { RefCell::new(LedgerState::Ready) }; // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] @@ -376,13 +372,13 @@ thread_local! { MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] pub enum LedgerField { Allowances, AllowancesExpirations, } -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] pub enum LedgerState { Migrating(LedgerField), Ready, @@ -394,20 +390,6 @@ impl Default for LedgerState { } } -pub struct MigrationState { - ledger_state: LedgerState, - timer_id: Option, -} - -impl Default for MigrationState { - fn default() -> Self { - Self { - ledger_state: LedgerState::default(), - timer_id: None, - } - } -} - #[derive(Debug, Deserialize, Serialize)] #[serde(bound = "")] pub struct Ledger { @@ -976,7 +958,7 @@ impl Ledger { } pub fn is_ready() -> bool { - MIGRATION_STATE.with(|s| matches!((*s.borrow()).ledger_state, LedgerState::Ready)) + LEDGER_STATE.with(|s| matches!(*s.borrow(), LedgerState::Ready)) } pub fn panic_if_not_ready() { @@ -986,19 +968,11 @@ pub fn panic_if_not_ready() { } pub fn ledger_state() -> LedgerState { - MIGRATION_STATE.with(|s| (*s.borrow()).ledger_state) + LEDGER_STATE.with(|s| *s.borrow()) } pub fn set_ledger_state(ledger_state: LedgerState) { - MIGRATION_STATE.with(|s| (*s.borrow_mut()).ledger_state = ledger_state); -} - -pub fn migration_timer_id() -> Option { - MIGRATION_STATE.with(|s| (*s.borrow()).timer_id) -} - -pub fn set_migration_timer_id(timer_id: Option) { - MIGRATION_STATE.with(|s| (*s.borrow_mut()).timer_id = timer_id); + LEDGER_STATE.with(|s| *s.borrow_mut() = ledger_state); } #[derive(Serialize, Deserialize, Debug, Default)] diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index dc9de680a87..17fa824f184 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -16,8 +16,7 @@ use ic_icrc1::{ Operation, Transaction, }; use ic_icrc1_ledger::{ - is_ready, ledger_state, migration_timer_id, panic_if_not_ready, set_ledger_state, - set_migration_timer_id, LEDGER_VERSION, UPGRADES_MEMORY, + is_ready, ledger_state, panic_if_not_ready, set_ledger_state, LEDGER_VERSION, UPGRADES_MEMORY, }; use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; use ic_ledger_canister_core::ledger::{ @@ -244,12 +243,6 @@ fn post_upgrade(args: Option) { migrate_next_part( MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), ); - if !is_ready() { - let timer_id = ic_cdk_timers::set_timer_interval(Duration::from_secs(1), || { - migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) - }); - set_migration_timer_id(Some(timer_id)); - } let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; @@ -293,16 +286,14 @@ fn migrate_next_part(instruction_limit: u64) { let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_mingration}, total instructions used in message: {}." , instruction_counter()); if !is_ready() { - log_message(format!("Migration partially done. {msg}").as_str()); + log_message( + format!("Migration partially done. Scheduling the next part. {msg}").as_str(), + ); + ic_cdk_timers::set_timer(Duration::from_secs(0), || { + migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) + }); } else { log_message(format!("Migration completed! {msg}").as_str()); - if let Some(timer_id) = migration_timer_id() { - log_message(format!("Canceling timer with timer_id {:?}", timer_id).as_str()); - ic_cdk_timers::clear_timer(timer_id); - set_migration_timer_id(None); - } else { - log_message("Migration timer was not scheduled."); - } } }); } diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index da5f3608828..a72f6b87369 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -488,15 +488,6 @@ fn icrc1_test_metrics_while_migrating() { ); } -#[test] -fn icrc1_test_migration_timer_canceled() { - ic_ledger_suite_state_machine_tests::test_migration_timer_canceled( - ledger_mainnet_wasm(), - ledger_wasm_lowupgradeinstructionlimits(), - encode_init_args, - ); -} - mod metrics { use crate::{encode_init_args, encode_upgrade_args, ledger_wasm}; use ic_ledger_suite_state_machine_tests::metrics::LedgerSuiteType; diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 22e2df829e6..f433859a255 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2723,7 +2723,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), true); test_endpoint("icrc1_total_supply", Encode!().unwrap(), true); - wait_ledger_ready(&env, canister_id, 20); + wait_ledger_ready(&env, canister_id, 10); test_endpoint("icrc1_transfer", Encode!(&transfer_args).unwrap(), false); test_endpoint("icrc2_approve", Encode!(&approve_args).unwrap(), false); @@ -2979,51 +2979,6 @@ pub fn test_metrics_while_migrating( ); } -pub fn test_migration_timer_canceled( - ledger_wasm_mainnet: Vec, - ledger_wasm_current_lowinstructionlimits: Vec, - encode_init_args: fn(InitArgs) -> T, -) where - T: CandidType, -{ - let account = Account::from(PrincipalId::new_user_test_id(1).0); - let initial_balances = vec![(account, 100_000_000u64)]; - - // Setup ledger as it is deployed on the mainnet. - let (env, canister_id) = setup(ledger_wasm_mainnet, encode_init_args, initial_balances); - - const APPROVE_AMOUNT: u64 = 150_000; - - for i in 2..40 { - let spender = Account::from(PrincipalId::new_user_test_id(i).0); - let approve_args = default_approve_args(spender, APPROVE_AMOUNT); - send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); - } - - env.upgrade_canister( - canister_id, - ledger_wasm_current_lowinstructionlimits, - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) - .unwrap(); - - wait_ledger_ready(&env, canister_id, 20); - - let stable_upgrade_migration_steps = - parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); - assert!(stable_upgrade_migration_steps > 1); - - env.advance_time(Duration::from_secs(1000)); - env.tick(); - - let later_stable_upgrade_migration_steps = - parse_metric(&env, canister_id, "ledger_stable_upgrade_migration_steps"); - assert_eq!( - stable_upgrade_migration_steps, - later_stable_upgrade_migration_steps - ); -} - pub fn default_approve_args(spender: impl Into, amount: u64) -> ApproveArgs { ApproveArgs { from_subaccount: None, From dafc6c7d481b48f31d7c5dbfb56db7ed9a1f0e79 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 12:49:32 +0000 Subject: [PATCH 101/124] make sure the timer was attempted to be scheduled --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index f433859a255..43c3734604b 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2901,7 +2901,10 @@ pub fn test_migration_resumes_from_frozen( freeze(&env, canister_id); env.advance_time(Duration::from_secs(1000)); - env.tick(); + // Make sure the timer was attempted to be scheduled. + for _ in 0..10 { + env.tick(); + } unfreeze(&env, canister_id); // even though 1000s passed, the ledger did not migrate when it was frozen assert!(!is_ledger_ready()); From 40b015c8849c422ef0ad618cf2465fed9798bdeb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 14:34:39 +0000 Subject: [PATCH 102/124] serializa Account with minicbor --- Cargo.lock | 3 + packages/icrc-ledger-types/BUILD.bazel | 1 + packages/icrc-ledger-types/Cargo.toml | 1 + packages/icrc-ledger-types/src/cbor/mod.rs | 1 + .../icrc-ledger-types/src/cbor/principal.rs | 17 +++ .../icrc-ledger-types/src/icrc1/account.rs | 74 ++-------- packages/icrc-ledger-types/src/lib.rs | 1 + .../common/ledger_core/BUILD.bazel | 1 + rs/ledger_suite/common/ledger_core/Cargo.toml | 1 + .../common/ledger_core/src/timestamp.rs | 16 ++- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 1 + rs/ledger_suite/icrc1/ledger/Cargo.toml | 1 + rs/ledger_suite/icrc1/ledger/src/lib.rs | 126 ++++++++++++++++-- 13 files changed, 170 insertions(+), 74 deletions(-) create mode 100644 packages/icrc-ledger-types/src/cbor/mod.rs create mode 100644 packages/icrc-ledger-types/src/cbor/principal.rs diff --git a/Cargo.lock b/Cargo.lock index 4dd4c6ad6c2..dbe54fc5908 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9048,6 +9048,7 @@ dependencies = [ "icrc-ledger-client", "icrc-ledger-types", "leb128", + "minicbor", "num-bigint 0.4.6", "num-traits", "proptest", @@ -9332,6 +9333,7 @@ dependencies = [ "candid", "ic-ledger-hash-of", "ic-stable-structures", + "minicbor", "num-traits", "proptest", "serde", @@ -13944,6 +13946,7 @@ dependencies = [ "hex", "ic-stable-structures", "itertools 0.12.1", + "minicbor", "num-bigint 0.4.6", "num-traits", "proptest", diff --git a/packages/icrc-ledger-types/BUILD.bazel b/packages/icrc-ledger-types/BUILD.bazel index 37acab199b5..038458b6dc5 100644 --- a/packages/icrc-ledger-types/BUILD.bazel +++ b/packages/icrc-ledger-types/BUILD.bazel @@ -24,6 +24,7 @@ rust_library( "@crate_index//:ic-cdk", "@crate_index//:ic-stable-structures", "@crate_index//:itertools", + "@crate_index//:minicbor", "@crate_index//:num-bigint", "@crate_index//:num-traits", "@crate_index//:serde", diff --git a/packages/icrc-ledger-types/Cargo.toml b/packages/icrc-ledger-types/Cargo.toml index 91b96ea0e65..2397d9d8c6d 100644 --- a/packages/icrc-ledger-types/Cargo.toml +++ b/packages/icrc-ledger-types/Cargo.toml @@ -17,6 +17,7 @@ crc32fast = "1.2.0" hex = { workspace = true } ic-stable-structures = { workspace = true } itertools = { workspace = true } +minicbor = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } diff --git a/packages/icrc-ledger-types/src/cbor/mod.rs b/packages/icrc-ledger-types/src/cbor/mod.rs new file mode 100644 index 00000000000..877a754fff3 --- /dev/null +++ b/packages/icrc-ledger-types/src/cbor/mod.rs @@ -0,0 +1 @@ +pub mod principal; \ No newline at end of file diff --git a/packages/icrc-ledger-types/src/cbor/principal.rs b/packages/icrc-ledger-types/src/cbor/principal.rs new file mode 100644 index 00000000000..027ae45f7dd --- /dev/null +++ b/packages/icrc-ledger-types/src/cbor/principal.rs @@ -0,0 +1,17 @@ +use candid::Principal; +use minicbor::decode::{Decoder, Error}; +use minicbor::encode::{Encoder, Write}; + +pub fn decode(d: &mut Decoder<'_>, _ctx: &mut Ctx) -> Result { + let bytes = d.bytes()?; + Principal::try_from_slice(bytes).map_err(|e| Error::message(e.to_string())) +} + +pub fn encode( + v: &Principal, + e: &mut Encoder, + _ctx: &mut Ctx, +) -> Result<(), minicbor::encode::Error> { + e.bytes(v.as_slice())?; + Ok(()) +} diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 017c8ff04bd..f7638321f32 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -7,6 +7,7 @@ use std::{ use base32::Alphabet; use candid::{types::principal::PrincipalError, CandidType, Deserialize, Principal}; use ic_stable_structures::{storable::Bound, Storable}; +use minicbor::{Decode, Encode}; use serde::Serialize; use std::borrow::Cow; @@ -15,9 +16,11 @@ pub type Subaccount = [u8; 32]; pub const DEFAULT_SUBACCOUNT: &Subaccount = &[0; 32]; // Account representation of ledgers supporting the ICRC1 standard -#[derive(Serialize, CandidType, Deserialize, Clone, Debug, Copy)] +#[derive(Serialize, CandidType, Deserialize, Clone, Debug, Copy, Encode, Decode)] pub struct Account { + #[cbor(n(0), with = "crate::cbor::principal")] pub owner: Principal, + #[cbor(n(1), with = "minicbor::bytes")] pub subaccount: Option, } @@ -169,52 +172,19 @@ impl FromStr for Account { } } -const MAX_SERIALIZATION_LEN: u32 = 63; - impl Storable for Account { fn to_bytes(&self) -> Cow<[u8]> { - let mut buffer: Vec = vec![0]; - - // Owner principal - buffer.extend(self.owner.as_slice()); - buffer[0] = (buffer.len() - 1) as u8; - - // Subaccount - if let Some(subaccount) = self.subaccount { - buffer.extend([32u8]); - buffer.extend(subaccount.as_slice()); - } else { - buffer.extend([0u8]); - } - - Cow::Owned(buffer) + let mut buf = vec![]; + minicbor::encode(self, &mut buf).expect("event encoding should always succeed"); + Cow::Owned(buf) } fn from_bytes(bytes: Cow<[u8]>) -> Self { - let mut index = 0usize; - let len_owner = bytes[index] as usize; - index += 1; - let owner = Principal::from_slice(&bytes[index..index + len_owner]); - index += len_owner; - - // Subaccount - let len_subaccount = bytes[index] as usize; - index += 1; - let subaccount = if len_subaccount > 0 { - let subaccount_bytes: [u8; 32] = - bytes[index..index + len_subaccount].try_into().unwrap(); - Some(subaccount_bytes) - } else { - None - }; - - Account { owner, subaccount } + minicbor::decode(bytes.as_ref()) + .unwrap_or_else(|e| panic!("failed to decode event bytes {}: {e}", hex::encode(bytes))) } - const BOUND: Bound = Bound::Bounded { - max_size: MAX_SERIALIZATION_LEN, - is_fixed_size: false, - }; + const BOUND: Bound = Bound::Unbounded; } #[cfg(test)] @@ -227,9 +197,7 @@ mod tests { use candid::Principal; - use crate::icrc1::account::{ - Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT, MAX_SERIALIZATION_LEN, - }; + use crate::icrc1::account::{Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT}; pub fn principal_strategy() -> impl Strategy { let bytes_strategy = prop::collection::vec(0..=255u8, 29); @@ -378,26 +346,6 @@ mod tests { ); } - #[test] - fn test_account_max_serialization_length() { - let owner = - Principal::from_text("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") - .unwrap(); - let subaccount = Some( - hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") - .unwrap() - .try_into() - .unwrap(), - ); - let account = Account { owner, subaccount }; - let serialized_len = account.to_bytes().len(); - assert_eq!( - serialized_len, - 1 + DEFAULT_SUBACCOUNT.len() + 1 + Principal::MAX_LENGTH_IN_BYTES - ); - assert_eq!(serialized_len as u32, MAX_SERIALIZATION_LEN); - } - #[test] fn test_account_serialization() { use proptest::{prop_assert_eq, proptest}; diff --git a/packages/icrc-ledger-types/src/lib.rs b/packages/icrc-ledger-types/src/lib.rs index f532acb4880..4e5cffbf9c0 100644 --- a/packages/icrc-ledger-types/src/lib.rs +++ b/packages/icrc-ledger-types/src/lib.rs @@ -1,3 +1,4 @@ +pub mod cbor; pub mod icrc; pub mod icrc1; pub mod icrc2; diff --git a/rs/ledger_suite/common/ledger_core/BUILD.bazel b/rs/ledger_suite/common/ledger_core/BUILD.bazel index ba8f054c248..edd5eae9ba9 100644 --- a/rs/ledger_suite/common/ledger_core/BUILD.bazel +++ b/rs/ledger_suite/common/ledger_core/BUILD.bazel @@ -12,6 +12,7 @@ rust_library( "//packages/ic-ledger-hash-of:ic_ledger_hash_of", "@crate_index//:candid", "@crate_index//:ic-stable-structures", + "@crate_index//:minicbor", "@crate_index//:num-traits", "@crate_index//:serde", "@crate_index//:serde_bytes", diff --git a/rs/ledger_suite/common/ledger_core/Cargo.toml b/rs/ledger_suite/common/ledger_core/Cargo.toml index 469dc7b61ed..0e46d70acdc 100644 --- a/rs/ledger_suite/common/ledger_core/Cargo.toml +++ b/rs/ledger_suite/common/ledger_core/Cargo.toml @@ -10,6 +10,7 @@ documentation.workspace = true candid = { workspace = true } ic-ledger-hash-of = { path = "../../../../packages/ic-ledger-hash-of" } ic-stable-structures = { workspace = true } +minicbor = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } diff --git a/rs/ledger_suite/common/ledger_core/src/timestamp.rs b/rs/ledger_suite/common/ledger_core/src/timestamp.rs index 17f05835e88..e59ca9e097a 100644 --- a/rs/ledger_suite/common/ledger_core/src/timestamp.rs +++ b/rs/ledger_suite/common/ledger_core/src/timestamp.rs @@ -1,5 +1,6 @@ use candid::CandidType; use ic_stable_structures::{storable::Bound, Storable}; +use minicbor::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::convert::TryInto; @@ -10,9 +11,22 @@ use std::time::{Duration, SystemTime}; mod tests; #[derive( - Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, CandidType, Deserialize, Serialize, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Debug, + CandidType, + Deserialize, + Serialize, + Encode, + Decode, )] pub struct TimeStamp { + #[n(0)] timestamp_nanos: u64, } diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index b1b0dd14df9..ef96d8106cf 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -38,6 +38,7 @@ package(default_visibility = ["//visibility:public"]) "@crate_index//:ic-cdk", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", + "@crate_index//:minicbor", "@crate_index//:serde", "@crate_index//:serde_bytes", ] + extra_deps, diff --git a/rs/ledger_suite/icrc1/ledger/Cargo.toml b/rs/ledger_suite/icrc1/ledger/Cargo.toml index 7c5007accf8..e103bb02982 100644 --- a/rs/ledger_suite/icrc1/ledger/Cargo.toml +++ b/rs/ledger_suite/icrc1/ledger/Cargo.toml @@ -34,6 +34,7 @@ ic-metrics-encoder = "1.1.1" ic-stable-structures = { workspace = true } icrc-ledger-client = { path = "../../../../packages/icrc-ledger-client" } icrc-ledger-types = { path = "../../../../packages/icrc-ledger-types" } +minicbor = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 8998610d1a7..680d0511726 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -30,6 +30,7 @@ use ic_ledger_core::{ }; use ic_ledger_hash_of::HashOf; use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory}; +use ic_stable_structures::{storable::Bound, Storable}; use ic_stable_structures::{DefaultMemoryImpl, StableBTreeMap}; use icrc_ledger_types::icrc3::transactions::Transaction as Tx; use icrc_ledger_types::icrc3::{blocks::GetBlocksResponse, transactions::GetTransactionsResponse}; @@ -45,6 +46,7 @@ use icrc_ledger_types::{ blocks::{ArchivedBlocks, GetBlocksRequest, GetBlocksResult}, }, }; +use minicbor::{Decode, Encode}; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use std::borrow::Cow; @@ -339,6 +341,95 @@ pub struct UpgradeArgs { pub change_archive_options: Option, } +#[derive(Clone, Eq, PartialEq, Debug, Encode, Decode)] +struct AccountSpender { + #[n(0)] + account: Account, + #[n(1)] + spender: Account, +} + +impl std::cmp::PartialOrd for AccountSpender { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl std::cmp::Ord for AccountSpender { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.account + .cmp(&other.account) + .then_with(|| self.spender.cmp(&other.spender)) + } +} + +impl Storable for AccountSpender { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buf = vec![]; + minicbor::encode(self, &mut buf).expect("AccountSpender encoding should always succeed"); + Cow::Owned(buf) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + minicbor::decode(bytes.as_ref()) + .unwrap_or_else(|e| panic!("failed to decode AccountSpender bytes {}: {e}", hex::encode(bytes))) + } + + const BOUND: Bound = Bound::Unbounded; +} + +impl From<&(Account, Account)> for AccountSpender { + fn from(pair: &(Account, Account)) -> Self { + Self { + account: pair.0, + spender: pair.1, + } + } +} + +impl Into<(Account, Account)> for AccountSpender { + fn into(self) -> (Account, Account) { + (self.account, self.spender) + } +} + +#[derive(Clone, Eq, PartialEq, Debug, Encode, Decode)] +struct Expiration { + #[n(0)] + timestamp: TimeStamp, + #[n(1)] + account_spender: AccountSpender, +} + +impl std::cmp::PartialOrd for Expiration { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl std::cmp::Ord for Expiration { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.timestamp + .cmp(&other.timestamp) + .then_with(|| self.account_spender.cmp(&other.account_spender)) + } +} + +impl Storable for Expiration { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buf = vec![]; + minicbor::encode(self, &mut buf).expect("Expiration encoding should always succeed"); + Cow::Owned(buf) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + minicbor::decode(bytes.as_ref()) + .unwrap_or_else(|e| panic!("failed to decode Expiration bytes {}: {e}", hex::encode(bytes))) + } + + const BOUND: Bound = Bound::Unbounded; +} + #[derive(Clone, Eq, PartialEq, Debug, CandidType, Deserialize)] #[allow(clippy::large_enum_variant)] pub enum LedgerArgument { @@ -363,12 +454,12 @@ thread_local! { // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] - pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = + pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_MEMORY_ID)))); // (timestamp, (from, spender)) - expiration set used for removing expired allowances. #[allow(clippy::type_complexity)] - pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = + pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); } @@ -986,7 +1077,8 @@ impl AllowancesData for StableAllowancesData { &self, account_spender: &(Self::AccountId, Self::AccountId), ) -> Option> { - ALLOWANCES_MEMORY.with_borrow(|allowances| allowances.get(account_spender)) + let account_spender = account_spender.into(); + ALLOWANCES_MEMORY.with_borrow(|allowances| allowances.get(&account_spender)) } fn set_allowance( @@ -994,12 +1086,14 @@ impl AllowancesData for StableAllowancesData { account_spender: (Self::AccountId, Self::AccountId), allowance: Allowance, ) { + let account_spender = (&account_spender).into(); ALLOWANCES_MEMORY .with_borrow_mut(|allowances| allowances.insert(account_spender, allowance)); } fn remove_allowance(&mut self, account_spender: &(Self::AccountId, Self::AccountId)) { - ALLOWANCES_MEMORY.with_borrow_mut(|allowances| allowances.remove(account_spender)); + let account_spender = account_spender.into(); + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| allowances.remove(&account_spender)); } fn insert_expiry( @@ -1007,8 +1101,13 @@ impl AllowancesData for StableAllowancesData { timestamp: TimeStamp, account_spender: (Self::AccountId, Self::AccountId), ) { + let account_spender = (&account_spender).into(); + let expiration = Expiration { + timestamp, + account_spender, + }; ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { - expirations.insert((timestamp, account_spender), ()); + expirations.insert(expiration, ()); }); } @@ -1017,8 +1116,13 @@ impl AllowancesData for StableAllowancesData { timestamp: TimeStamp, account_spender: (Self::AccountId, Self::AccountId), ) { + let account_spender = (&account_spender).into(); + let expiration = Expiration { + timestamp, + account_spender, + }; ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { - expirations.remove(&(timestamp, account_spender)); + expirations.remove(&expiration); }); } @@ -1039,8 +1143,9 @@ impl AllowancesData for StableAllowancesData { } fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { - ALLOWANCES_EXPIRATIONS_MEMORY - .with_borrow(|expirations| expirations.first_key_value().map(|kv| kv.0)) + let result = ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow(|expirations| expirations.first_key_value().map(|kv| kv.0)); + result.map(|e| (e.timestamp, e.account_spender.into())) } fn oldest_arrivals(&self, _n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { @@ -1049,8 +1154,9 @@ impl AllowancesData for StableAllowancesData { } fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { - ALLOWANCES_EXPIRATIONS_MEMORY - .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)) + let result = ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)); + result.map(|e| (e.timestamp, e.account_spender.into())) } fn pop_first_allowance( From 1e349d4b9f872e04b645dd38984850159e46731b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 14:41:48 +0000 Subject: [PATCH 103/124] fix error message --- packages/icrc-ledger-types/src/icrc1/account.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index f7638321f32..39a07aaf519 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -175,13 +175,14 @@ impl FromStr for Account { impl Storable for Account { fn to_bytes(&self) -> Cow<[u8]> { let mut buf = vec![]; - minicbor::encode(self, &mut buf).expect("event encoding should always succeed"); + minicbor::encode(self, &mut buf).expect("account encoding should always succeed"); Cow::Owned(buf) } fn from_bytes(bytes: Cow<[u8]>) -> Self { - minicbor::decode(bytes.as_ref()) - .unwrap_or_else(|e| panic!("failed to decode event bytes {}: {e}", hex::encode(bytes))) + minicbor::decode(bytes.as_ref()).unwrap_or_else(|e| { + panic!("failed to decode account bytes {}: {e}", hex::encode(bytes)) + }) } const BOUND: Bound = Bound::Unbounded; From 50119ed50223ac14e37f4aed9ce8f631996687fd Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 14:56:55 +0000 Subject: [PATCH 104/124] clippy --- packages/icrc-ledger-types/src/cbor/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/icrc-ledger-types/src/cbor/mod.rs b/packages/icrc-ledger-types/src/cbor/mod.rs index 877a754fff3..f9fe4658e5b 100644 --- a/packages/icrc-ledger-types/src/cbor/mod.rs +++ b/packages/icrc-ledger-types/src/cbor/mod.rs @@ -1 +1 @@ -pub mod principal; \ No newline at end of file +pub mod principal; From b661488b792253091f63340948bca20476467aca Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 16:05:54 +0000 Subject: [PATCH 105/124] clippy --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 680d0511726..b901cdbaacc 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -371,8 +371,12 @@ impl Storable for AccountSpender { } fn from_bytes(bytes: Cow<[u8]>) -> Self { - minicbor::decode(bytes.as_ref()) - .unwrap_or_else(|e| panic!("failed to decode AccountSpender bytes {}: {e}", hex::encode(bytes))) + minicbor::decode(bytes.as_ref()).unwrap_or_else(|e| { + panic!( + "failed to decode AccountSpender bytes {}: {e}", + hex::encode(bytes) + ) + }) } const BOUND: Bound = Bound::Unbounded; @@ -423,8 +427,12 @@ impl Storable for Expiration { } fn from_bytes(bytes: Cow<[u8]>) -> Self { - minicbor::decode(bytes.as_ref()) - .unwrap_or_else(|e| panic!("failed to decode Expiration bytes {}: {e}", hex::encode(bytes))) + minicbor::decode(bytes.as_ref()).unwrap_or_else(|e| { + panic!( + "failed to decode Expiration bytes {}: {e}", + hex::encode(bytes) + ) + }) } const BOUND: Bound = Bound::Unbounded; From ead1399ce8aafe3f826d0ecf9be47230413631a2 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 16:08:12 +0000 Subject: [PATCH 106/124] clippy --- packages/icrc-ledger-types/src/icrc1/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index 39a07aaf519..ced24f33bb9 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -198,7 +198,7 @@ mod tests { use candid::Principal; - use crate::icrc1::account::{Account, ICRC1TextReprError, DEFAULT_SUBACCOUNT}; + use crate::icrc1::account::{Account, ICRC1TextReprError}; pub fn principal_strategy() -> impl Strategy { let bytes_strategy = prop::collection::vec(0..=255u8, 29); From 16a64ee470fb48888c7cace98e097c5c1d10f9fb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Tue, 19 Nov 2024 16:19:50 +0000 Subject: [PATCH 107/124] clippy --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index b901cdbaacc..0bf797b60f0 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -391,9 +391,9 @@ impl From<&(Account, Account)> for AccountSpender { } } -impl Into<(Account, Account)> for AccountSpender { - fn into(self) -> (Account, Account) { - (self.account, self.spender) +impl From for (Account, Account) { + fn from(val: AccountSpender) -> Self { + (val.account, val.spender) } } From 4c97ac505e14565e6a46e3eae1a4bcac691fe289 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 09:00:12 +0000 Subject: [PATCH 108/124] build fix --- rs/ledger_suite/tests/sm-tests/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 498b96b4442..02f8968207a 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2669,7 +2669,7 @@ pub fn test_downgrade_from_incompatible_version( // Upgrade to the next version. env.upgrade_canister( canister_id, - ledger_wasm_nextledgerversion, + ledger_wasm_nextledgerversion.clone(), Encode!(&LedgerArgument::Upgrade(None)).unwrap(), ) .expect("failed to upgrade to next version"); From d538875c8f696b5f16f9b973795d8ff145f25467 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 12:34:14 +0000 Subject: [PATCH 109/124] use minicbor for allowance serialization --- Cargo.lock | 2 + .../common/ledger_core/src/approvals.rs | 53 ------------------ .../common/ledger_core/src/approvals/tests.rs | 34 ------------ rs/ledger_suite/icrc1/ledger/src/lib.rs | 54 +++++++++++++++++- rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel | 6 +- rs/ledger_suite/icrc1/tokens_u256/Cargo.toml | 1 + .../icrc1/tokens_u256/src/encode_decode.rs | 55 +++++++++++++++++++ rs/ledger_suite/icrc1/tokens_u256/src/lib.rs | 7 ++- rs/ledger_suite/icrc1/tokens_u64/BUILD.bazel | 1 + rs/ledger_suite/icrc1/tokens_u64/Cargo.toml | 1 + rs/ledger_suite/icrc1/tokens_u64/src/lib.rs | 7 ++- 11 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs diff --git a/Cargo.lock b/Cargo.lock index 999de219eec..915c1e6cf06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9105,6 +9105,7 @@ dependencies = [ "hex", "ic-ledger-core", "ic-stable-structures", + "minicbor", "num-bigint 0.4.6", "num-traits", "proptest", @@ -9119,6 +9120,7 @@ dependencies = [ "ciborium", "ic-ledger-core", "ic-stable-structures", + "minicbor", "num-traits", "proptest", "serde", diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index dae3ec969c5..617d3335abb 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -1,13 +1,7 @@ use crate::timestamp::TimeStamp; use crate::tokens::{CheckedSub, TokensType, Zero}; -use candid::Nat; -use ic_stable_structures::{storable::Bound, Storable}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use std::{ - borrow::Cow, - io::{Cursor, Read}, -}; #[cfg(test)] mod tests; @@ -492,50 +486,3 @@ fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } -impl + TryFrom> Storable for Allowance { - fn to_bytes(&self) -> Cow<[u8]> { - let mut buffer = vec![]; - let amount: Nat = self.amount.clone().into(); - amount - .encode(&mut buffer) - .expect("Unable to serialize amount"); - if let Some(expires_at) = self.expires_at { - buffer.extend([8u8]); - buffer.extend(expires_at.as_nanos_since_unix_epoch().to_le_bytes()); - } else { - buffer.extend([0u8]); - } - // We don't serialize arrived_at - it is not used after stable structures migration. - Cow::Owned(buffer) - } - - fn from_bytes(bytes: Cow<[u8]>) -> Self { - let mut cursor = Cursor::new(bytes.into_owned()); - let amount = Nat::decode(&mut cursor).expect("Unable to deserialize amount"); - let amount = Tokens::try_from(amount).expect("Unable to convert Nat to Tokens"); - // arrived_at was not serialized, use a default value. - let arrived_at = TimeStamp::from_nanos_since_unix_epoch(0); - let mut expires_at_length_bytes = [0u8; 1]; - cursor - .read_exact(&mut expires_at_length_bytes) - .expect("could not read expires_at_length_bytes"); - let expires_at = if expires_at_length_bytes[0] > 0 { - let mut expires_at_bytes = [0u8; 8]; - cursor - .read_exact(&mut expires_at_bytes) - .expect("could not read expires_at"); - Some(TimeStamp::from_nanos_since_unix_epoch(u64::from_le_bytes( - expires_at_bytes, - ))) - } else { - None - }; - Self { - amount, - arrived_at, - expires_at, - } - } - - const BOUND: Bound = Bound::Unbounded; -} diff --git a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs index c112741de90..196fab9adba 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs @@ -620,37 +620,3 @@ fn expected_allowance_if_zero_no_approval() { } ); } - -use proptest::prelude::{any, prop_assert_eq, proptest}; -use proptest::strategy::Strategy; - -#[test] -fn allowance_serialization() { - fn arb_token() -> impl Strategy { - any::().prop_map(Tokens::from_e8s) - } - fn arb_timestamp() -> impl Strategy { - any::().prop_map(TimeStamp::from_nanos_since_unix_epoch) - } - fn arb_opt_expiration() -> impl Strategy> { - proptest::option::of(any::().prop_map(TimeStamp::from_nanos_since_unix_epoch)) - } - fn arb_allowance() -> impl Strategy> { - (arb_token(), arb_opt_expiration(), arb_timestamp()).prop_map( - |(amount, expires_at, arrived_at)| Allowance { - amount, - expires_at, - arrived_at, - }, - ) - } - proptest!(|(allowance in arb_allowance())| { - let new_allowance: Allowance = Allowance::from_bytes(allowance.to_bytes()); - prop_assert_eq!(new_allowance.amount, allowance.amount); - prop_assert_eq!(new_allowance.expires_at, allowance.expires_at); - prop_assert_eq!( - new_allowance.arrived_at, - TimeStamp::from_nanos_since_unix_epoch(0) - ); - }) -} diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 0bf797b60f0..f74ab794edc 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -438,6 +438,52 @@ impl Storable for Expiration { const BOUND: Bound = Bound::Unbounded; } +#[derive(Clone, Debug, Encode, Decode)] +struct StorableAllowance { + #[n(0)] + amount: Tokens, + #[n(1)] + expires_at: Option, +} + +impl Storable for StorableAllowance { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buf = vec![]; + minicbor::encode(self, &mut buf).expect("StorableAllowance encoding should always succeed"); + Cow::Owned(buf) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + minicbor::decode(bytes.as_ref()).unwrap_or_else(|e| { + panic!( + "failed to decode StorableAllowance bytes {}: {e}", + hex::encode(bytes) + ) + }) + } + + const BOUND: Bound = Bound::Unbounded; +} + +impl From> for StorableAllowance { + fn from(val: Allowance) -> Self { + Self { + amount: val.amount, + expires_at: val.expires_at, + } + } +} + +impl From for Allowance { + fn from(val: StorableAllowance) -> Self { + Self { + amount: val.amount, + expires_at: val.expires_at, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), + } + } +} + #[derive(Clone, Eq, PartialEq, Debug, CandidType, Deserialize)] #[allow(clippy::large_enum_variant)] pub enum LedgerArgument { @@ -462,7 +508,7 @@ thread_local! { // (from, spender) -> allowance - map storing ledger allowances. #[allow(clippy::type_complexity)] - pub static ALLOWANCES_MEMORY: RefCell, VirtualMemory>> = + pub static ALLOWANCES_MEMORY: RefCell>> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_MEMORY_ID)))); // (timestamp, (from, spender)) - expiration set used for removing expired allowances. @@ -1086,7 +1132,9 @@ impl AllowancesData for StableAllowancesData { account_spender: &(Self::AccountId, Self::AccountId), ) -> Option> { let account_spender = account_spender.into(); - ALLOWANCES_MEMORY.with_borrow(|allowances| allowances.get(&account_spender)) + ALLOWANCES_MEMORY + .with_borrow(|allowances| allowances.get(&account_spender)) + .map(|a| a.into()) } fn set_allowance( @@ -1096,7 +1144,7 @@ impl AllowancesData for StableAllowancesData { ) { let account_spender = (&account_spender).into(); ALLOWANCES_MEMORY - .with_borrow_mut(|allowances| allowances.insert(account_spender, allowance)); + .with_borrow_mut(|allowances| allowances.insert(account_spender, allowance.into())); } fn remove_allowance(&mut self, account_spender: &(Self::AccountId, Self::AccountId)) { diff --git a/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel b/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel index 6cd325cbc4d..e389bb18fd6 100644 --- a/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel +++ b/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel @@ -7,13 +7,17 @@ COMMON_DEPS = [ "@crate_index//:candid", "@crate_index//:ciborium", "@crate_index//:ic-stable-structures", + "@crate_index//:minicbor", "@crate_index//:num-traits", "@crate_index//:serde", ] rust_library( name = "tokens_u256", - srcs = ["src/lib.rs"], + srcs = [ + "src/encode_decode.rs", + "src/lib.rs", + ], crate_name = "ic_icrc1_tokens_u256", version = "0.1.0", deps = COMMON_DEPS + [ diff --git a/rs/ledger_suite/icrc1/tokens_u256/Cargo.toml b/rs/ledger_suite/icrc1/tokens_u256/Cargo.toml index 414ae0cf58e..c93d25cc31d 100644 --- a/rs/ledger_suite/icrc1/tokens_u256/Cargo.toml +++ b/rs/ledger_suite/icrc1/tokens_u256/Cargo.toml @@ -10,6 +10,7 @@ ciborium = { workspace = true } ethnum = { workspace = true } ic-ledger-core = { path = "../../common/ledger_core" } ic-stable-structures = { workspace = true } +minicbor = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } diff --git a/rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs b/rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs new file mode 100644 index 00000000000..5df93d1f6ce --- /dev/null +++ b/rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs @@ -0,0 +1,55 @@ +use ethnum::u256; +use minicbor::data::Tag; +use minicbor::decode::{Decoder, Error}; +use minicbor::encode::{Encoder, Write}; + +const U32_MAX: u256 = u256::new(u32::MAX as u128); +const U64_MAX: u256 = u256::new(u64::MAX as u128); + +pub fn decode(d: &mut Decoder<'_>, _ctx: &mut Ctx) -> Result { + let pos = d.position(); + match d.u64() { + Ok(n) => return Ok(u256::from(n)), + Err(e) if e.is_type_mismatch() => { + d.set_position(pos); + } + Err(e) => return Err(e), + } + + let tag: Tag = d.tag()?; + if tag != Tag::PosBignum { + return Err(Error::message( + "failed to parse u256: expected a PosBignum tag", + )); + } + let bytes = d.bytes()?; + if bytes.len() > 32 { + return Err(Error::message(format!( + "failed to parse u256: expected at most 32 bytes, got: {}", + bytes.len() + ))); + } + let mut be_bytes = [0u8; 32]; + be_bytes[32 - bytes.len()..32].copy_from_slice(bytes); + Ok(u256::from_be_bytes(be_bytes)) +} + +pub fn encode( + v: &u256, + e: &mut Encoder, + _ctx: &mut Ctx, +) -> Result<(), minicbor::encode::Error> { + if v <= &U32_MAX { + e.u32(v.as_u32())?; + } else if v <= &U64_MAX { + e.u64(v.as_u64())?; + } else { + let be_bytes = v.to_be_bytes(); + let non_zero_pos = be_bytes + .iter() + .position(|x| *x != 0) + .unwrap_or(be_bytes.len()); + e.tag(Tag::PosBignum)?.bytes(&be_bytes[non_zero_pos..])?; + } + Ok(()) +} diff --git a/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs b/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs index 1a589f00e47..d4541da47fd 100644 --- a/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs +++ b/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs @@ -3,6 +3,7 @@ use ciborium::tag::Required; use ethnum::u256; use ic_ledger_core::tokens::{CheckedAdd, CheckedSub, Zero}; use ic_stable_structures::storable::{Bound, Storable}; +use minicbor::{Decode, Encode}; use num_traits::Bounded; use serde::{ de::{self, Deserializer}, @@ -12,6 +13,8 @@ use serde::{ use std::borrow::Cow; use std::fmt; +pub mod encode_decode; + /// The tag number for big positive integers. // See https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums const BIGNUM_CBOR_TAG: u64 = 2; @@ -25,8 +28,8 @@ type TaggedRepr = Required; struct U256Repr(#[serde(with = "ethnum::serde::compressed_bytes::be")] u256); /// 256-bit token amounts. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default)] -pub struct U256(u256); +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default, Decode, Encode)] +pub struct U256(#[cbor(n(0), with = "encode_decode")] u256); impl U256 { pub const ZERO: Self = Self(u256::ZERO); diff --git a/rs/ledger_suite/icrc1/tokens_u64/BUILD.bazel b/rs/ledger_suite/icrc1/tokens_u64/BUILD.bazel index 59b8a8cf81f..36c4290f520 100644 --- a/rs/ledger_suite/icrc1/tokens_u64/BUILD.bazel +++ b/rs/ledger_suite/icrc1/tokens_u64/BUILD.bazel @@ -12,6 +12,7 @@ rust_library( "//rs/ledger_suite/common/ledger_core", "@crate_index//:candid", "@crate_index//:ic-stable-structures", + "@crate_index//:minicbor", "@crate_index//:num-traits", "@crate_index//:serde", ], diff --git a/rs/ledger_suite/icrc1/tokens_u64/Cargo.toml b/rs/ledger_suite/icrc1/tokens_u64/Cargo.toml index 405eb0636bc..7b3735ca112 100644 --- a/rs/ledger_suite/icrc1/tokens_u64/Cargo.toml +++ b/rs/ledger_suite/icrc1/tokens_u64/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" candid = { workspace = true } ic-ledger-core = { path = "../../common/ledger_core" } ic-stable-structures = { workspace = true } +minicbor = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } diff --git a/rs/ledger_suite/icrc1/tokens_u64/src/lib.rs b/rs/ledger_suite/icrc1/tokens_u64/src/lib.rs index 46d80b4f0b9..099d99adb31 100644 --- a/rs/ledger_suite/icrc1/tokens_u64/src/lib.rs +++ b/rs/ledger_suite/icrc1/tokens_u64/src/lib.rs @@ -1,15 +1,18 @@ use candid::Nat; use ic_ledger_core::tokens::{CheckedAdd, CheckedSub, Zero}; use ic_stable_structures::storable::{Bound, Storable}; +use minicbor::{Decode, Encode}; use num_traits::{Bounded, ToPrimitive}; use serde::{de::Deserializer, Deserialize, Serialize}; use std::borrow::Cow; use std::fmt; use std::str::FromStr; -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default, Serialize)] +#[derive( + Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default, Serialize, Encode, Decode, +)] #[serde(transparent)] -pub struct U64(u64); +pub struct U64(#[n(0)] u64); impl U64 { pub const ZERO: Self = Self(0); From a4eeb3e5035649d7be837e7dd0671e2b86d7f57b Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 12:45:53 +0000 Subject: [PATCH 110/124] clippy --- rs/ledger_suite/common/ledger_core/src/approvals.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 617d3335abb..2c034208d66 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -485,4 +485,3 @@ where fn remote_future() -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(u64::MAX) } - From f1af825877932c8ea9c8fe32f20582572ac67efb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 13:02:10 +0000 Subject: [PATCH 111/124] clippy --- rs/ledger_suite/common/ledger_core/src/approvals/tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs index 196fab9adba..c6ea7acb0b4 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs @@ -3,7 +3,6 @@ use std::collections::HashSet; use super::*; use crate::timestamp::TimeStamp; use crate::tokens::Tokens; -use ic_stable_structures::Storable; use std::cmp; fn ts(n: u64) -> TimeStamp { From 13da037a398c425f5b62d3afd03b4dfa92cba85e Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 15:04:44 +0000 Subject: [PATCH 112/124] rename decoder --- rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel | 2 +- rs/ledger_suite/icrc1/tokens_u256/src/lib.rs | 4 ++-- .../tokens_u256/src/{encode_decode.rs => minicbor_u256.rs} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename rs/ledger_suite/icrc1/tokens_u256/src/{encode_decode.rs => minicbor_u256.rs} (100%) diff --git a/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel b/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel index e389bb18fd6..85c2c598f42 100644 --- a/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel +++ b/rs/ledger_suite/icrc1/tokens_u256/BUILD.bazel @@ -15,8 +15,8 @@ COMMON_DEPS = [ rust_library( name = "tokens_u256", srcs = [ - "src/encode_decode.rs", "src/lib.rs", + "src/minicbor_u256.rs", ], crate_name = "ic_icrc1_tokens_u256", version = "0.1.0", diff --git a/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs b/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs index d4541da47fd..e3ac320d734 100644 --- a/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs +++ b/rs/ledger_suite/icrc1/tokens_u256/src/lib.rs @@ -13,7 +13,7 @@ use serde::{ use std::borrow::Cow; use std::fmt; -pub mod encode_decode; +pub mod minicbor_u256; /// The tag number for big positive integers. // See https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums @@ -29,7 +29,7 @@ struct U256Repr(#[serde(with = "ethnum::serde::compressed_bytes::be")] u256); /// 256-bit token amounts. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Default, Decode, Encode)] -pub struct U256(#[cbor(n(0), with = "encode_decode")] u256); +pub struct U256(#[cbor(n(0), with = "minicbor_u256")] u256); impl U256 { pub const ZERO: Self = Self(u256::ZERO); diff --git a/rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs b/rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs similarity index 100% rename from rs/ledger_suite/icrc1/tokens_u256/src/encode_decode.rs rename to rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs From 1501fa5f88a43cbc6204df384a60ca9cd101c5ea Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 15:06:39 +0000 Subject: [PATCH 113/124] comment about arrived_at unused --- rs/ledger_suite/icrc1/ledger/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index f74ab794edc..0e2456aa289 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -479,6 +479,7 @@ impl From for Allowance { Self { amount: val.amount, expires_at: val.expires_at, + // This field is not used and will be removed in subsequent PR. arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), } } From fe34d6db7349d29f62267507cd5fa7e53f80c19c Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Wed, 20 Nov 2024 17:36:05 +0000 Subject: [PATCH 114/124] clear stable allowances in case of incomplete migration --- .../common/ledger_core/src/approvals.rs | 6 ++++ rs/ledger_suite/icrc1/ledger/src/lib.rs | 13 ++++++++ rs/ledger_suite/icrc1/ledger/src/main.rs | 7 ++-- rs/ledger_suite/tests/sm-tests/src/lib.rs | 33 ++++++++++++++++--- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 2c034208d66..ff57a2e65a8 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -82,6 +82,8 @@ pub trait AllowancesData { fn len_arrivals(&self) -> usize; fn clear_arrivals(&mut self); + + fn clear_all(&mut self); } #[derive(Debug, Deserialize, Serialize)] @@ -206,6 +208,10 @@ where fn clear_arrivals(&mut self) { self.arrival_queue.clear(); } + + fn clear_all(&mut self) { + panic!("The method `clear_all` should not be called for HeapAllowancesData") + } } #[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)] diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index 0e2456aa289..e347275400d 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -715,6 +715,10 @@ impl Ledger { } } + pub fn clear_stable_allowance_data(&mut self) { + self.stable_approvals.allowances_data.clear_all(); + } + pub fn clear_arrivals(&mut self) { self.approvals.allowances_data.clear_arrivals(); } @@ -1243,4 +1247,13 @@ impl AllowancesData for StableAllowancesData { fn clear_arrivals(&mut self) { panic!("The method `clear_arrivals` should not be called for StableAllowancesData") } + + fn clear_all(&mut self) { + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| { + allowances.clear_new(); + }); + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.clear_new(); + }); + } } diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index aa19a4832b7..d57daa7fdd7 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -136,12 +136,13 @@ fn pre_upgrade() { let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { - Access::with_ledger(|ledger| { + Access::with_ledger_mut(|ledger| { if !is_ready() { // This means that migration did not complete and the correct state // of the ledger is still in UPGRADES_MEMORY. - ic_cdk::println!("Ledger not ready, skipping write to UPGRADES_MEMORY."); - log!(&LOG, "Ledger not ready, skipping write to UPGRADES_MEMORY."); + // We also have to clear incompletely migrated stable allowances data. + log_message("Ledger not ready, skipping write to UPGRADES_MEMORY and clearing stable allowance data."); + ledger.clear_stable_allowance_data(); return; } let writer = Writer::new(bs, 0); diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 02f8968207a..6e99123e286 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2822,7 +2822,7 @@ pub fn test_incomplete_migration( send_approvals(); - let check_approvals = || { + let check_approvals = |non_zero_from: u64| { for i in 2..2 + NUM_APPROVALS { let allowance = Account::get_allowance( &env, @@ -2830,15 +2830,20 @@ pub fn test_incomplete_migration( account, Account::from(PrincipalId::new_user_test_id(i).0), ); - assert_eq!(allowance.allowance, Nat::from(APPROVE_AMOUNT)); + let expected_allowance = if i < non_zero_from { + Nat::from(0u64) + } else { + Nat::from(APPROVE_AMOUNT) + }; + assert_eq!(allowance.allowance, expected_allowance); } }; - check_approvals(); + check_approvals(2); env.upgrade_canister( canister_id, - ledger_wasm_current_lowinstructionlimits, + ledger_wasm_current_lowinstructionlimits.clone(), Encode!(&LedgerArgument::Upgrade(None)).unwrap(), ) .unwrap(); @@ -2861,7 +2866,25 @@ pub fn test_incomplete_migration( .unwrap(); // All approvals should still be in UPGRADES_MEMORY and downgrade should succeed. - check_approvals(); + check_approvals(2); + + for i in 2..5 { + let spender = Account::from(PrincipalId::new_user_test_id(i).0); + let approve_args = default_approve_args(spender, 0); + send_approval(&env, canister_id, account.owner, &approve_args).expect("approval failed"); + } + + check_approvals(5); + + env.upgrade_canister( + canister_id, + ledger_wasm_current_lowinstructionlimits, + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) + .unwrap(); + wait_ledger_ready(&env, canister_id, 20); + + check_approvals(5); } pub fn test_migration_resumes_from_frozen( From e1a72b06a313f06a49ae15af6c96a9aca77b2458 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 08:22:34 +0000 Subject: [PATCH 115/124] simplify clearing stable allowances --- .../common/ledger_core/src/approvals.rs | 6 ----- rs/ledger_suite/icrc1/ledger/src/lib.rs | 22 ++++++++----------- rs/ledger_suite/icrc1/ledger/src/main.rs | 7 +++--- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index ff57a2e65a8..2c034208d66 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -82,8 +82,6 @@ pub trait AllowancesData { fn len_arrivals(&self) -> usize; fn clear_arrivals(&mut self); - - fn clear_all(&mut self); } #[derive(Debug, Deserialize, Serialize)] @@ -208,10 +206,6 @@ where fn clear_arrivals(&mut self) { self.arrival_queue.clear(); } - - fn clear_all(&mut self) { - panic!("The method `clear_all` should not be called for HeapAllowancesData") - } } #[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)] diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index e347275400d..1ba059593c5 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -715,10 +715,6 @@ impl Ledger { } } - pub fn clear_stable_allowance_data(&mut self) { - self.stable_approvals.allowances_data.clear_all(); - } - pub fn clear_arrivals(&mut self) { self.approvals.allowances_data.clear_arrivals(); } @@ -1125,6 +1121,15 @@ pub fn set_ledger_state(ledger_state: LedgerState) { LEDGER_STATE.with(|s| *s.borrow_mut() = ledger_state); } +pub fn clear_stable_allowance_data() { + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| { + allowances.clear_new(); + }); + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.clear_new(); + }); +} + #[derive(Serialize, Deserialize, Debug, Default)] pub struct StableAllowancesData {} @@ -1247,13 +1252,4 @@ impl AllowancesData for StableAllowancesData { fn clear_arrivals(&mut self) { panic!("The method `clear_arrivals` should not be called for StableAllowancesData") } - - fn clear_all(&mut self) { - ALLOWANCES_MEMORY.with_borrow_mut(|allowances| { - allowances.clear_new(); - }); - ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { - expirations.clear_new(); - }); - } } diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index d57daa7fdd7..0131e5d7cb4 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -16,7 +16,8 @@ use ic_icrc1::{ Operation, Transaction, }; use ic_icrc1_ledger::{ - is_ready, ledger_state, panic_if_not_ready, set_ledger_state, LEDGER_VERSION, UPGRADES_MEMORY, + clear_stable_allowance_data, is_ready, ledger_state, panic_if_not_ready, set_ledger_state, + LEDGER_VERSION, UPGRADES_MEMORY, }; use ic_icrc1_ledger::{InitArgs, Ledger, LedgerArgument, LedgerField, LedgerState}; use ic_ledger_canister_core::ledger::{ @@ -136,13 +137,13 @@ fn pre_upgrade() { let start = ic_cdk::api::instruction_counter(); UPGRADES_MEMORY.with_borrow_mut(|bs| { - Access::with_ledger_mut(|ledger| { + Access::with_ledger(|ledger| { if !is_ready() { // This means that migration did not complete and the correct state // of the ledger is still in UPGRADES_MEMORY. // We also have to clear incompletely migrated stable allowances data. log_message("Ledger not ready, skipping write to UPGRADES_MEMORY and clearing stable allowance data."); - ledger.clear_stable_allowance_data(); + clear_stable_allowance_data(); return; } let writer = Writer::new(bs, 0); From 8867fa3b47b0232e8c0de05e9a2e98a2a2ffd65a Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 09:47:48 +0000 Subject: [PATCH 116/124] start migration only if ledger_version smaller --- rs/ledger_suite/icrc1/ledger/src/main.rs | 24 ++++++++++++++--------- rs/ledger_suite/tests/sm-tests/src/lib.rs | 14 ++++++------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index 0131e5d7cb4..e01bed1c1da 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -214,14 +214,16 @@ fn post_upgrade(args: Option) { ic_cdk::println!("Successfully read state from memory manager managed stable structures"); LEDGER.with_borrow_mut(|ledger| *ledger = Some(state)); - Access::with_ledger_mut(|ledger| { + let upgrade_from_version = Access::with_ledger_mut(|ledger| { if ledger.ledger_version > LEDGER_VERSION { panic!( "Trying to downgrade from incompatible version {}. Current version is {}.", ledger.ledger_version, LEDGER_VERSION ); } + let upgrade_from_version = ledger.ledger_version; ledger.ledger_version = LEDGER_VERSION; + upgrade_from_version }); if let Some(args) = args { @@ -237,14 +239,18 @@ fn post_upgrade(args: Option) { PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); - set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); - Access::with_ledger_mut(|ledger| { - ledger.clear_arrivals(); - }); - log_message("Migration started."); - migrate_next_part( - MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), - ); + if upgrade_from_version < LEDGER_VERSION { + set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); + Access::with_ledger_mut(|ledger| { + ledger.clear_arrivals(); + }); + log_message("Migration started."); + migrate_next_part( + MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), + ); + } else { + set_ledger_state(LedgerState::Ready); + } let end = ic_cdk::api::instruction_counter(); let instructions_consumed = end - start; diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 6e99123e286..97cfe251c9e 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -2419,27 +2419,27 @@ pub fn test_upgrade_serialization( }; add_tx_and_verify(); - let mut test_upgrade = |ledger_wasm: Vec| { + let mut test_upgrade = |ledger_wasm: Vec, expected_migration_steps: u64| { env.upgrade_canister(ledger_id, ledger_wasm, upgrade_args.clone()) .unwrap(); if migration_to_stable_structures { wait_ledger_ready(&env, ledger_id, 10); let stable_upgrade_migration_steps = parse_metric(&env, ledger_id, "ledger_stable_upgrade_migration_steps"); - assert_eq!(stable_upgrade_migration_steps, 1); + assert_eq!(stable_upgrade_migration_steps, expected_migration_steps); } add_tx_and_verify(); }; // Test if the old serialized approvals and balances are correctly deserialized - test_upgrade(ledger_wasm_current.clone()); + test_upgrade(ledger_wasm_current.clone(), 1); // Test the new wasm serialization - test_upgrade(ledger_wasm_current.clone()); + test_upgrade(ledger_wasm_current.clone(), 0); // Test deserializing from memory manager - test_upgrade(ledger_wasm_current.clone()); + test_upgrade(ledger_wasm_current.clone(), 0); if !migration_to_stable_structures { // Test downgrade to mainnet wasm - test_upgrade(ledger_wasm_mainnet.clone()); + test_upgrade(ledger_wasm_mainnet.clone(), 0); } else { // Downgrade from stable structures to mainnet not possible. match env.upgrade_canister( @@ -2600,7 +2600,7 @@ pub fn icrc1_test_upgrade_serialization_fixed_tx( } // Test the new wasm serialization - test_upgrade(ledger_wasm_current_lowinstructionlimits, balances, 1); + test_upgrade(ledger_wasm_current_lowinstructionlimits, balances, 0); // See if the additional approvals are there for a1 in &accounts { From 0cac10fda015bfb8b03248d9fa886f95f1cdd498 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 10:47:02 +0000 Subject: [PATCH 117/124] proptest storable allowance --- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 50 +++++++++++++++-------- rs/ledger_suite/icrc1/ledger/src/tests.rs | 44 +++++++++++++++++++- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index 649786b0436..106e677c42c 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -313,23 +313,39 @@ rust_test( ] ] -rust_test( - name = "ledger_unit_test", - compile_data = [ - "//rs/ledger_suite/icrc1/archive:archive_canister.wasm.gz", - ], - crate = "ledger", - deps = [ - # Keep sorted. - ":ledger", - "//packages/icrc-ledger-types:icrc_ledger_types", - "//rs/ledger_suite/common/ledger_canister_core", - "//rs/ledger_suite/common/ledger_core", - "//rs/ledger_suite/icrc1", - "//rs/ledger_suite/tests/sm-tests:ic-ledger-suite-state-machine-tests", - "//rs/types/base_types", - ], -) +[ + rust_test( + name = "ledger_unit_test" + name_suffix, + compile_data = [ + "//rs/ledger_suite/icrc1/archive:archive_canister.wasm.gz", + ], + crate = "ledger", + crate_features = features, + deps = [ + # Keep sorted. + ":ledger" + name_suffix, + "//packages/icrc-ledger-types:icrc_ledger_types", + "//rs/ledger_suite/common/ledger_canister_core", + "//rs/ledger_suite/common/ledger_core", + "//rs/ledger_suite/icrc1", + "//rs/ledger_suite/tests/sm-tests:ic-ledger-suite-state-machine-tests", + "//rs/types/base_types", + "@crate_index//:proptest", + ] + extra_deps, + ) + for (name_suffix, features, extra_deps) in [ + ( + "", + [], + [], + ), + ( + "_u256", + ["u256-tokens"], + ["//rs/ledger_suite/icrc1/tokens_u256"], + ), + ] +] # Usage: # Benchmarking: diff --git a/rs/ledger_suite/icrc1/ledger/src/tests.rs b/rs/ledger_suite/icrc1/ledger/src/tests.rs index 56f432fba52..ca5634b81d4 100644 --- a/rs/ledger_suite/icrc1/ledger/src/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/src/tests.rs @@ -1,4 +1,4 @@ -use crate::{InitArgs, Ledger}; +use crate::{InitArgs, Ledger, StorableAllowance}; use ic_base_types::PrincipalId; use ic_canister_log::Sink; use ic_icrc1::{Operation, Transaction}; @@ -6,8 +6,11 @@ use ic_ledger_canister_core::archive::ArchiveOptions; use ic_ledger_canister_core::ledger::{LedgerContext, LedgerTransaction, TxApplyError}; use ic_ledger_core::approvals::Allowance; use ic_ledger_core::timestamp::TimeStamp; +use ic_stable_structures::Storable; use icrc_ledger_types::icrc::generic_metadata_value::MetadataValue as Value; use icrc_ledger_types::icrc1::account::Account; +use proptest::prelude::{any, prop_assert_eq, proptest}; +use proptest::strategy::Strategy; use ic_ledger_suite_state_machine_tests::{ ARCHIVE_TRIGGER_THRESHOLD, BLOB_META_KEY, BLOB_META_VALUE, DECIMAL_PLACES, FEE, INT_META_KEY, @@ -630,3 +633,42 @@ fn test_approval_burn_from() { assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); assert_eq!(tokens_to_u64(ctx.balances().total_supply()), 90_000); } + +#[cfg(not(feature = "u256-tokens"))] +fn arb_token() -> impl Strategy { + any::().prop_map(Tokens::new) +} + +#[cfg(feature = "u256-tokens")] +fn arb_token() -> impl Strategy { + (any::(), any::()).prop_map(|(hi, lo)| Tokens::from_words(hi, lo)) +} + +#[test] +fn allowance_serialization() { + fn arb_timestamp() -> impl Strategy { + any::().prop_map(TimeStamp::from_nanos_since_unix_epoch) + } + fn arb_opt_expiration() -> impl Strategy> { + proptest::option::of(any::().prop_map(TimeStamp::from_nanos_since_unix_epoch)) + } + fn arb_allowance() -> impl Strategy> { + (arb_token(), arb_opt_expiration(), arb_timestamp()).prop_map( + |(amount, expires_at, arrived_at)| Allowance { + amount, + expires_at, + arrived_at, + }, + ) + } + proptest!(|(allowance in arb_allowance())| { + let storable_allowance: StorableAllowance = allowance.clone().into(); + let new_allowance: Allowance = StorableAllowance::from_bytes(storable_allowance.to_bytes()).into(); + prop_assert_eq!(new_allowance.amount, allowance.amount); + prop_assert_eq!(new_allowance.expires_at, allowance.expires_at); + prop_assert_eq!( + new_allowance.arrived_at, + TimeStamp::from_nanos_since_unix_epoch(0) + ); + }) +} From 6b071675db1f1722c024c1ecf5b894755f98c645 Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 12:10:30 +0000 Subject: [PATCH 118/124] test minicbor serialization stable --- .../icrc-ledger-types/src/icrc1/account.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/icrc-ledger-types/src/icrc1/account.rs b/packages/icrc-ledger-types/src/icrc1/account.rs index ced24f33bb9..5e50397a543 100644 --- a/packages/icrc-ledger-types/src/icrc1/account.rs +++ b/packages/icrc-ledger-types/src/icrc1/account.rs @@ -194,6 +194,7 @@ mod tests { use ic_stable_structures::Storable; use proptest::prelude::prop; use proptest::strategy::Strategy; + use std::borrow::Cow; use std::str::FromStr; use candid::Principal; @@ -354,4 +355,43 @@ mod tests { prop_assert_eq!(Account::from_bytes(account.to_bytes()), account); }) } + + #[test] + fn test_account_serialization_stability() { + let owner = + Principal::from_str("k2t6j-2nvnp-4zjm3-25dtz-6xhaa-c7boj-5gayf-oj3xs-i43lp-teztq-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20") + .unwrap() + .try_into() + .unwrap(), + ); + let mut accounts = vec![Account { owner, subaccount }]; + let mut serialized_accounts = vec![hex::decode("82581db56bf994b37ae8e79f5ce000be1727a6060ae4eef24736b7cc999c3c0258200102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20").unwrap()]; + let owner = + Principal::from_str("gjfkw-yiolw-ncij7-yzhg2-gq6ec-xi6jy-feyni-g26f4-x7afk-thx6z-6ae") + .unwrap(); + let subaccount = Some( + hex::decode("0000000000000000000000000000000000000000000000000000000000000000") + .unwrap() + .try_into() + .unwrap(), + ); + accounts.push(Account { owner, subaccount }); + serialized_accounts.push(hex::decode("82581d0e5d9a2427f8c9cda343c415d1e4e0a4c3506d78bcbfc0554cf7f67c0258200000000000000000000000000000000000000000000000000000000000000000").unwrap()); + + let owner = Principal::from_str("2chl6-4hpzw-vqaaa-aaaaa-c").unwrap(); + let subaccount = None; + accounts.push(Account { owner, subaccount }); + serialized_accounts.push(hex::decode("8149efcdab000000000001").unwrap()); + + for (i, account) in accounts.iter().enumerate() { + assert_eq!(account.to_bytes(), serialized_accounts[i].clone()); + assert_eq!( + *account, + Account::from_bytes(Cow::Owned(serialized_accounts[i].clone())) + ); + } + } } From e96ab185b95ec621da65e1b756a41383b577fb3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Thu, 21 Nov 2024 13:24:06 +0100 Subject: [PATCH 119/124] Adapt ICRC golden state tests to handle ledger V3 stable structures migration --- WORKSPACE.bazel | 2 + mainnet-canisters.json | 4 + rs/ledger_suite/icrc1/BUILD.bazel | 2 + .../tests/golden_state_upgrade_downgrade.rs | 104 +++++++++++++----- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 2c28698c88f..3cabb35ab87 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -34,6 +34,7 @@ canisters( "sns_governance": "sns-governance-canister.wasm.gz", "swap": "sns-swap-canister.wasm.gz", "sns_ledger": "ic-icrc1-ledger.wasm.gz", + "sns_ledger_v2": "ic-icrc1-ledger.wasm.gz", "sns_archive": "ic-icrc1-archive.wasm.gz", "sns_index": "ic-icrc1-index-ng.wasm.gz", }, @@ -59,6 +60,7 @@ canisters( "sns_governance": "mainnet_sns-governance-canister", "swap": "mainnet_sns-swap-canister", "sns_ledger": "mainnet_ic-icrc1-ledger", + "sns_ledger_v2": "mainnet_ic-icrc1-ledger-v2", "sns_archive": "mainnet_ic-icrc1-archive", "sns_index": "mainnet_ic-icrc1-index-ng", }, diff --git a/mainnet-canisters.json b/mainnet-canisters.json index db92f2ee9cb..a1014864156 100644 --- a/mainnet-canisters.json +++ b/mainnet-canisters.json @@ -79,6 +79,10 @@ "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743", "sha256": "3d808fa63a3d8ebd4510c0400aa078e99a31afaa0515f0b68778f929ce4b2a46" }, + "sns_ledger_v2": { + "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743", + "sha256": "3d808fa63a3d8ebd4510c0400aa078e99a31afaa0515f0b68778f929ce4b2a46" + }, "sns_root": { "rev": "aa91ecacdf3824e193e21b70e0127e8d3edab51a", "sha256": "431cb333feb3f762f742b0dea58745633a2a2ca41075e9933183d850b4ddb259" diff --git a/rs/ledger_suite/icrc1/BUILD.bazel b/rs/ledger_suite/icrc1/BUILD.bazel index 5adb58c8d76..d7271e6e3a3 100644 --- a/rs/ledger_suite/icrc1/BUILD.bazel +++ b/rs/ledger_suite/icrc1/BUILD.bazel @@ -130,6 +130,7 @@ rust_test( "@mainnet_ic-icrc1-archive//file", "@mainnet_ic-icrc1-index-ng//file", "@mainnet_ic-icrc1-ledger//file", + "@mainnet_ic-icrc1-ledger-v2//file", ], env = { "CARGO_MANIFEST_DIR": "rs/ledger_suite/icrc1", @@ -142,6 +143,7 @@ rust_test( "IC_ICRC1_ARCHIVE_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-archive//file)", "IC_ICRC1_INDEX_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-index-ng//file)", "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)", + "IC_ICRC1_LEDGER_DEPLOYED_VERSION_2_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger-v2//file)", "IC_ICRC1_ARCHIVE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz)", "IC_ICRC1_INDEX_NG_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/index-ng:index_ng_canister" + name_suffix + ".wasm.gz)", "IC_ICRC1_LEDGER_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/ledger:ledger_canister" + name_suffix + ".wasm)", diff --git a/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs b/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs index c36245c47ea..37f9cf3ea3a 100644 --- a/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs +++ b/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs @@ -13,7 +13,7 @@ use ic_ledger_suite_state_machine_tests::{ TransactionGenerationParameters, }; use ic_nns_test_utils_golden_nns_state::new_state_machine_with_golden_fiduciary_state_or_panic; -use ic_state_machine_tests::StateMachine; +use ic_state_machine_tests::{ErrorCode, StateMachine, UserError}; use icrc_ledger_types::icrc1::account::Account; use lazy_static::lazy_static; use std::str::FromStr; @@ -44,7 +44,8 @@ lazy_static! { )), Wasm::from_bytes(load_wasm_using_env_var( "CKBTC_IC_ICRC1_ARCHIVE_DEPLOYED_VERSION_WASM_PATH", - )) + )), + None, ); pub static ref MAINNET_SNS_WASMS: Wasms = Wasms::new( Wasm::from_bytes(load_wasm_using_env_var( @@ -55,12 +56,16 @@ lazy_static! { )), Wasm::from_bytes(load_wasm_using_env_var( "IC_ICRC1_ARCHIVE_DEPLOYED_VERSION_WASM_PATH", - )) + )), + Some(Wasm::from_bytes(load_wasm_using_env_var( + "IC_ICRC1_LEDGER_DEPLOYED_VERSION_2_WASM_PATH" + ))), ); pub static ref MASTER_WASMS: Wasms = Wasms::new( Wasm::from_bytes(index_ng_wasm()), Wasm::from_bytes(ledger_wasm()), - Wasm::from_bytes(archive_wasm()) + Wasm::from_bytes(archive_wasm()), + None, ); } @@ -75,12 +80,14 @@ lazy_static! { )), Wasm::from_bytes(load_wasm_using_env_var( "CKETH_IC_ICRC1_ARCHIVE_DEPLOYED_VERSION_WASM_PATH", - )) + )), + None, ); pub static ref MASTER_WASMS: Wasms = Wasms::new( Wasm::from_bytes(index_ng_wasm()), Wasm::from_bytes(ledger_wasm()), - Wasm::from_bytes(archive_wasm()) + Wasm::from_bytes(archive_wasm()), + None, ); } @@ -88,14 +95,21 @@ pub struct Wasms { index_wasm: Wasm, ledger_wasm: Wasm, archive_wasm: Wasm, + ledger_wasm_v2: Option, } impl Wasms { - fn new(index_wasm: Wasm, ledger_wasm: Wasm, archive_wasm: Wasm) -> Self { + fn new( + index_wasm: Wasm, + ledger_wasm: Wasm, + archive_wasm: Wasm, + ledger_wasm_v2: Option, + ) -> Self { Self { index_wasm, ledger_wasm, archive_wasm, + ledger_wasm_v2, } } } @@ -188,7 +202,7 @@ impl LedgerSuiteConfig { } } - fn upgrade_archives(&self, state_machine: &StateMachine, wasm: &Wasm) { + fn upgrade_archives_or_panic(&self, state_machine: &StateMachine, wasm: &Wasm) { let canister_id = CanisterId::unchecked_from_principal(PrincipalId::from_str(self.ledger_id).unwrap()); let archives = list_archives(state_machine, canister_id); @@ -208,7 +222,7 @@ impl LedgerSuiteConfig { println!("Upgraded {} archive(s)", num_archives); } - fn upgrade_index(&self, state_machine: &StateMachine, wasm: &Wasm) { + fn upgrade_index_or_panic(&self, state_machine: &StateMachine, wasm: &Wasm) { let canister_id = CanisterId::unchecked_from_principal(PrincipalId::from_str(self.index_id).unwrap()); let index_upgrade_arg = IndexArg::Upgrade(IndexUpgradeArg { @@ -222,38 +236,70 @@ impl LedgerSuiteConfig { println!("Upgraded {} index '{}'", self.canister_name, self.index_id); } - fn upgrade_ledger(&self, state_machine: &StateMachine, wasm: &Wasm) { + fn upgrade_ledger(&self, state_machine: &StateMachine, wasm: &Wasm) -> Result<(), UserError> { let canister_id = CanisterId::unchecked_from_principal(PrincipalId::from_str(self.ledger_id).unwrap()); let args = ic_icrc1_ledger::LedgerArgument::Upgrade(None); let args = Encode!(&args).unwrap(); - state_machine - .upgrade_canister(canister_id, wasm.clone().bytes(), args.clone()) - .expect("should successfully upgrade ledger canister"); - println!( - "Upgraded {} ledger '{}'", - self.canister_name, self.ledger_id - ); + match state_machine.upgrade_canister(canister_id, wasm.clone().bytes(), args.clone()) { + Ok(_) => { + println!( + "Upgraded {} ledger '{}'", + self.canister_name, self.ledger_id + ); + Ok(()) + } + Err(e) => { + println!( + "Error upgrading {} ledger '{}': {:?}", + self.canister_name, self.ledger_id, e + ); + Err(e) + } + } } fn upgrade_to_mainnet(&self, state_machine: &StateMachine) { // Upgrade each canister twice to exercise pre-upgrade - self.upgrade_index(state_machine, &self.mainnet_wasms.index_wasm); - self.upgrade_index(state_machine, &self.mainnet_wasms.index_wasm); - self.upgrade_ledger(state_machine, &self.mainnet_wasms.ledger_wasm); - self.upgrade_ledger(state_machine, &self.mainnet_wasms.ledger_wasm); - self.upgrade_archives(state_machine, &self.mainnet_wasms.archive_wasm); - self.upgrade_archives(state_machine, &self.mainnet_wasms.archive_wasm); + self.upgrade_index_or_panic(state_machine, &self.mainnet_wasms.index_wasm); + self.upgrade_index_or_panic(state_machine, &self.mainnet_wasms.index_wasm); + match self.upgrade_ledger(state_machine, &self.mainnet_wasms.ledger_wasm) { + Ok(_) => { + panic!("should not successfully downgrade ledger"); + } + Err(user_error) => user_error.assert_contains( + ErrorCode::CanisterCalledTrap, + "Trying to downgrade from incompatible version", + ), + } + self.upgrade_archives_or_panic(state_machine, &self.mainnet_wasms.archive_wasm); + self.upgrade_archives_or_panic(state_machine, &self.mainnet_wasms.archive_wasm); } fn upgrade_to_master(&self, state_machine: &StateMachine) { // Upgrade each canister twice to exercise pre-upgrade - self.upgrade_index(state_machine, &self.master_wasms.index_wasm); - self.upgrade_index(state_machine, &self.master_wasms.index_wasm); - self.upgrade_ledger(state_machine, &self.master_wasms.ledger_wasm); - self.upgrade_ledger(state_machine, &self.master_wasms.ledger_wasm); - self.upgrade_archives(state_machine, &self.master_wasms.archive_wasm); - self.upgrade_archives(state_machine, &self.master_wasms.archive_wasm); + self.upgrade_index_or_panic(state_machine, &self.master_wasms.index_wasm); + self.upgrade_index_or_panic(state_machine, &self.master_wasms.index_wasm); + self.upgrade_ledger(state_machine, &self.master_wasms.ledger_wasm) + .or_else(|e| { + match ( + e.description().contains( + "Cannot upgrade from scratch stable memory, please upgrade to memory manager first." + ), + &self.mainnet_wasms.ledger_wasm_v2 + ) { + // The upgrade may fail if the target canister is too old - in the case of + // migration to stable structures, the ledger canister must be at least at V2, + // i.e., the ledger state must be managed by the memory manager. + (true, Some(wasm_v2)) => {self.upgrade_ledger(state_machine, wasm_v2)} + _ => Err(e) + } + }) + .expect("should successfully upgrade ledger"); + self.upgrade_ledger(state_machine, &self.master_wasms.ledger_wasm) + .expect("should successfully upgrade ledger"); + self.upgrade_archives_or_panic(state_machine, &self.master_wasms.archive_wasm); + self.upgrade_archives_or_panic(state_machine, &self.master_wasms.archive_wasm); } } From 5ebfed8e6d7ece1c3b67fe2b0beb5ab074361325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Thu, 21 Nov 2024 13:36:08 +0100 Subject: [PATCH 120/124] Upgrade the ledger to master twice also after first upgrading to V2 --- .../icrc1/tests/golden_state_upgrade_downgrade.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs b/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs index 37f9cf3ea3a..070915d955c 100644 --- a/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs +++ b/rs/ledger_suite/icrc1/tests/golden_state_upgrade_downgrade.rs @@ -291,7 +291,11 @@ impl LedgerSuiteConfig { // The upgrade may fail if the target canister is too old - in the case of // migration to stable structures, the ledger canister must be at least at V2, // i.e., the ledger state must be managed by the memory manager. - (true, Some(wasm_v2)) => {self.upgrade_ledger(state_machine, wasm_v2)} + (true, Some(wasm_v2)) => { + self.upgrade_ledger(state_machine, wasm_v2) + .expect("should successfully upgrade ledger to V2"); + self.upgrade_ledger(state_machine, &self.master_wasms.ledger_wasm) + } _ => Err(e) } }) From 4ee6b80d9efaaa975e0bb2523be9b9ea9cd1fe5d Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 12:41:35 +0000 Subject: [PATCH 121/124] build fix --- rs/ledger_suite/icrc1/ledger/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel index ee5a2243842..3a283db1377 100644 --- a/rs/ledger_suite/icrc1/ledger/BUILD.bazel +++ b/rs/ledger_suite/icrc1/ledger/BUILD.bazel @@ -108,6 +108,7 @@ package(default_visibility = ["//visibility:public"]) "icrc3-compatible-data-certificate", ], [ + "//rs/ledger_suite/icrc1/tokens_u64", ], ), ] From 0b4ac9ce5802a516bb125ad4ac53994529f2afdb Mon Sep 17 00:00:00 2001 From: Maciej Modelski Date: Thu, 21 Nov 2024 14:15:24 +0000 Subject: [PATCH 122/124] clear stable allowances in post_upgrade --- rs/ledger_suite/icrc1/ledger/src/main.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rs/ledger_suite/icrc1/ledger/src/main.rs b/rs/ledger_suite/icrc1/ledger/src/main.rs index e01bed1c1da..199990f9c32 100644 --- a/rs/ledger_suite/icrc1/ledger/src/main.rs +++ b/rs/ledger_suite/icrc1/ledger/src/main.rs @@ -141,9 +141,7 @@ fn pre_upgrade() { if !is_ready() { // This means that migration did not complete and the correct state // of the ledger is still in UPGRADES_MEMORY. - // We also have to clear incompletely migrated stable allowances data. - log_message("Ledger not ready, skipping write to UPGRADES_MEMORY and clearing stable allowance data."); - clear_stable_allowance_data(); + log_message("Ledger not ready, skipping write to UPGRADES_MEMORY."); return; } let writer = Writer::new(bs, 0); @@ -240,6 +238,10 @@ fn post_upgrade(args: Option) { PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); if upgrade_from_version < LEDGER_VERSION { + if upgrade_from_version == 0 { + log_message("Upgrading from version 0 which does not use stable memory, clearing stable allowance data."); + clear_stable_allowance_data(); + } set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); Access::with_ledger_mut(|ledger| { ledger.clear_arrivals(); From bdc2bcf87e3856dbc054eba8c358dee38705a57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Thu, 21 Nov 2024 16:22:23 +0100 Subject: [PATCH 123/124] Buildifier --- rs/ledger_suite/icrc1/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/ledger_suite/icrc1/BUILD.bazel b/rs/ledger_suite/icrc1/BUILD.bazel index d7271e6e3a3..98cdb922ecb 100644 --- a/rs/ledger_suite/icrc1/BUILD.bazel +++ b/rs/ledger_suite/icrc1/BUILD.bazel @@ -129,8 +129,8 @@ rust_test( "@mainnet_cketh_ic-icrc1-ledger-u256//file", "@mainnet_ic-icrc1-archive//file", "@mainnet_ic-icrc1-index-ng//file", - "@mainnet_ic-icrc1-ledger//file", "@mainnet_ic-icrc1-ledger-v2//file", + "@mainnet_ic-icrc1-ledger//file", ], env = { "CARGO_MANIFEST_DIR": "rs/ledger_suite/icrc1", From 969e6c2115c49302a919febe21dce1937e6cf099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20Bj=C3=B6rkqvist?= Date: Thu, 28 Nov 2024 15:05:37 +0100 Subject: [PATCH 124/124] Incorrectly handled merge conflicts --- packages/icrc-ledger-types/src/cbor/mod.rs | 1 - .../icrc-ledger-types/src/cbor/principal.rs | 17 ------ packages/icrc-ledger-types/src/lib.rs | 1 - .../icrc1/tests/upgrade_downgrade.rs | 37 ++++++++----- .../icrc1/tokens_u256/src/minicbor_u256.rs | 55 ------------------- 5 files changed, 22 insertions(+), 89 deletions(-) delete mode 100644 packages/icrc-ledger-types/src/cbor/mod.rs delete mode 100644 packages/icrc-ledger-types/src/cbor/principal.rs delete mode 100644 rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs diff --git a/packages/icrc-ledger-types/src/cbor/mod.rs b/packages/icrc-ledger-types/src/cbor/mod.rs deleted file mode 100644 index f9fe4658e5b..00000000000 --- a/packages/icrc-ledger-types/src/cbor/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod principal; diff --git a/packages/icrc-ledger-types/src/cbor/principal.rs b/packages/icrc-ledger-types/src/cbor/principal.rs deleted file mode 100644 index 027ae45f7dd..00000000000 --- a/packages/icrc-ledger-types/src/cbor/principal.rs +++ /dev/null @@ -1,17 +0,0 @@ -use candid::Principal; -use minicbor::decode::{Decoder, Error}; -use minicbor::encode::{Encoder, Write}; - -pub fn decode(d: &mut Decoder<'_>, _ctx: &mut Ctx) -> Result { - let bytes = d.bytes()?; - Principal::try_from_slice(bytes).map_err(|e| Error::message(e.to_string())) -} - -pub fn encode( - v: &Principal, - e: &mut Encoder, - _ctx: &mut Ctx, -) -> Result<(), minicbor::encode::Error> { - e.bytes(v.as_slice())?; - Ok(()) -} diff --git a/packages/icrc-ledger-types/src/lib.rs b/packages/icrc-ledger-types/src/lib.rs index 4e5cffbf9c0..f532acb4880 100644 --- a/packages/icrc-ledger-types/src/lib.rs +++ b/packages/icrc-ledger-types/src/lib.rs @@ -1,4 +1,3 @@ -pub mod cbor; pub mod icrc; pub mod icrc1; pub mod icrc2; diff --git a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs index 9e15fb6ecdf..d44725258db 100644 --- a/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs +++ b/rs/ledger_suite/icrc1/tests/upgrade_downgrade.rs @@ -67,23 +67,30 @@ fn should_upgrade_and_downgrade_ledger_canister_suite() { ) .unwrap(); - // Downgrade to mainnet is not possible for this version. - // env.advance_time(Duration::from_secs(60)); - // env.tick(); + env.advance_time(Duration::from_secs(60)); + env.tick(); - // env.upgrade_canister( - // index_id, - // index_ng_mainnet_wasm(), - // Encode!(&index_upgrade_arg).unwrap(), - // ) - // .unwrap(); + env.upgrade_canister( + index_id, + index_ng_mainnet_wasm(), + Encode!(&index_upgrade_arg).unwrap(), + ) + .unwrap(); - // env.upgrade_canister( - // ledger_id, - // ledger_mainnet_wasm(), - // Encode!(&ledger_upgrade_arg).unwrap(), - // ) - // .unwrap(); + match env.upgrade_canister( + ledger_id, + ledger_mainnet_wasm(), + Encode!(&ledger_upgrade_arg).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade to mainnet should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; } fn default_archive_options() -> ArchiveOptions { diff --git a/rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs b/rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs deleted file mode 100644 index 5df93d1f6ce..00000000000 --- a/rs/ledger_suite/icrc1/tokens_u256/src/minicbor_u256.rs +++ /dev/null @@ -1,55 +0,0 @@ -use ethnum::u256; -use minicbor::data::Tag; -use minicbor::decode::{Decoder, Error}; -use minicbor::encode::{Encoder, Write}; - -const U32_MAX: u256 = u256::new(u32::MAX as u128); -const U64_MAX: u256 = u256::new(u64::MAX as u128); - -pub fn decode(d: &mut Decoder<'_>, _ctx: &mut Ctx) -> Result { - let pos = d.position(); - match d.u64() { - Ok(n) => return Ok(u256::from(n)), - Err(e) if e.is_type_mismatch() => { - d.set_position(pos); - } - Err(e) => return Err(e), - } - - let tag: Tag = d.tag()?; - if tag != Tag::PosBignum { - return Err(Error::message( - "failed to parse u256: expected a PosBignum tag", - )); - } - let bytes = d.bytes()?; - if bytes.len() > 32 { - return Err(Error::message(format!( - "failed to parse u256: expected at most 32 bytes, got: {}", - bytes.len() - ))); - } - let mut be_bytes = [0u8; 32]; - be_bytes[32 - bytes.len()..32].copy_from_slice(bytes); - Ok(u256::from_be_bytes(be_bytes)) -} - -pub fn encode( - v: &u256, - e: &mut Encoder, - _ctx: &mut Ctx, -) -> Result<(), minicbor::encode::Error> { - if v <= &U32_MAX { - e.u32(v.as_u32())?; - } else if v <= &U64_MAX { - e.u64(v.as_u64())?; - } else { - let be_bytes = v.to_be_bytes(); - let non_zero_pos = be_bytes - .iter() - .position(|x| *x != 0) - .unwrap_or(be_bytes.len()); - e.tag(Tag::PosBignum)?.bytes(&be_bytes[non_zero_pos..])?; - } - Ok(()) -}