diff --git a/Cargo.lock b/Cargo.lock index 56c7d021ba8..604edc80b6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4423,7 +4423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" dependencies = [ "futures-io", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pki-types", ] @@ -5047,7 +5047,7 @@ dependencies = [ "clap 4.5.20", "hyper 1.5.1", "hyper-util", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pemfile 2.2.0", "serde_json", "tokio", @@ -5176,7 +5176,7 @@ dependencies = [ "hyper 1.5.1", "hyper-util", "log", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -5700,7 +5700,7 @@ dependencies = [ "rand 0.8.5", "rcgen", "reqwest 0.12.9", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-acme", "rustls-pemfile 2.2.0", "rustls-platform-verifier", @@ -5793,7 +5793,7 @@ dependencies = [ "rcgen", "regex", "reqwest 0.12.9", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pemfile 2.2.0", "serde", "serde_bytes", @@ -6134,7 +6134,7 @@ dependencies = [ "prost 0.13.3", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.18", + "rustls 0.23.19", "serde", "serde_cbor", "tokio", @@ -7024,7 +7024,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rsa", - "rustls 0.23.18", + "rustls 0.23.19", "serde", "sha2 0.10.8", "simple_asn1", @@ -7784,7 +7784,7 @@ dependencies = [ "ic-types-test-utils", "rand 0.8.5", "rand_chacha 0.3.1", - "rustls 0.23.18", + "rustls 0.23.19", "tempfile", "tokio", ] @@ -7984,7 +7984,7 @@ dependencies = [ "ic-types", "pkcs8", "rand 0.8.5", - "rustls 0.23.18", + "rustls 0.23.19", "signature", "time", "tokio", @@ -8020,7 +8020,7 @@ dependencies = [ "ic-types", "json5", "maplit", - "rustls 0.23.18", + "rustls 0.23.19", "serde", "thiserror 2.0.3", "x509-parser", @@ -8033,7 +8033,7 @@ dependencies = [ "ic-base-types", "ic-crypto-tls-interfaces", "mockall", - "rustls 0.23.18", + "rustls 0.23.19", ] [[package]] @@ -8550,7 +8550,7 @@ dependencies = [ "rand 0.8.5", "reqwest 0.12.9", "rstest", - "rustls 0.23.18", + "rustls 0.23.19", "serde", "serde_bytes", "serde_cbor", @@ -8670,7 +8670,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "rstest", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pemfile 2.2.0", "serde", "serde_json", @@ -10593,6 +10593,7 @@ dependencies = [ "csv", "hex", "ic-base-types", + "ic-ledger-canister-core", "ic-nns-constants", "ic-nns-governance-api", "ic-nns-gtc", @@ -10823,7 +10824,7 @@ dependencies = [ "quinn", "quinn-udp", "rcgen", - "rustls 0.23.18", + "rustls 0.23.19", "serde", "slog", "tempfile", @@ -10987,7 +10988,7 @@ dependencies = [ "prost 0.13.3", "quinn", "rstest", - "rustls 0.23.18", + "rustls 0.23.19", "slog", "socket2 0.5.7", "static_assertions", @@ -14039,6 +14040,7 @@ dependencies = [ "ic-ledger-test-utils", "ic-nns-constants", "ic-nns-test-utils-golden-nns-state", + "ic-stable-structures", "ic-state-machine-tests", "ic-test-utilities-compare-dirs", "icrc-ledger-types", @@ -14829,7 +14831,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem 3.0.4", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pemfile 2.2.0", "secrecy", "serde", @@ -15023,6 +15025,7 @@ dependencies = [ "ic-base-types", "ic-canister-log 0.2.0", "ic-cdk 0.16.0", + "ic-cdk-timers", "ic-error-types", "ic-icrc1", "ic-icrc1-test-utils", @@ -15039,8 +15042,10 @@ dependencies = [ "icrc-ledger-types", "intmap", "lazy_static", + "minicbor", "num-traits", "on_wire", + "proptest", "serde", "serde_bytes", "serde_cbor", @@ -17947,7 +17952,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustls 0.23.19", "socket2 0.5.7", "thiserror 1.0.68", "tokio", @@ -17964,7 +17969,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustls 0.23.19", "slab", "thiserror 1.0.68", "tinyvec", @@ -18590,7 +18595,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-native-certs 0.8.0", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -19005,9 +19010,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "brotli 7.0.0", "brotli-decompressor", @@ -19107,7 +19112,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.18", + "rustls 0.23.19", "rustls-native-certs 0.7.3", "rustls-platform-verifier-android", "rustls-webpki 0.102.8", @@ -21187,7 +21192,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.18", + "rustls 0.23.19", "rustls-pki-types", "tokio", ] diff --git a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs index 08ee45f7605..c8190e968e0 100644 --- a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs +++ b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs @@ -332,52 +332,6 @@ where .unwrap(); } - // We estimate that an approval takes up twice as much space as a balance: - // balance = account + num_tokens - // approval = 2 * account + num_tokens + timestamp - let max_number_of_approvals = - (effective_max_number_of_accounts - ledger.balances().store.len()) / 2; - - if ledger.approvals().len() > max_number_of_approvals { - let num_approvals_to_trim = ledger.approvals().len() - max_number_of_approvals; - // There might be some more expired approvals to prune. - ledger.approvals_mut().prune(now, num_approvals_to_trim); - - if ledger.approvals().len() > max_number_of_approvals { - let approvals_to_trim = ledger - .approvals() - .select_approvals_to_trim(ledger.approvals().len() - max_number_of_approvals); - - for approval in approvals_to_trim { - let approve_tx = L::Transaction::approve( - approval.0, - approval.1, - L::Tokens::zero(), - Some(now), - Some(TRIMMED_MEMO), - ); - - approve_tx - .apply(ledger, now, L::Tokens::zero()) - .expect("failed to reset approval to zero"); - - let parent_hash = ledger.blockchain().last_hash; - let fee_collector = ledger.fee_collector().cloned(); - - ledger - .blockchain_mut() - .add_block(L::Block::from_transaction( - parent_hash, - approve_tx, - now, - L::Tokens::zero(), - fee_collector, - )) - .unwrap(); - } - } - } - Ok((height, ledger.blockchain().last_hash.unwrap())) } diff --git a/rs/ledger_suite/common/ledger_core/src/approvals.rs b/rs/ledger_suite/common/ledger_core/src/approvals.rs index 2c034208d66..ebb3f5b03be 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals.rs @@ -17,7 +17,7 @@ pub enum ApproveError { } // The implementations of this trait should store the allowance data -// for (account, spender) pairs and the expirations and arrivals +// for (account, spender) pairs and the expirations // of the allowances. The functions of the trait are meant to be simple // `insert` and `remove` type functions that can be implemented with // regular BTreeMaps or using the stable structures. @@ -50,18 +50,6 @@ pub trait AllowancesData { account_spender: (Self::AccountId, Self::AccountId), ); - fn insert_arrival( - &mut self, - timestamp: TimeStamp, - account_spender: (Self::AccountId, Self::AccountId), - ); - - fn remove_arrival( - &mut self, - timestamp: TimeStamp, - account_spender: (Self::AccountId, Self::AccountId), - ); - #[allow(clippy::type_complexity)] fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))>; @@ -73,14 +61,10 @@ pub trait AllowancesData { &mut self, ) -> Option<((Self::AccountId, Self::AccountId), Allowance)>; - fn oldest_arrivals(&self, n: usize) -> Vec<(Self::AccountId, Self::AccountId)>; - fn len_allowances(&self) -> usize; fn len_expirations(&self) -> usize; - fn len_arrivals(&self) -> usize; - fn clear_arrivals(&mut self); } @@ -150,22 +134,6 @@ where self.expiration_queue.remove(&(timestamp, account_spender)); } - fn insert_arrival( - &mut self, - timestamp: TimeStamp, - account_spender: (Self::AccountId, Self::AccountId), - ) { - self.arrival_queue.insert((timestamp, account_spender)); - } - - fn remove_arrival( - &mut self, - timestamp: TimeStamp, - account_spender: (Self::AccountId, Self::AccountId), - ) { - self.arrival_queue.remove(&(timestamp, account_spender)); - } - fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { self.expiration_queue.first().cloned() } @@ -180,17 +148,6 @@ where self.allowances.pop_first() } - fn oldest_arrivals(&self, n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { - let mut result = vec![]; - for (_t, key) in &self.arrival_queue { - if result.len() >= n { - break; - } - result.push(key.clone()); - } - result - } - fn len_allowances(&self) -> usize { self.allowances.len() } @@ -199,10 +156,6 @@ where self.expiration_queue.len() } - fn len_arrivals(&self) -> usize { - self.arrival_queue.len() - } - fn clear_arrivals(&mut self) { self.arrival_queue.clear(); } @@ -323,7 +276,6 @@ where if let Some(expires_at) = expires_at { table.allowances_data.insert_expiry(expires_at, key.clone()); } - table.allowances_data.insert_arrival(now, key.clone()); table.allowances_data.set_allowance( key, Allowance { @@ -349,9 +301,6 @@ where return Err(ApproveError::AllowanceChanged { current_allowance }); } } - table - .allowances_data - .remove_arrival(old_allowance.arrived_at, key.clone()); if amount == AD::Tokens::zero() { if let Some(expires_at) = old_allowance.expires_at { table.allowances_data.remove_expiry(expires_at, key.clone()); @@ -359,7 +308,6 @@ where table.allowances_data.remove_allowance(&key); return Ok(amount); } - table.allowances_data.insert_arrival(now, key.clone()); table.allowances_data.set_allowance( key.clone(), Allowance { @@ -421,9 +369,6 @@ where if let Some(expires_at) = old_allowance.expires_at { table.allowances_data.remove_expiry(expires_at, key.clone()); } - table - .allowances_data - .remove_arrival(old_allowance.arrived_at, key.clone()); table.allowances_data.remove_allowance(&key); } else { table.allowances_data.set_allowance(key, new_allowance); @@ -435,12 +380,6 @@ where }) } - /// Returns a vector of pairs (account, spender) of size min(n, approvals_size) - /// that represent approvals selected for trimming. - pub fn select_approvals_to_trim(&self, n: usize) -> Vec<(AD::AccountId, AD::AccountId)> { - self.allowances_data.oldest_arrivals(n) - } - /// Prunes allowances that are expired, removes at most `limit` allowances. pub fn prune(&mut self, now: TimeStamp, limit: usize) -> usize { self.with_postconditions_check(|table| { @@ -460,9 +399,6 @@ where let key = (account, spender); if let Some(allowance) = table.allowances_data.get_allowance(&key) { if allowance.expires_at.unwrap_or_else(remote_future) <= now { - table - .allowances_data - .remove_arrival(allowance.arrived_at, key.clone()); table.allowances_data.remove_allowance(&key); pruned += 1; } diff --git a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs index c6ea7acb0b4..d2f8abc06df 100644 --- a/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs +++ b/rs/ledger_suite/common/ledger_core/src/approvals/tests.rs @@ -1,9 +1,6 @@ -use std::collections::HashSet; - use super::*; use crate::timestamp::TimeStamp; use crate::tokens::Tokens; -use std::cmp; fn ts(n: u64) -> TimeStamp { TimeStamp::from_nanos_since_unix_epoch(n) @@ -447,87 +444,6 @@ fn allowance_table_remove_zero_allowance() { assert_eq!(table.len(), 0); } -#[test] -fn allowance_table_select_approvals_for_trimming() { - let mut table = TestAllowanceTable::default(); - - let approvals_len = 10; - for i in 1..approvals_len + 1 { - let expiration = if i > 5 { None } else { Some(ts(20 - i)) }; - table - .approve( - &Account(0), - &Account(i), - tokens(100), - expiration, - ts(i), - None, - ) - .unwrap(); - } - - for i in 0..approvals_len + 2 { - let remove = table.select_approvals_to_trim(i as usize); - let remove_set: HashSet<(Account, Account)> = remove.into_iter().collect(); - assert_eq!(remove_set.len(), cmp::min(i, approvals_len) as usize); - - for spender in 1..i + 1 { - assert!( - i > approvals_len || remove_set.contains(&(Account(0), Account(spender))), - "approval for spender {} should be selected for trimming", - spender - ); - } - } - - fn spender_id(approval_key: &(Account, Account)) -> u64 { - approval_key.1 .0 - } - - let remove = table.select_approvals_to_trim(1); - assert_eq!(remove.len(), 1); - assert_eq!(spender_id(&remove[0]), 1); - - // Update the approval to change its place in the prune queue. - table - .approve( - &Account(0), - &Account(1), - tokens(100), - Some(ts(15)), - ts(14), - None, - ) - .unwrap(); - - let remove = table.select_approvals_to_trim(1); - assert_eq!(remove.len(), 1); - assert_eq!(spender_id(&remove[0]), 2); - - // Use up the allowance to remove it from the prune queue. - table - .use_allowance(&Account(0), &Account(2), tokens(100), ts(1)) - .unwrap(); - - let remove = table.select_approvals_to_trim(1); - assert_eq!(remove.len(), 1); - assert_eq!(spender_id(&remove[0]), 3); - - // Reset the allowance to zero; the approval should be removed from the queue. - table - .approve(&Account(0), &Account(3), tokens(0), None, ts(15), None) - .unwrap(); - - let remove = table.select_approvals_to_trim(1); - assert_eq!(remove.len(), 1); - assert_eq!(spender_id(&remove[0]), 4); - // approvals for 2 and 3 were removed - assert_eq!( - table.select_approvals_to_trim(100).len(), - approvals_len as usize - 2 - ); -} - #[test] fn arrival_table_updated_correctly() { let mut table = TestAllowanceTable::default(); diff --git a/rs/ledger_suite/common/ledger_core/src/tokens.rs b/rs/ledger_suite/common/ledger_core/src/tokens.rs index 1e09f3300ed..3707aa1dae4 100644 --- a/rs/ledger_suite/common/ledger_core/src/tokens.rs +++ b/rs/ledger_suite/common/ledger_core/src/tokens.rs @@ -1,4 +1,5 @@ use candid::{CandidType, Nat}; +use minicbor::{Decode, Encode}; use num_traits::{Bounded, ToPrimitive}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::fmt; @@ -116,10 +117,13 @@ impl TokensType for T where CandidType, Deserialize, Serialize, + Decode, + Encode, )] pub struct Tokens { /// Number of 10^-8 Tokens. /// Named because the equivalent part of a Bitcoin is called a Satoshi + #[n(0)] e8s: u64, } diff --git a/rs/ledger_suite/icp/BUILD.bazel b/rs/ledger_suite/icp/BUILD.bazel index c33422eb231..999d83e0d54 100644 --- a/rs/ledger_suite/icp/BUILD.bazel +++ b/rs/ledger_suite/icp/BUILD.bazel @@ -45,6 +45,7 @@ rust_library( "@crate_index//:crc32fast", "@crate_index//:hex", "@crate_index//:ic-cdk", + "@crate_index//:ic-stable-structures", "@crate_index//:prost", "@crate_index//:serde", "@crate_index//:serde_bytes", diff --git a/rs/ledger_suite/icp/Cargo.toml b/rs/ledger_suite/icp/Cargo.toml index ad07ddc061b..f988f889ea4 100644 --- a/rs/ledger_suite/icp/Cargo.toml +++ b/rs/ledger_suite/icp/Cargo.toml @@ -20,6 +20,7 @@ ic-crypto-sha2 = { path = "../../crypto/sha2" } ic-ledger-canister-core = { path = "../common/ledger_canister_core" } ic-ledger-core = { path = "../common/ledger_core" } ic-ledger-hash-of = { path = "../../../packages/ic-ledger-hash-of" } +ic-stable-structures = { workspace = true } icrc-ledger-types = { path = "../../../packages/icrc-ledger-types" } lazy_static = { workspace = true } on_wire = { path = "../../rust_canisters/on_wire" } diff --git a/rs/ledger_suite/icp/ledger.did b/rs/ledger_suite/icp/ledger.did index e35a627d6e5..a4778eb2446 100644 --- a/rs/ledger_suite/icp/ledger.did +++ b/rs/ledger_suite/icp/ledger.did @@ -535,4 +535,6 @@ service: (LedgerCanisterPayload) -> { icrc21_canister_call_consent_message: (icrc21_consent_message_request) -> (icrc21_consent_message_response); icrc10_supported_standards : () -> (vec record { name : text; url : text }) query; + + is_ledger_ready: () -> (bool) query; } diff --git a/rs/ledger_suite/icp/ledger/BUILD.bazel b/rs/ledger_suite/icp/ledger/BUILD.bazel index 3db01b4451b..0d9bea50cb9 100644 --- a/rs/ledger_suite/icp/ledger/BUILD.bazel +++ b/rs/ledger_suite/icp/ledger/BUILD.bazel @@ -39,9 +39,12 @@ package(default_visibility = ["//visibility:public"]) "//rs/rust_canisters/dfn_core", "//rs/types/base_types", "@crate_index//:candid", + "@crate_index//:hex", + "@crate_index//:ic-cdk", "@crate_index//:ic-stable-structures", "@crate_index//:intmap", "@crate_index//:lazy_static", + "@crate_index//:minicbor", "@crate_index//:num-traits", "@crate_index//:serde", "@crate_index//:serde_bytes", @@ -69,6 +72,7 @@ rust_test( rustc_env = { "LEDGER_ARCHIVE_NODE_CANISTER_WASM_PATH": "$(execpath //rs/ledger_suite/icp/archive:ledger-archive-node-canister-wasm.wasm.gz)", }, + deps = ["@crate_index//:proptest"], ) rust_ledger_canister(name = "ledger-canister-wasm") @@ -88,6 +92,11 @@ rust_ledger_canister( extra_deps = [":ledger_next_version"], ) +rust_ledger_canister( + name = "ledger-canister-wasm-low-limits", + crate_features = ["low-upgrade-instruction-limits"], +) + rust_test( name = "ledger_canister_unit_test", compile_data = LEDGER_CANISTER_DATA, @@ -107,6 +116,7 @@ rust_ic_test( data = [ ":ledger-canister-wasm", ":ledger-canister-wasm-allowance-getter", + ":ledger-canister-wasm-low-limits", ":ledger-canister-wasm-next-version", "@mainnet_icp_ledger_canister//file", ], @@ -117,6 +127,7 @@ rust_ic_test( "LEDGER_CANISTER_WASM_PATH": "$(rootpath :ledger-canister-wasm)", "LEDGER_CANISTER_ALLOWANCE_GETTER_WASM_PATH": "$(rootpath :ledger-canister-wasm-allowance-getter)", "LEDGER_CANISTER_NEXT_VERSION_WASM_PATH": "$(rootpath :ledger-canister-wasm-next-version)", + "LEDGER_CANISTER_LOW_LIMITS_WASM_PATH": "$(rootpath :ledger-canister-wasm-low-limits)", }, flaky = True, tags = ["cpu:4"], diff --git a/rs/ledger_suite/icp/ledger/Cargo.toml b/rs/ledger_suite/icp/ledger/Cargo.toml index 5d2708999a6..7d778ae417c 100644 --- a/rs/ledger_suite/icp/ledger/Cargo.toml +++ b/rs/ledger_suite/icp/ledger/Cargo.toml @@ -19,9 +19,11 @@ dfn_candid = { path = "../../../rust_canisters/dfn_candid" } dfn_core = { path = "../../../rust_canisters/dfn_core" } dfn_http_metrics = { path = "../../../rust_canisters/dfn_http_metrics" } dfn_protobuf = { path = "../../../rust_canisters/dfn_protobuf" } +hex = { workspace = true } ic-base-types = { path = "../../../types/base_types" } ic-canister-log = { path = "../../../rust_canisters/canister_log" } ic-cdk = { workspace = true } +ic-cdk-timers = { workspace = true } ic-limits = { path = "../../../limits" } ic-icrc1 = { path = "../../icrc1" } ic-ledger-canister-core = { path = "../../common/ledger_canister_core" } @@ -33,6 +35,7 @@ icp-ledger = { path = "../" } icrc-ledger-types = { path = "../../../../packages/icrc-ledger-types" } intmap = { version = "1.1.0", features = ["serde"] } lazy_static = { workspace = true } +minicbor = { workspace = true } num-traits = { workspace = true } on_wire = { path = "../../../rust_canisters/on_wire" } serde = { workspace = true } @@ -41,15 +44,16 @@ serde_cbor = { workspace = true } [dev-dependencies] candid_parser = { workspace = true } -hex = { workspace = true } ic-agent = { workspace = true } ic-error-types = { path = "../../../types/error_types" } ic-ledger-suite-state-machine-tests = { path = "../../tests/sm-tests" } ic-icrc1-test-utils = { path = "../../icrc1/test_utils" } ic-state-machine-tests = { path = "../../../state_machine_tests" } ic-test-utilities-load-wasm = { path = "../../../test_utilities/load_wasm" } +proptest = { workspace = true } [features] notify-method = [] icp-allowance-getter = [] next-ledger-version = [] +low-upgrade-instruction-limits = [] diff --git a/rs/ledger_suite/icp/ledger/ledger_canisters.bzl b/rs/ledger_suite/icp/ledger/ledger_canisters.bzl index 0fef5e2b0e8..2d0f496dae3 100644 --- a/rs/ledger_suite/icp/ledger/ledger_canisters.bzl +++ b/rs/ledger_suite/icp/ledger/ledger_canisters.bzl @@ -41,6 +41,7 @@ LEDGER_CANISTER_DEPS = [ "@crate_index//:candid", "@crate_index//:ciborium", "@crate_index//:ic-cdk", + "@crate_index//:ic-cdk-timers", "@crate_index//:ic-metrics-encoder", "@crate_index//:ic-stable-structures", "@crate_index//:num-traits", diff --git a/rs/ledger_suite/icp/ledger/src/lib.rs b/rs/ledger_suite/icp/ledger/src/lib.rs index ab9052d3fab..d401537dd74 100644 --- a/rs/ledger_suite/icp/ledger/src/lib.rs +++ b/rs/ledger_suite/icp/ledger/src/lib.rs @@ -6,13 +6,16 @@ use ic_ledger_canister_core::ledger::{ self as core_ledger, LedgerContext, LedgerData, TransactionInfo, }; use ic_ledger_core::{ - approvals::AllowanceTable, approvals::HeapAllowancesData, balances::Balances, - block::EncodedBlock, timestamp::TimeStamp, + approvals::{Allowance, AllowanceTable, AllowancesData}, + balances::Balances, + block::EncodedBlock, + timestamp::TimeStamp, }; use ic_ledger_core::{block::BlockIndex, tokens::Tokens}; use ic_ledger_hash_of::HashOf; use ic_stable_structures::memory_manager::{MemoryId, MemoryManager, VirtualMemory}; -use ic_stable_structures::DefaultMemoryImpl; +use ic_stable_structures::{storable::Bound, Storable}; +use ic_stable_structures::{DefaultMemoryImpl, StableBTreeMap}; use icp_ledger::{ AccountIdentifier, Block, FeatureFlags, LedgerAllowances, LedgerBalances, Memo, Operation, PaymentError, Transaction, TransferError, TransferFee, UpgradeArgs, DEFAULT_TRANSFER_FEE, @@ -20,6 +23,7 @@ use icp_ledger::{ use icrc_ledger_types::icrc1::account::Account; use intmap::IntMap; use lazy_static::lazy_static; +use minicbor::{Decode, Encode}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::cell::RefCell; @@ -69,6 +73,54 @@ fn default_ledger_version() -> u64 { } const UPGRADES_MEMORY_ID: MemoryId = MemoryId::new(0); +const ALLOWANCES_MEMORY_ID: MemoryId = MemoryId::new(1); +const ALLOWANCES_EXPIRATIONS_MEMORY_ID: MemoryId = MemoryId::new(2); + +#[derive(Clone, Debug, Encode, Decode)] +struct StorableAllowance { + #[n(0)] + amount: Tokens, + #[n(1)] + expires_at: Option, +} + +impl Storable for StorableAllowance { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buf = vec![]; + minicbor::encode(self, &mut buf).expect("StorableAllowance encoding should always succeed"); + Cow::Owned(buf) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + minicbor::decode(bytes.as_ref()).unwrap_or_else(|e| { + panic!( + "failed to decode StorableAllowance bytes {}: {e}", + hex::encode(bytes) + ) + }) + } + + const BOUND: Bound = Bound::Unbounded; +} + +impl From> for StorableAllowance { + fn from(val: Allowance) -> Self { + Self { + amount: val.amount, + expires_at: val.expires_at, + } + } +} + +impl From for Allowance { + fn from(val: StorableAllowance) -> Self { + Self { + amount: val.amount, + expires_at: val.expires_at, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), + } + } +} thread_local! { static MEMORY_MANAGER: RefCell> = RefCell::new( @@ -78,6 +130,36 @@ thread_local! { // The memory where the ledger must write and read its state during an upgrade. pub static UPGRADES_MEMORY: RefCell> = MEMORY_MANAGER.with(|memory_manager| RefCell::new(memory_manager.borrow().get(UPGRADES_MEMORY_ID))); + + pub static LEDGER_STATE: RefCell = const { RefCell::new(LedgerState::Ready) }; + + // (from, spender) -> allowance - map storing ledger allowances. + #[allow(clippy::type_complexity)] + pub static ALLOWANCES_MEMORY: RefCell>> = + MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_MEMORY_ID)))); + + // (timestamp, (from, spender)) - expiration set used for removing expired allowances. + #[allow(clippy::type_complexity)] + pub static ALLOWANCES_EXPIRATIONS_MEMORY: RefCell>> = + MEMORY_MANAGER.with(|memory_manager| RefCell::new(StableBTreeMap::init(memory_manager.borrow().get(ALLOWANCES_EXPIRATIONS_MEMORY_ID)))); +} + +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] +pub enum LedgerField { + Allowances, + AllowancesExpirations, +} + +#[derive(Copy, Clone, Serialize, Deserialize, Debug)] +pub enum LedgerState { + Migrating(LedgerField), + Ready, +} + +impl Default for LedgerState { + fn default() -> Self { + Self::Ready + } } /// The ledger versions represent backwards incompatible versions of the ledger. @@ -85,17 +167,20 @@ thread_local! { /// Upgrading from version N to version N+1 should always be possible. /// We have the following ledger versions: /// * 0 - the whole ledger state is stored on the heap. +/// * 1 - the allowances are stored in stable structures. #[cfg(not(feature = "next-ledger-version"))] -pub const LEDGER_VERSION: u64 = 0; +pub const LEDGER_VERSION: u64 = 1; #[cfg(feature = "next-ledger-version")] -pub const LEDGER_VERSION: u64 = 1; +pub const LEDGER_VERSION: u64 = 2; #[derive(Debug, Deserialize, Serialize)] pub struct Ledger { - pub balances: LedgerBalances, + balances: LedgerBalances, #[serde(default)] - pub approvals: LedgerAllowances, + approvals: LedgerAllowances, + #[serde(default)] + stable_approvals: AllowanceTable, pub blockchain: Blockchain, // A cap on the maximum number of accounts. pub maximum_number_of_accounts: usize, @@ -143,24 +228,28 @@ pub struct Ledger { impl LedgerContext for Ledger { type AccountId = AccountIdentifier; - type AllowancesData = HeapAllowancesData; + type AllowancesData = StableAllowancesData; type BalancesStore = BTreeMap; type Tokens = Tokens; fn balances(&self) -> &Balances { + panic_if_not_ready(); &self.balances } fn balances_mut(&mut self) -> &mut Balances { + panic_if_not_ready(); &mut self.balances } fn approvals(&self) -> &AllowanceTable { - &self.approvals + panic_if_not_ready(); + &self.stable_approvals } fn approvals_mut(&mut self) -> &mut AllowanceTable { - &mut self.approvals + panic_if_not_ready(); + &mut self.stable_approvals } fn fee_collector(&self) -> Option<&ic_ledger_core::block::FeeCollector> { @@ -241,6 +330,7 @@ impl Default for Ledger { fn default() -> Self { Self { approvals: Default::default(), + stable_approvals: Default::default(), balances: LedgerBalances::default(), blockchain: Blockchain::default(), maximum_number_of_accounts: 28_000_000, @@ -478,6 +568,34 @@ impl Ledger { self.feature_flags = feature_flags; } } + + pub fn migrate_one_allowance(&mut self) -> bool { + match self.approvals.allowances_data.pop_first_allowance() { + Some((account_spender, allowance)) => { + self.stable_approvals + .allowances_data + .set_allowance(account_spender, allowance); + true + } + None => false, + } + } + + pub fn migrate_one_expiration(&mut self) -> bool { + match self.approvals.allowances_data.pop_first_expiry() { + Some((timestamp, account_spender)) => { + self.stable_approvals + .allowances_data + .insert_expiry(timestamp, account_spender); + true + } + None => false, + } + } + + pub fn clear_arrivals(&mut self) { + self.approvals.allowances_data.clear_arrivals(); + } } pub fn add_payment( @@ -504,3 +622,114 @@ pub fn change_notification_state( TimeStamp::from(now()), ) } + +pub fn is_ready() -> bool { + LEDGER_STATE.with(|s| matches!(*s.borrow(), LedgerState::Ready)) +} + +pub fn panic_if_not_ready() { + if !is_ready() { + ic_cdk::trap("The Ledger is not ready"); + } +} + +pub fn ledger_state() -> LedgerState { + LEDGER_STATE.with(|s| *s.borrow()) +} + +pub fn set_ledger_state(ledger_state: LedgerState) { + LEDGER_STATE.with(|s| *s.borrow_mut() = ledger_state); +} + +pub fn clear_stable_allowance_data() { + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| { + allowances.clear_new(); + }); + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.clear_new(); + }); +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct StableAllowancesData {} + +impl AllowancesData for StableAllowancesData { + type AccountId = AccountIdentifier; + type Tokens = Tokens; + + fn get_allowance( + &self, + account_spender: &(Self::AccountId, Self::AccountId), + ) -> Option> { + ALLOWANCES_MEMORY + .with_borrow(|allowances| allowances.get(account_spender)) + .map(|a| a.into()) + } + + fn set_allowance( + &mut self, + account_spender: (Self::AccountId, Self::AccountId), + allowance: Allowance, + ) { + ALLOWANCES_MEMORY + .with_borrow_mut(|allowances| allowances.insert(account_spender, allowance.into())); + } + + fn remove_allowance(&mut self, account_spender: &(Self::AccountId, Self::AccountId)) { + ALLOWANCES_MEMORY.with_borrow_mut(|allowances| allowances.remove(account_spender)); + } + + fn insert_expiry( + &mut self, + timestamp: TimeStamp, + account_spender: (Self::AccountId, Self::AccountId), + ) { + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.insert((timestamp, account_spender), ()); + }); + } + + fn remove_expiry( + &mut self, + timestamp: TimeStamp, + account_spender: (Self::AccountId, Self::AccountId), + ) { + ALLOWANCES_EXPIRATIONS_MEMORY.with_borrow_mut(|expirations| { + expirations.remove(&(timestamp, account_spender)); + }); + } + + fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow(|expirations| expirations.first_key_value().map(|kv| kv.0)) + } + + fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)) + } + + fn pop_first_allowance( + &mut self, + ) -> Option<((Self::AccountId, Self::AccountId), Allowance)> { + panic!("The method `pop_first_allowance` should not be called for StableAllowancesData") + } + + fn len_allowances(&self) -> usize { + ALLOWANCES_MEMORY + .with_borrow(|allowances| allowances.len()) + .try_into() + .unwrap() + } + + fn len_expirations(&self) -> usize { + ALLOWANCES_EXPIRATIONS_MEMORY + .with_borrow(|expirations| expirations.len()) + .try_into() + .unwrap() + } + + fn clear_arrivals(&mut self) { + panic!("The method `clear_arrivals` should not be called for StableAllowancesData") + } +} diff --git a/rs/ledger_suite/icp/ledger/src/main.rs b/rs/ledger_suite/icp/ledger/src/main.rs index c49257c8ccb..1b16dba8bf6 100644 --- a/rs/ledger_suite/icp/ledger/src/main.rs +++ b/rs/ledger_suite/icp/ledger/src/main.rs @@ -10,7 +10,9 @@ use dfn_core::{ use dfn_protobuf::protobuf; use ic_base_types::CanisterId; use ic_canister_log::{LogEntry, Sink}; +use ic_cdk::api::instruction_counter; use ic_icrc1::endpoints::{convert_transfer_error, StandardRecord}; +use ic_ledger_canister_core::ledger::LedgerContext; use ic_ledger_canister_core::runtime::heap_memory_size_bytes; use ic_ledger_canister_core::{ archive::{Archive, ArchiveOptions}, @@ -54,7 +56,11 @@ use icrc_ledger_types::{ icrc1::transfer::TransferArg, icrc21::{errors::Icrc21Error, requests::ConsentMessageRequest, responses::ConsentInfo}, }; -use ledger_canister::{Ledger, LEDGER, LEDGER_VERSION, MAX_MESSAGE_SIZE_BYTES, UPGRADES_MEMORY}; +use ledger_canister::{ + clear_stable_allowance_data, is_ready, ledger_state, panic_if_not_ready, set_ledger_state, + Ledger, LedgerField, LedgerState, LEDGER, LEDGER_VERSION, MAX_MESSAGE_SIZE_BYTES, + UPGRADES_MEMORY, +}; use num_traits::cast::ToPrimitive; #[allow(unused_imports)] use on_wire::IntoWire; @@ -298,7 +304,7 @@ async fn icrc1_send( }); } let ledger = LEDGER.read().unwrap(); - let balance = ledger.balances.account_balance(&from); + let balance = ledger.balances().account_balance(&from); let min_burn_amount = ledger.transfer_fee.min(balance); if amount < min_burn_amount { return Err(CoreTransferError::BadBurn { min_burn_amount }); @@ -367,6 +373,7 @@ thread_local! { static NOTIFY_METHOD_CALLS: RefCell = const { RefCell::new(0) }; static PRE_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; static POST_UPGRADE_INSTRUCTIONS_CONSUMED: RefCell = const { RefCell::new(0) }; + static STABLE_UPGRADE_MIGRATION_STEPS: RefCell = const { RefCell::new(0) }; } /// You can notify a canister that you have made a payment to it. The @@ -585,7 +592,7 @@ fn block(block_index: BlockIndex) -> Option> { /// Get an account balance. /// If the account does not exist it will return 0 Tokens fn account_balance(account: AccountIdentifier) -> Tokens { - LEDGER.read().unwrap().balances.account_balance(&account) + LEDGER.read().unwrap().balances().account_balance(&account) } #[candid_method(query, rename = "icrc1_balance_of")] @@ -645,12 +652,12 @@ fn icrc1_fee() -> Nat { /// The total number of Tokens not inside the minting canister fn total_supply() -> Tokens { - LEDGER.read().unwrap().balances.total_supply() + LEDGER.read().unwrap().balances().total_supply() } #[candid_method(query, rename = "icrc1_total_supply")] fn icrc1_total_supply() -> Nat { - Nat::from(LEDGER.read().unwrap().balances.total_supply().get_e8s()) + Nat::from(LEDGER.read().unwrap().balances().total_supply().get_e8s()) } #[candid_method(query, rename = "symbol")] @@ -752,6 +759,16 @@ fn main() { // We use 8MiB buffer const BUFFER_SIZE: usize = 8388608; +#[cfg(not(feature = "low-upgrade-instruction-limits"))] +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 190_000_000_000; +#[cfg(not(feature = "low-upgrade-instruction-limits"))] +const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 1_900_000_000; + +#[cfg(feature = "low-upgrade-instruction-limits")] +const MAX_INSTRUCTIONS_PER_UPGRADE: u64 = 5_000_000; +#[cfg(feature = "low-upgrade-instruction-limits")] +const MAX_INSTRUCTIONS_PER_TIMER_CALL: u64 = 500_000; + fn post_upgrade(args: Option) { let start = dfn_core::api::performance_counter(0); @@ -766,24 +783,15 @@ fn post_upgrade(args: Option) { Err(_) => false, }; - let mut ledger = LEDGER.write().unwrap(); let mut pre_upgrade_instructions_consumed = 0; - if !memory_manager_found { - // The ledger was written with dfn_core and has to be read with dfn_core in order - // to skip the first bytes that contain the length of the stable memory. - let mut stable_reader = dfn_core::stable::StableReader::new(); - *ledger = - ciborium::de::from_reader(&mut stable_reader).expect("Decoding stable memory failed"); - let mut pre_upgrade_instructions_counter_bytes = [0u8; 8]; - pre_upgrade_instructions_consumed = - match stable_reader.read_exact(&mut pre_upgrade_instructions_counter_bytes) { - Ok(_) => u64::from_le_bytes(pre_upgrade_instructions_counter_bytes), - Err(_) => { - // If upgrading from a version that didn't write the instructions counter to stable memory - 0u64 - } - }; - } else { + { + let mut ledger = LEDGER.write().unwrap(); + if !memory_manager_found { + let msg = + "Cannot upgrade from scratch stable memory, please upgrade to memory manager first."; + print(msg); + panic!("{msg}"); + } *ledger = UPGRADES_MEMORY.with_borrow(|bs| { let reader = Reader::new(bs, 0); let mut buffered_reader = BufferedReader::new(BUFFER_SIZE, reader); @@ -801,18 +809,18 @@ fn post_upgrade(args: Option) { }; ledger_state }); - } - if ledger.ledger_version > LEDGER_VERSION { - panic!( - "Trying to downgrade from incompatible version {}. Current version is {}.", - ledger.ledger_version, LEDGER_VERSION - ); - } - ledger.ledger_version = LEDGER_VERSION; + let upgrade_from_version = ledger.ledger_version; + if ledger.ledger_version > LEDGER_VERSION { + panic!( + "Trying to downgrade from incompatible version {}. Current version is {}.", + ledger.ledger_version, LEDGER_VERSION + ); + } + ledger.ledger_version = LEDGER_VERSION; - if let Some(args) = args { - match args { + if let Some(args) = args { + match args { LedgerCanisterPayload::Init(_) => trap_with("Cannot upgrade the canister with an Init argument. Please provide an Upgrade argument."), LedgerCanisterPayload::Upgrade(upgrade_args) => { if let Some(upgrade_args) = upgrade_args { @@ -820,15 +828,30 @@ fn post_upgrade(args: Option) { } } } + } + set_certified_data( + &ledger + .blockchain + .last_hash + .map(|h| h.into_bytes()) + .unwrap_or([0u8; 32]), + ); + PRE_UPGRADE_INSTRUCTIONS_CONSUMED + .with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); + + if upgrade_from_version == 0 { + set_ledger_state(LedgerState::Migrating(LedgerField::Allowances)); + print("Upgrading from version 0 which does not use stable structures, clearing stable allowance data."); + clear_stable_allowance_data(); + ledger.clear_arrivals(); + } + } + if !is_ready() { + print("Migration started."); + migrate_next_part( + MAX_INSTRUCTIONS_PER_UPGRADE.saturating_sub(pre_upgrade_instructions_consumed), + ); } - set_certified_data( - &ledger - .blockchain - .last_hash - .map(|h| h.into_bytes()) - .unwrap_or([0u8; 32]), - ); - PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow_mut() = pre_upgrade_instructions_consumed); let end = dfn_core::api::performance_counter(0); let post_upgrade_instructions_consumed = end - start; @@ -836,6 +859,52 @@ fn post_upgrade(args: Option) { .with(|n| *n.borrow_mut() = post_upgrade_instructions_consumed); } +fn migrate_next_part(instruction_limit: u64) { + let instructions_migration_start = instruction_counter(); + STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow_mut() += 1); + let mut migrated_allowances = 0; + let mut migrated_expirations = 0; + + print("Migrating part of the ledger state."); + + let mut ledger = LEDGER.write().unwrap(); + while instruction_counter() < instruction_limit { + let field = match ledger_state() { + LedgerState::Migrating(ledger_field) => ledger_field, + LedgerState::Ready => break, + }; + match field { + LedgerField::Allowances => { + if ledger.migrate_one_allowance() { + migrated_allowances += 1; + } else { + set_ledger_state(LedgerState::Migrating(LedgerField::AllowancesExpirations)); + } + } + LedgerField::AllowancesExpirations => { + if ledger.migrate_one_expiration() { + migrated_expirations += 1; + } else { + set_ledger_state(LedgerState::Ready); + } + } + } + } + let instructions_migration = instruction_counter() - instructions_migration_start; + let msg = format!("Number of elements migrated: allowances: {migrated_allowances} expirations: {migrated_expirations}. Migration step instructions: {instructions_migration}, total instructions used in message: {}." , + instruction_counter()); + if !is_ready() { + print(format!( + "Migration partially done. Scheduling the next part. {msg}" + )); + ic_cdk_timers::set_timer(Duration::from_secs(0), || { + migrate_next_part(MAX_INSTRUCTIONS_PER_TIMER_CALL) + }); + } else { + print(format!("Migration completed! {msg}")); + } +} + #[export_name = "canister_post_upgrade"] fn post_upgrade_() { over_init(|CandidOne(args)| post_upgrade(args)); @@ -852,6 +921,12 @@ fn pre_upgrade() { .read() // This should never happen, but it's better to be safe than sorry .unwrap_or_else(|poisoned| poisoned.into_inner()); + if !is_ready() { + // This means that migration did not complete and the correct state + // of the ledger is still in UPGRADES_MEMORY. + print!("Ledger not ready, skipping write to UPGRADES_MEMORY."); + return; + } UPGRADES_MEMORY.with_borrow_mut(|bs| { let writer = Writer::new(bs, 0); let mut buffered_writer = BufferedWriter::new(BUFFER_SIZE, writer); @@ -885,6 +960,7 @@ impl LedgerAccess for Access { /// Canister endpoints #[export_name = "canister_update send_pb"] fn send_() { + panic_if_not_ready(); over_async( protobuf, |SendArgs { @@ -904,6 +980,7 @@ fn send_() { #[candid_method(update, rename = "send_dfx")] async fn send_dfx(arg: SendArgs) -> BlockIndex { + panic_if_not_ready(); transfer_candid(TransferArgs::from(arg)) .await .unwrap_or_else(|e| { @@ -918,6 +995,7 @@ async fn send_dfx(arg: SendArgs) -> BlockIndex { /// I STRONGLY recommend that you use "send_pb" instead. #[export_name = "canister_update send_dfx"] fn send_dfx_() { + panic_if_not_ready(); over_async(candid_one, send_dfx); } @@ -952,6 +1030,7 @@ fn notify_() { #[candid_method(update, rename = "transfer")] async fn transfer_candid(arg: TransferArgs) -> Result { + panic_if_not_ready(); let to_account = AccountIdentifier::from_address(arg.to).unwrap_or_else(|e| { trap_with(&format!("Invalid account identifier: {}", e)); }); @@ -970,6 +1049,7 @@ async fn transfer_candid(arg: TransferArgs) -> Result async fn icrc1_transfer( arg: TransferArg, ) -> Result { + panic_if_not_ready(); let from_account = Account { owner: Principal::from(caller()), subaccount: arg.from_subaccount, @@ -998,11 +1078,13 @@ async fn icrc1_transfer( #[export_name = "canister_update transfer"] fn transfer() { + panic_if_not_ready(); over_async_may_reject(candid_one, |arg| async { Ok(transfer_candid(arg).await) }) } #[export_name = "canister_update icrc1_transfer"] fn icrc1_transfer_candid() { + panic_if_not_ready(); over_async_may_reject(candid_one, |arg: TransferArg| async { if !LEDGER.read().unwrap().can_send(&caller()) { return Err("Anonymous principal cannot hold tokens on the ledger.".to_string()); @@ -1014,6 +1096,7 @@ fn icrc1_transfer_candid() { #[candid_method(update, rename = "icrc2_transfer_from")] async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result { + panic_if_not_ready(); if !LEDGER.read().unwrap().feature_flags.icrc2 { trap_with("ICRC-2 features are not enabled on the ledger."); } @@ -1045,6 +1128,7 @@ async fn icrc2_transfer_from(arg: TransferFromArgs) -> Result>) -> std::i ledger.blockchain.num_archived_blocks.saturating_add(ledger.blockchain.blocks.len() as u64) as f64, "Total number of blocks stored in the main memory, plus total number of blocks sent to the archive.", )?; - w.encode_gauge( - "ledger_balances_token_pool", - ledger.balances.token_pool.get_tokens() as f64, - "Total number of Tokens in the pool.", - )?; - w.encode_gauge( - "ledger_balance_store_entries", - ledger.balances.store.len() as f64, - "Total number of accounts in the balance store.", - )?; + if is_ready() { + w.encode_gauge( + "ledger_balances_token_pool", + ledger.balances().token_pool.get_tokens() as f64, + "Total number of Tokens in the pool.", + )?; + w.encode_gauge( + "ledger_balance_store_entries", + ledger.balances().store.len() as f64, + "Total number of accounts in the balance store.", + )?; + } w.encode_gauge( "ledger_most_recent_block_time_seconds", ledger.blockchain.last_timestamp.as_nanos_since_unix_epoch() as f64 / 1_000_000_000.0, @@ -1428,11 +1514,13 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i num_archives as f64, "Total number of archives.", )?; - w.encode_gauge( - "ledger_num_approvals", - ledger.approvals.get_num_approvals() as f64, - "Total number of approvals.", - )?; + if is_ready() { + w.encode_gauge( + "ledger_num_approvals", + ledger.approvals().get_num_approvals() as f64, + "Total number of approvals.", + )?; + } let pre_upgrade_instructions = PRE_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow()); let post_upgrade_instructions = POST_UPGRADE_INSTRUCTIONS_CONSUMED.with(|n| *n.borrow()); w.encode_gauge( @@ -1450,6 +1538,11 @@ fn encode_metrics(w: &mut ic_metrics_encoder::MetricsEncoder>) -> std::i pre_upgrade_instructions.saturating_add(post_upgrade_instructions) as f64, "Total number of instructions consumed during the last upgrade.", )?; + w.encode_counter( + "ledger_stable_upgrade_migration_steps", + STABLE_UPGRADE_MIGRATION_STEPS.with(|n| *n.borrow()) as f64, + "Number of steps used to migrate data to stable structures.", + )?; Ok(()) } @@ -1501,6 +1594,7 @@ fn query_encoded_blocks_() { #[candid_method(update, rename = "icrc2_approve")] async fn icrc2_approve(arg: ApproveArgs) -> Result { + panic_if_not_ready(); if !LEDGER.read().unwrap().feature_flags.icrc2 { trap_with("ICRC-2 features are not enabled on the ledger."); } @@ -1537,7 +1631,7 @@ async fn icrc2_approve(arg: ApproveArgs) -> Result { let current_allowance = LEDGER .read() .unwrap() - .approvals + .approvals() .allowance(&from, &spender, now) .amount; return Err(ApproveError::AllowanceChanged { @@ -1594,6 +1688,7 @@ async fn icrc2_approve(arg: ApproveArgs) -> Result { #[export_name = "canister_update icrc2_approve"] fn icrc2_approve_candid() { + panic_if_not_ready(); over_async_may_reject(candid_one, |arg: ApproveArgs| async { if !LEDGER.read().unwrap().can_send(&caller()) { return Err( @@ -1608,7 +1703,7 @@ fn icrc2_approve_candid() { fn get_allowance(from: AccountIdentifier, spender: AccountIdentifier) -> Allowance { let now = TimeStamp::from_nanos_since_unix_epoch(time_nanos()); let ledger = LEDGER.read().unwrap(); - let allowance = ledger.approvals.allowance(&from, &spender, now); + let allowance = ledger.approvals().allowance(&from, &spender, now); Allowance { allowance: Nat::from(allowance.amount.get_e8s()), expires_at: allowance.expires_at.map(|t| t.as_nanos_since_unix_epoch()), @@ -1675,6 +1770,16 @@ fn icrc10_supported_standards_candid() { over(candid_one, |()| icrc10_supported_standards()) } +#[candid_method(query, rename = "is_ledger_ready")] +fn is_ledger_ready() -> bool { + is_ready() +} + +#[export_name = "canister_query is_ledger_ready"] +fn is_ledger_ready_candid() { + over(candid_one, |()| is_ledger_ready()) +} + candid::export_service!(); #[export_name = "canister_query __get_candid_interface_tmp_hack"] diff --git a/rs/ledger_suite/icp/ledger/src/tests.rs b/rs/ledger_suite/icp/ledger/src/tests.rs index d4838e9ce4e..c5fe4fd4a70 100644 --- a/rs/ledger_suite/icp/ledger/src/tests.rs +++ b/rs/ledger_suite/icp/ledger/src/tests.rs @@ -1,4 +1,4 @@ -use crate::{AccountIdentifier, Ledger}; +use crate::{AccountIdentifier, Ledger, StorableAllowance}; use ic_base_types::{CanisterId, PrincipalId}; use ic_ledger_canister_core::{ archive::Archive, @@ -11,10 +11,13 @@ use ic_ledger_core::{ timestamp::TimeStamp, tokens::{CheckedAdd, CheckedSub, Tokens}, }; +use ic_stable_structures::Storable; use icp_ledger::{ apply_operation, ArchiveOptions, Block, LedgerBalances, Memo, Operation, PaymentError, Transaction, TransferError, DEFAULT_TRANSFER_FEE, }; +use proptest::prelude::{any, prop_assert_eq, proptest}; +use proptest::strategy::Strategy; use std::collections::HashSet; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime}; @@ -887,7 +890,7 @@ fn test_approvals_are_not_cumulative() { Allowance { amount: approved_amount, expires_at: None, - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -915,7 +918,7 @@ fn test_approvals_are_not_cumulative() { Allowance { amount: new_allowance, expires_at: Some(expiration), - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), } ); } @@ -988,7 +991,7 @@ fn test_approval_transfer_from() { Allowance { amount: tokens(40_000), expires_at: None, - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -1015,7 +1018,7 @@ fn test_approval_transfer_from() { Allowance { amount: tokens(40_000), expires_at: None, - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); assert_eq!(ctx.balances().account_balance(&from), tokens(80_000),); @@ -1048,7 +1051,7 @@ fn test_approval_expiration_override() { Allowance { amount: tokens(100_000), expires_at: Some(ts(2000)), - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -1059,7 +1062,7 @@ fn test_approval_expiration_override() { Allowance { amount: tokens(200_000), expires_at: Some(ts(1500)), - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -1070,7 +1073,7 @@ fn test_approval_expiration_override() { Allowance { amount: tokens(300_000), expires_at: Some(ts(2500)), - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -1085,7 +1088,7 @@ fn test_approval_expiration_override() { Allowance { amount: tokens(300_000), expires_at: Some(ts(2500)), - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); } @@ -1339,7 +1342,7 @@ fn test_approval_burn_from() { Allowance { amount: tokens(50_000), expires_at: None, - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); @@ -1364,10 +1367,43 @@ fn test_approval_burn_from() { Allowance { amount: tokens(50_000), expires_at: None, - arrived_at: now, + arrived_at: TimeStamp::from_nanos_since_unix_epoch(0), }, ); assert_eq!(ctx.balances().account_balance(&from), tokens(90_000)); assert_eq!(ctx.balances().account_balance(&spender), Tokens::ZERO); assert_eq!(ctx.balances().total_supply().get_e8s(), 90_000); } + +#[test] +fn allowance_serialization() { + fn arb_token() -> impl Strategy { + any::().prop_map(tokens) + } + + fn arb_timestamp() -> impl Strategy { + any::().prop_map(TimeStamp::from_nanos_since_unix_epoch) + } + fn arb_opt_expiration() -> impl Strategy> { + proptest::option::of(any::().prop_map(TimeStamp::from_nanos_since_unix_epoch)) + } + fn arb_allowance() -> impl Strategy> { + (arb_token(), arb_opt_expiration(), arb_timestamp()).prop_map( + |(amount, expires_at, arrived_at)| Allowance { + amount, + expires_at, + arrived_at, + }, + ) + } + proptest!(|(allowance in arb_allowance())| { + let storable_allowance: StorableAllowance = allowance.clone().into(); + let new_allowance: Allowance = StorableAllowance::from_bytes(storable_allowance.to_bytes()).into(); + prop_assert_eq!(new_allowance.amount, allowance.amount); + prop_assert_eq!(new_allowance.expires_at, allowance.expires_at); + prop_assert_eq!( + new_allowance.arrived_at, + TimeStamp::from_nanos_since_unix_epoch(0) + ); + }) +} diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs index 94db23d807e..fbbd48ce829 100644 --- a/rs/ledger_suite/icp/ledger/tests/tests.rs +++ b/rs/ledger_suite/icp/ledger/tests/tests.rs @@ -18,8 +18,8 @@ use icp_ledger::{ CandidOperation, CandidTransaction, FeatureFlags, GetBlocksArgs, GetBlocksRes, GetBlocksResult, GetEncodedBlocksResult, IcpAllowanceArgs, InitArgs, IterBlocksArgs, IterBlocksRes, LedgerCanisterInitPayload, LedgerCanisterPayload, LedgerCanisterUpgradePayload, Operation, - QueryBlocksResponse, QueryEncodedBlocksResponse, TimeStamp, UpgradeArgs, DEFAULT_TRANSFER_FEE, - MAX_BLOCKS_PER_INGRESS_REPLICATED_QUERY_REQUEST, MAX_BLOCKS_PER_REQUEST, + QueryBlocksResponse, QueryEncodedBlocksResponse, SendArgs, TimeStamp, UpgradeArgs, + DEFAULT_TRANSFER_FEE, MAX_BLOCKS_PER_INGRESS_REPLICATED_QUERY_REQUEST, MAX_BLOCKS_PER_REQUEST, }; use icrc_ledger_types::icrc1::{ account::Account, @@ -30,7 +30,7 @@ use icrc_ledger_types::icrc2::approve::ApproveArgs; use num_traits::cast::ToPrimitive; use on_wire::{FromWire, IntoWire}; use serde_bytes::ByteBuf; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, SystemTime}; @@ -66,6 +66,14 @@ fn ledger_wasm_allowance_getter() -> Vec { ) } +fn ledger_wasm_low_instruction_limits() -> Vec { + ic_test_utilities_load_wasm::load_wasm( + std::env::var("CARGO_MANIFEST_DIR").unwrap(), + "ledger-canister-low-limits", + &[], + ) +} + fn encode_init_args( args: ic_ledger_suite_state_machine_tests::InitArgs, ) -> LedgerCanisterInitPayload { @@ -1246,96 +1254,90 @@ fn test_upgrade_serialization() { upgrade_args, minter, false, - false, ); } #[test] -fn test_upgrade_serialization_fixed_tx() { - let ledger_wasm_mainnet = ledger_wasm_mainnet(); - let ledger_wasm_current = ledger_wasm(); - - let p1 = PrincipalId::new_user_test_id(1); - let p2 = PrincipalId::new_user_test_id(2); - let p3 = PrincipalId::new_user_test_id(3); - let accounts = vec![ - Account::from(p1.0), - Account::from(p2.0), - Account::from(p3.0), - ]; - - let env = StateMachine::new(); - let mut initial_balances = HashMap::new(); - for account in &accounts { - initial_balances.insert((*account).into(), Tokens::from_e8s(10_000_000)); - } +fn test_multi_step_migration() { + ic_ledger_suite_state_machine_tests::icrc1_test_multi_step_migration( + ledger_wasm_mainnet(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} - let payload = LedgerCanisterInitPayload::builder() - .minting_account(MINTER.into()) - .icrc1_minting_account(MINTER) - .initial_values(initial_balances) - .transfer_fee(Tokens::from_e8s(10_000)) - .token_symbol_and_name("ICP", "Internet Computer") - .build() - .unwrap(); - let canister_id = env - .install_canister( - ledger_wasm_mainnet.clone(), - CandidOne(payload).into_bytes().unwrap(), - None, - ) - .expect("Unable to install the Ledger canister with the new init"); +#[test] +fn test_downgrade_from_incompatible_version() { + ic_ledger_suite_state_machine_tests::test_downgrade_from_incompatible_version( + ledger_wasm_mainnet(), + ledger_wasm_next_version(), + ledger_wasm(), + encode_init_args, + false, + ); +} - let approve_args = default_approve_args(p2.0, 120_000); - send_approval(&env, canister_id, p1.0, &approve_args).expect("approval failed"); - let mut approve_args = default_approve_args(p3.0, 130_000); - let expiration = - system_time_to_nanos(env.time()) + Duration::from_secs(5 * 3600).as_nanos() as u64; - approve_args.expires_at = Some(expiration); - send_approval(&env, canister_id, p1.0, &approve_args).expect("approval failed"); - - let mut balances = BTreeMap::new(); - for account in &accounts { - balances.insert(account, balance_of(&env, canister_id, *account)); - } +#[test] +fn test_stable_migration_endpoints_disabled() { + let send_args = SendArgs { + memo: icp_ledger::Memo::default(), + amount: Tokens::from_e8s(1), + fee: Tokens::from_e8s(10_000), + from_subaccount: None, + to: PrincipalId::new_user_test_id(2).into(), + created_at_time: None, + }; - let test_upgrade = |ledger_wasm: Vec| { - env.upgrade_canister( - canister_id, - ledger_wasm, - Encode!(&LedgerCanisterPayload::Upgrade(None)).unwrap(), - ) - .unwrap(); + let send_dfx_args = Encode!(&send_args).unwrap(); + let send_pb_args = ProtoBuf(send_args).into_bytes().unwrap(); - let allowance = Account::get_allowance(&env, canister_id, p1.0, p2.0); - assert_eq!(allowance.allowance.0.to_u64().unwrap(), 120_000); - assert_eq!(allowance.expires_at, None); + let ai = AccountIdentifier { hash: [1u8; 28] }; + let transfer_args = Encode!(&icp_ledger::TransferArgs { + memo: icp_ledger::Memo::default(), + amount: Tokens::from_e8s(1), + fee: Tokens::from_e8s(10_000), + from_subaccount: None, + to: ai.to_address(), + created_at_time: None, + }) + .unwrap(); - let allowance = Account::get_allowance(&env, canister_id, p1.0, p3.0); - assert_eq!(allowance.allowance.0.to_u64().unwrap(), 130_000); - assert_eq!(allowance.expires_at, Some(expiration)); + ic_ledger_suite_state_machine_tests::icrc1_test_stable_migration_endpoints_disabled( + ledger_wasm_mainnet(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + vec![ + ("send_pb", send_pb_args), + ("send_dfx", send_dfx_args), + ("transfer", transfer_args), + ], + ); +} - for account in &accounts { - assert_eq!(balances[account], balance_of(&env, canister_id, *account)); - } - }; +#[test] +fn test_incomplete_migration() { + ic_ledger_suite_state_machine_tests::test_incomplete_migration( + ledger_wasm_mainnet(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); +} - // Test if the old serialized approvals and balances are correctly deserialized - test_upgrade(ledger_wasm_current.clone()); - // Test the new wasm serialization - test_upgrade(ledger_wasm_current); - // Test if downgrade works - test_upgrade(ledger_wasm_mainnet); +#[test] +fn test_incomplete_migration_to_current() { + ic_ledger_suite_state_machine_tests::test_incomplete_migration_to_current( + ledger_wasm_mainnet(), + ledger_wasm_low_instruction_limits(), + encode_init_args, + ); } #[test] -fn test_downgrade_from_incompatible_version() { - ic_ledger_suite_state_machine_tests::test_downgrade_from_incompatible_version( +fn test_metrics_while_migrating() { + ic_ledger_suite_state_machine_tests::test_metrics_while_migrating( ledger_wasm_mainnet(), - ledger_wasm_next_version(), - ledger_wasm(), + ledger_wasm_low_instruction_limits(), encode_init_args, - true, ); } @@ -1507,15 +1509,6 @@ fn test_balances_overflow() { ic_ledger_suite_state_machine_tests::test_balances_overflow(ledger_wasm(), encode_init_args); } -#[test] -fn test_approval_trimming() { - ic_ledger_suite_state_machine_tests::test_approval_trimming( - ledger_wasm(), - encode_init_args, - true, - ); -} - #[test] fn account_identifier_test() { let env = StateMachine::new(); diff --git a/rs/ledger_suite/icp/src/account_identifier.rs b/rs/ledger_suite/icp/src/account_identifier.rs index 3698f909f82..ed50f86cc9c 100644 --- a/rs/ledger_suite/icp/src/account_identifier.rs +++ b/rs/ledger_suite/icp/src/account_identifier.rs @@ -2,8 +2,10 @@ use candid::{CandidType, Principal}; use dfn_core::CanisterId; use ic_base_types::{CanisterIdError, PrincipalId, PrincipalIdError}; use ic_crypto_sha2::Sha224; +use ic_stable_structures::{storable::Bound, Storable}; use icrc_ledger_types::icrc1::account::Account; use serde::{de, de::Error, Deserialize, Serialize}; +use std::borrow::Cow; use std::{ convert::{TryFrom, TryInto}, fmt::{Display, Formatter}, @@ -51,6 +53,25 @@ impl From for AccountIdentifier { } } +impl Storable for AccountIdentifier { + fn to_bytes(&self) -> Cow<[u8]> { + let mut buffer: Vec = vec![]; + buffer.extend(self.hash.as_slice()); + Cow::Owned(buffer) + } + + fn from_bytes(bytes: Cow<[u8]>) -> Self { + AccountIdentifier { + hash: bytes[0..28].try_into().unwrap(), + } + } + + const BOUND: Bound = Bound::Bounded { + max_size: 28, + is_fixed_size: true, + }; +} + pub static SUB_ACCOUNT_ZERO: Subaccount = Subaccount([0; 32]); static ACCOUNT_DOMAIN_SEPARATOR: &[u8] = b"\x0Aaccount-id"; @@ -492,3 +513,10 @@ fn test_account_id_from_hex() { Ok(AccountIdentifier { hash: [0; 28] }) ); } + +#[test] +fn test_account_id_serialization() { + let length_56 = "00000000000000000000000000000000000000000000000000000012"; + let id = AccountIdentifier::from_hex(length_56).expect("failed to generate account"); + assert_eq!(AccountIdentifier::from_bytes(id.to_bytes()), id); +} diff --git a/rs/ledger_suite/icp/src/lib.rs b/rs/ledger_suite/icp/src/lib.rs index 935cb21981b..bb614749a81 100644 --- a/rs/ledger_suite/icp/src/lib.rs +++ b/rs/ledger_suite/icp/src/lib.rs @@ -1243,6 +1243,7 @@ pub fn max_blocks_per_request(principal_id: &PrincipalId) -> usize { #[cfg(test)] mod test { + use ic_stable_structures::storable::Storable; use std::str::FromStr; use proptest::{arbitrary::any, prop_assert_eq, prop_oneof, proptest, strategy::Strategy}; @@ -1406,4 +1407,11 @@ mod test { prop_assert_eq!(block, decoded) }) } + + #[test] + fn test_storable_serialization() { + proptest!(|(a in arb_account())| { + prop_assert_eq!(AccountIdentifier::from_bytes(a.to_bytes()), a) + }) + } } diff --git a/rs/ledger_suite/icp/tests/upgrade_downgrade.rs b/rs/ledger_suite/icp/tests/upgrade_downgrade.rs index 9edfa85b3e1..46ec2e24d3e 100644 --- a/rs/ledger_suite/icp/tests/upgrade_downgrade.rs +++ b/rs/ledger_suite/icp/tests/upgrade_downgrade.rs @@ -22,6 +22,7 @@ use icp_ledger::{ LedgerCanisterUpgradePayload, Memo, Subaccount, TransferArgs, DEFAULT_TRANSFER_FEE, }; use maplit::hashmap; +use pocket_ic::CallError; use pocket_ic::{PocketIc, PocketIcBuilder}; use std::time::Duration; @@ -141,26 +142,42 @@ impl Setup { } } - fn upgrade_ledger_canister(&self, upgrade_to_version: UpgradeToVersion) { + fn upgrade_ledger_canister(&self, upgrade_to_version: UpgradeToVersion, should_succeed: bool) { let ledger_wasm = match upgrade_to_version { UpgradeToVersion::MainNet => build_mainnet_ledger_wasm(), UpgradeToVersion::Latest => build_ledger_wasm(), }; let ledger_upgrade_args = LedgerCanisterUpgradePayload::builder().build().unwrap(); let canister_id = candid::Principal::from(LEDGER_CANISTER_ID); - self.pocket_ic - .upgrade_canister( - canister_id, - ledger_wasm.bytes(), - Encode!(&ledger_upgrade_args).unwrap(), - None, - ) - .unwrap(); + match self.pocket_ic.upgrade_canister( + canister_id, + ledger_wasm.bytes(), + Encode!(&ledger_upgrade_args).unwrap(), + None, + ) { + Ok(_) => { + if !should_succeed { + panic!("Upgrade should fail!"); + } + } + Err(e) => { + if should_succeed { + panic!("Upgrade should succeed!"); + } else { + match e { + CallError::Reject(_) => panic!("Expected UserError!"), + CallError::UserError(user_error) => assert!(user_error + .description + .contains("Trying to downgrade from incompatible version")), + }; + } + } + }; let expected_module_hash = mainnet_ledger_canister_sha256sum(); self.assert_canister_module_hash( canister_id, &expected_module_hash, - upgrade_to_version == UpgradeToVersion::MainNet, + upgrade_to_version == UpgradeToVersion::MainNet && should_succeed, ); } @@ -428,13 +445,13 @@ fn should_upgrade_and_downgrade_canister_suite() { setup.create_icp_transfers_until_archive_is_spawned(); setup.upgrade_index_canister(UpgradeToVersion::Latest); - setup.upgrade_ledger_canister(UpgradeToVersion::Latest); + setup.upgrade_ledger_canister(UpgradeToVersion::Latest, true); setup.upgrade_archive_canisters(UpgradeToVersion::Latest); setup.assert_index_ledger_parity(true); setup.upgrade_index_canister(UpgradeToVersion::MainNet); - setup.upgrade_ledger_canister(UpgradeToVersion::MainNet); + setup.upgrade_ledger_canister(UpgradeToVersion::MainNet, false); setup.upgrade_archive_canisters(UpgradeToVersion::MainNet); setup.assert_index_ledger_parity(true); diff --git a/rs/ledger_suite/icrc1/ledger/src/lib.rs b/rs/ledger_suite/icrc1/ledger/src/lib.rs index f08190eb7e3..c1c5839db3e 100644 --- a/rs/ledger_suite/icrc1/ledger/src/lib.rs +++ b/rs/ledger_suite/icrc1/ledger/src/lib.rs @@ -1211,33 +1211,12 @@ impl AllowancesData for StableAllowancesData { }); } - fn insert_arrival( - &mut self, - _timestamp: TimeStamp, - _account_spender: (Self::AccountId, Self::AccountId), - ) { - // We do not store arrivals in stable structures. - } - - fn remove_arrival( - &mut self, - _timestamp: TimeStamp, - _account_spender: (Self::AccountId, Self::AccountId), - ) { - // We do not store arrivals in stable structures. - } - fn first_expiry(&self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { let result = ALLOWANCES_EXPIRATIONS_MEMORY .with_borrow(|expirations| expirations.first_key_value().map(|kv| kv.0)); result.map(|e| (e.timestamp, e.account_spender.into())) } - fn oldest_arrivals(&self, _n: usize) -> Vec<(Self::AccountId, Self::AccountId)> { - // We do not store arrivals in stable structures. - vec![] - } - fn pop_first_expiry(&mut self) -> Option<(TimeStamp, (Self::AccountId, Self::AccountId))> { let result = ALLOWANCES_EXPIRATIONS_MEMORY .with_borrow_mut(|expirations| expirations.pop_first().map(|kv| kv.0)); @@ -1264,11 +1243,6 @@ impl AllowancesData for StableAllowancesData { .unwrap() } - fn len_arrivals(&self) -> usize { - // We do not store arrivals in stable structures. - 0 - } - fn clear_arrivals(&mut self) { panic!("The method `clear_arrivals` should not be called for StableAllowancesData") } diff --git a/rs/ledger_suite/icrc1/ledger/tests/tests.rs b/rs/ledger_suite/icrc1/ledger/tests/tests.rs index a098a08da8b..5277179ae8a 100644 --- a/rs/ledger_suite/icrc1/ledger/tests/tests.rs +++ b/rs/ledger_suite/icrc1/ledger/tests/tests.rs @@ -395,15 +395,6 @@ fn test_balances_overflow() { ic_ledger_suite_state_machine_tests::test_balances_overflow(ledger_wasm(), encode_init_args); } -#[test] -fn test_approval_trimming() { - ic_ledger_suite_state_machine_tests::test_approval_trimming( - ledger_wasm(), - encode_init_args, - false, - ); -} - #[test] fn test_archive_controllers() { ic_ledger_suite_state_machine_tests::test_archive_controllers(ledger_wasm()); @@ -461,7 +452,6 @@ fn icrc1_test_upgrade_serialization() { upgrade_args, minter, true, - true, ); } @@ -491,6 +481,7 @@ fn icrc1_test_stable_migration_endpoints_disabled() { ledger_mainnet_wasm(), ledger_wasm_lowupgradeinstructionlimits(), encode_init_args, + vec![], ); } diff --git a/rs/ledger_suite/tests/sm-tests/src/lib.rs b/rs/ledger_suite/tests/sm-tests/src/lib.rs index 357a6fd26db..f76b277f72e 100644 --- a/rs/ledger_suite/tests/sm-tests/src/lib.rs +++ b/rs/ledger_suite/tests/sm-tests/src/lib.rs @@ -57,7 +57,6 @@ use proptest::test_runner::{Config as TestRunnerConfig, TestCaseResult, TestRunn use std::sync::Arc; use std::time::{Instant, UNIX_EPOCH}; use std::{ - cmp, collections::{BTreeMap, HashMap}, time::{Duration, SystemTime}, }; @@ -2414,7 +2413,6 @@ pub fn test_upgrade_serialization( upgrade_args: Vec, minter: Arc, verify_blocks: bool, - migration_to_stable_structures: bool, ) where Tokens: TokensType + Default + std::fmt::Display + From, { @@ -2464,12 +2462,10 @@ pub fn test_upgrade_serialization( let mut test_upgrade = |ledger_wasm: Vec, expected_migration_steps: u64| { env.upgrade_canister(ledger_id, ledger_wasm, upgrade_args.clone()) .unwrap(); - if migration_to_stable_structures { - wait_ledger_ready(&env, ledger_id, 10); - let stable_upgrade_migration_steps = - parse_metric(&env, ledger_id, "ledger_stable_upgrade_migration_steps"); - assert_eq!(stable_upgrade_migration_steps, expected_migration_steps); - } + wait_ledger_ready(&env, ledger_id, 10); + let stable_upgrade_migration_steps = + parse_metric(&env, ledger_id, "ledger_stable_upgrade_migration_steps"); + assert_eq!(stable_upgrade_migration_steps, expected_migration_steps); add_tx_and_verify(); }; @@ -2479,26 +2475,21 @@ pub fn test_upgrade_serialization( test_upgrade(ledger_wasm_current.clone(), 0); // Test deserializing from memory manager test_upgrade(ledger_wasm_current.clone(), 0); - if !migration_to_stable_structures { - // Test downgrade to mainnet wasm - test_upgrade(ledger_wasm_mainnet.clone(), 0); - } else { - // Downgrade from stable structures to mainnet not possible. - match env.upgrade_canister( - ledger_id, - ledger_wasm_mainnet.clone(), - Encode!(&LedgerArgument::Upgrade(None)).unwrap(), - ) { - Ok(_) => { - panic!("Upgrade from future ledger version should fail!") - } - Err(e) => { - assert!(e - .description() - .contains("Trying to downgrade from incompatible version")) - } - }; - } + // Downgrade from stable structures to mainnet not possible. + match env.upgrade_canister( + ledger_id, + ledger_wasm_mainnet.clone(), + Encode!(&LedgerArgument::Upgrade(None)).unwrap(), + ) { + Ok(_) => { + panic!("Upgrade from future ledger version should fail!") + } + Err(e) => { + assert!(e + .description() + .contains("Trying to downgrade from incompatible version")) + } + }; if verify_blocks { // This will also verify the ledger blocks. // The current implementation of the InMemoryLedger cannot get blocks @@ -2745,6 +2736,7 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( ledger_wasm_mainnet: Vec, ledger_wasm_current_lowinstructionlimits: Vec, encode_init_args: fn(InitArgs) -> T, + additional_endpoints: Vec<(&str, Vec)>, ) where T: CandidType, { @@ -2817,6 +2809,9 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), true); test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), true); test_endpoint("icrc1_total_supply", Encode!().unwrap(), true); + for (endpoint_name, args) in additional_endpoints.clone() { + test_endpoint(endpoint_name, args, true); + } wait_ledger_ready(&env, canister_id, 10); @@ -2830,6 +2825,9 @@ pub fn icrc1_test_stable_migration_endpoints_disabled( test_endpoint("icrc2_allowance", Encode!(&allowance_args).unwrap(), false); test_endpoint("icrc1_balance_of", Encode!(&account).unwrap(), false); test_endpoint("icrc1_total_supply", Encode!().unwrap(), false); + for (endpoint_name, args) in additional_endpoints { + test_endpoint(endpoint_name, args, false); + } } pub fn test_incomplete_migration( @@ -3149,8 +3147,8 @@ pub fn test_metrics_while_migrating( assert!( !metrics .iter() - .any(|line| line.contains("ledger_total_supply")), - "ledger_total_supply should not be in metrics" + .any(|line| line.contains("ledger_num_approvals")), + "ledger_num_approvals should not be in metrics" ); let is_ledger_ready = Decode!( @@ -3174,8 +3172,8 @@ pub fn test_metrics_while_migrating( assert!( metrics .iter() - .any(|line| line.contains("ledger_total_supply")), - "Did not find ledger_total_supply metric" + .any(|line| line.contains("ledger_num_approvals")), + "Did not find ledger_num_approvals metric" ); } @@ -3909,108 +3907,6 @@ where assert_eq!(total_supply(&env, canister_id), credited - 1 - 2); } -pub fn test_approval_trimming( - ledger_wasm: Vec, - encode_init_args: fn(InitArgs) -> T, - trimming_enabled: bool, -) where - T: CandidType, -{ - let env = StateMachine::new(); - - let args = encode_init_args(InitArgs { - feature_flags: Some(FeatureFlags { icrc2: true }), - maximum_number_of_accounts: Some(9), - accounts_overflow_trim_quantity: Some(2), - ..init_args(vec![]) - }); - let args = Encode!(&args).unwrap(); - let canister_id = env.install_canister(ledger_wasm, args, None).unwrap(); - - let minter = minting_account(&env, canister_id).unwrap(); - - for i in 0..4 { - transfer( - &env, - canister_id, - minter, - PrincipalId::new_user_test_id(i).0, - 1_000_000, - ) - .expect("failed to mint tokens"); - } - - let num_approvals = 3; - for i in 0..num_approvals { - let mut approve_args = default_approve_args(PrincipalId::new_user_test_id(i).0, 10_000); - if i < 2 { - approve_args.expires_at = Some( - system_time_to_nanos(env.time()) - + Duration::from_secs((i + 1) * 3600).as_nanos() as u64, - ); - } - send_approval( - &env, - canister_id, - PrincipalId::new_user_test_id(3).0, - &approve_args, - ) - .expect("approval failed"); - } - - for i in 0..4 { - assert_ne!( - balance_of(&env, canister_id, PrincipalId::new_user_test_id(i).0), - 0 - ); - } - - fn total_allowance(env: &StateMachine, canister_id: CanisterId, num_approvals: u64) -> Nat { - let mut allowance = Nat::from(0_u8); - for i in 0..num_approvals { - allowance += Account::get_allowance( - env, - canister_id, - PrincipalId::new_user_test_id(3).0, - PrincipalId::new_user_test_id(i).0, - ) - .allowance; - } - allowance - } - - assert_eq!( - total_allowance(&env, canister_id, num_approvals), - Nat::from(30_000u32) - ); - - let mut new_accounts = 0; - for i in 4..11 { - transfer( - &env, - canister_id, - minter, - PrincipalId::new_user_test_id(i).0, - 1_000_000, - ) - .expect("failed to mint tokens"); - new_accounts += 1; - - let remaining_approvals = if trimming_enabled { - cmp::max(num_approvals as i64 - (new_accounts + 1) / 2, 0) as u64 - } else { - // The ICRC ledger does not trim approvals. We still want to run - // this test to make sure the trimming code does not cause panic, etc. - // Once ICP ledger approvals are not trimmed, this test will be removed entirely. - num_approvals - }; - assert_eq!( - total_allowance(&env, canister_id, num_approvals), - Nat::from(10_000 * remaining_approvals) - ); - } -} - pub fn test_icrc1_test_suite( ledger_wasm: Vec, encode_init_args: fn(InitArgs) -> T, diff --git a/rs/nns/inspector/BUILD.bazel b/rs/nns/inspector/BUILD.bazel index 0ad7574ed85..531724a2940 100644 --- a/rs/nns/inspector/BUILD.bazel +++ b/rs/nns/inspector/BUILD.bazel @@ -4,6 +4,7 @@ package(default_visibility = ["//visibility:public"]) DEPENDENCIES = [ # Keep sorted. + "//rs/ledger_suite/common/ledger_canister_core", "//rs/ledger_suite/icp:icp_ledger", "//rs/ledger_suite/icp/ledger", "//rs/nns/constants", diff --git a/rs/nns/inspector/Cargo.toml b/rs/nns/inspector/Cargo.toml index 8d38ba9a6d9..83ceb74fc7a 100644 --- a/rs/nns/inspector/Cargo.toml +++ b/rs/nns/inspector/Cargo.toml @@ -9,6 +9,7 @@ clap = { workspace = true } csv = "1.1" hex = { workspace = true } ic-base-types = { path = "../../types/base_types" } +ic-ledger-canister-core = { path = "../../ledger_suite/common/ledger_canister_core" } ic-nns-constants = { path = "../constants" } ic-nns-governance-api = { path = "../governance/api" } ic-nns-gtc = { path = "../gtc" } diff --git a/rs/nns/inspector/src/main.rs b/rs/nns/inspector/src/main.rs index 18e40854ee4..8c77da9169a 100644 --- a/rs/nns/inspector/src/main.rs +++ b/rs/nns/inspector/src/main.rs @@ -2,6 +2,7 @@ use clap::Parser; use ic_base_types::CanisterId; +use ic_ledger_canister_core::ledger::LedgerContext; use ic_nns_constants::{ CYCLES_MINTING_CANISTER_ID, GENESIS_TOKEN_CANISTER_ID, GOVERNANCE_CANISTER_ID, LEDGER_CANISTER_ID, REGISTRY_CANISTER_ID, @@ -328,7 +329,7 @@ fn decode_ledger_stable_memory(cbor: PathBuf, output: &Path) { Ok(l) => l, }; let mut records: Vec = ledger - .balances + .balances() .store .iter() .map(|(key, icpts)| LedgerBalanceRecord {