feat: ICRC-ledger: FI-1439: Implement V4 for ICRC ledger - migrate balances to stable structures

WORKSPACE.bazel

@@ -26,9 +26,13 @@ canisters(
         "sns-wasm": "sns-wasm-canister.wasm.gz",
         "ck_btc_archive": "ic-icrc1-archive.wasm.gz",
         "ck_btc_ledger": "ic-icrc1-ledger.wasm.gz",
+        "ck_btc_ledger_v1": "ic-icrc1-ledger.wasm.gz",
+        "ck_btc_ledger_v2": "ic-icrc1-ledger.wasm.gz",
         "ck_btc_index": "ic-icrc1-index-ng.wasm.gz",
         "ck_eth_archive": "ic-icrc1-archive-u256.wasm.gz",
         "ck_eth_ledger": "ic-icrc1-ledger-u256.wasm.gz",
+        "ck_eth_ledger_v1": "ic-icrc1-ledger-u256.wasm.gz",
+        "ck_eth_ledger_v2": "ic-icrc1-ledger-u256.wasm.gz",
         "ck_eth_index": "ic-icrc1-index-ng-u256.wasm.gz",
         "sns_root": "sns-root-canister.wasm.gz",
         "sns_governance": "sns-governance-canister.wasm.gz",
@@ -52,9 +56,13 @@ canisters(
         "sns-wasm": "mainnet_nns_sns-wasm-canister",
         "ck_btc_archive": "mainnet_ckbtc_ic-icrc1-archive",
         "ck_btc_ledger": "mainnet_ckbtc_ic-icrc1-ledger",
+        "ck_btc_ledger_v1": "mainnet_ckbtc_ic-icrc1-ledger-v1",
+        "ck_btc_ledger_v2": "mainnet_ckbtc_ic-icrc1-ledger-v2",
         "ck_btc_index": "mainnet_ckbtc-index-ng",
         "ck_eth_archive": "mainnet_cketh_ic-icrc1-archive-u256",
         "ck_eth_ledger": "mainnet_cketh_ic-icrc1-ledger-u256",
+        "ck_eth_ledger_v1": "mainnet_cketh_ic-icrc1-ledger-u256-v1",
+        "ck_eth_ledger_v2": "mainnet_cketh_ic-icrc1-ledger-u256-v2",
         "ck_eth_index": "mainnet_cketh-index-ng",
         "sns_root": "mainnet_sns-root-canister",
         "sns_governance": "mainnet_sns-governance-canister",

mainnet-canister-revisions.json

@@ -15,6 +15,14 @@
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "25071c2c55ad4571293e00d8e277f442aec7aed88109743ac52df3125209ff45"
   },
+  "ck_btc_ledger_v1": {
+    "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31",
+    "sha256": "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"
+  },
+  "ck_btc_ledger_v2": {
+    "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743",
+    "sha256": "3d808fa63a3d8ebd4510c0400aa078e99a31afaa0515f0b68778f929ce4b2a46"
+  },
   "ck_eth_archive": {
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "2d25f7831894100d48aa9043c65e87c293487523f0958c15760027d004fbbda9"
@@ -27,6 +35,14 @@
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "9637743e1215a4db376a62ee807a0986faf20833be2b332df09b3d5dbdd7339e"
   },
+  "ck_eth_ledger_v1": {
+    "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31",
+    "sha256": "e6072806ae22868ee09c07923d093b1b0b687dba540d22cfc1e1a5392bfcca46"
+  },
+  "ck_eth_ledger_v2": {
+    "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743",
+    "sha256": "98a7b7391608dc4a554d6964bad24157b6aaf890a05bbaad3fcc92033d9c7b02"
+  },
   "cycles-minting": {
     "rev": "ee52ab3056cf5f39b09b08de70bdd20485c8b2dc",
     "sha256": "bbb8995cb749ba9e2c721ff507f5e5313f32e69b1adf3df20e3901ed56a70b42"

rs/ethereum/ledger-suite-orchestrator/src/scheduler/mod.rs

@@ -932,8 +932,6 @@ fn icrc1_ledger_init_arg(
         ),
         max_memo_length: Some(MAX_MEMO_LENGTH),
         feature_flags: Some(ICRC2_FEATURE),
-        maximum_number_of_accounts: None,
-        accounts_overflow_trim_quantity: None,
     }
 }
 

rs/ledger_suite/common/ledger_canister_core/src/ledger.rs

@@ -208,6 +208,7 @@ pub enum TransferError<Tokens> {
 const APPROVE_PRUNE_LIMIT: usize = 100;
 
 /// Adds a new block with the specified transaction to the ledger.
+/// Trim balances if necessary.
 pub fn apply_transaction<L>(
     ledger: &mut L,
     transaction: L::Transaction,
@@ -217,6 +218,22 @@ pub fn apply_transaction<L>(
 where
     L: LedgerData,
     L::BalancesStore: InspectableBalancesStore,
+{
+    let result = apply_transaction_no_trimming(ledger, transaction, now, effective_fee);
+    trim_balances(ledger, now);
+    result
+}
+
+/// Adds a new block with the specified transaction to the ledger.
+/// Do not perform any balance trimming.
+pub fn apply_transaction_no_trimming<L>(
+    ledger: &mut L,
+    transaction: L::Transaction,
+    now: TimeStamp,
+    effective_fee: L::Tokens,
+) -> Result<(BlockIndex, HashOf<EncodedBlock>), TransferError<L::Tokens>>
+where
+    L: LedgerData,
 {
     let num_pruned = purge_old_transactions(ledger, now);
 
@@ -301,6 +318,16 @@ where
                 transaction_hash: tx_hash,
             });
     }
+
+    Ok((height, ledger.blockchain().last_hash.unwrap()))
+}
+
+/// Trim balances. Can be used e.g. if the ledger is low on heap memory.
+fn trim_balances<L>(ledger: &mut L, now: TimeStamp)
+where
+    L: LedgerData,
+    L::BalancesStore: InspectableBalancesStore,
+{
     let effective_max_number_of_accounts =
         ledger.max_number_of_accounts() + ledger.accounts_overflow_trim_quantity() - 1;
 
@@ -331,8 +358,6 @@ where
             ))
             .unwrap();
     }
-
-    Ok((height, ledger.blockchain().last_hash.unwrap()))
 }
 
 /// Finds the archive canister that contains the block with the specified height.

rs/ledger_suite/icp/ledger/tests/tests.rs

@@ -1250,6 +1250,7 @@ fn test_upgrade_serialization() {
         upgrade_args,
         minter,
         false,
+        false,
     );
 }
 

rs/ledger_suite/icrc1/index-ng/tests/tests.rs

@@ -55,7 +55,6 @@ fn upgrade_ledger(
         change_fee_collector,
         max_memo_length: None,
         feature_flags: None,
-        accounts_overflow_trim_quantity: None,
         change_archive_options: None,
     }));
     env.upgrade_canister(ledger_id, ledger_wasm(), Encode!(&args).unwrap())

rs/ledger_suite/icrc1/ledger/BUILD.bazel

@@ -278,14 +278,22 @@ rust_test(
             "//rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz",
             "//rs/ledger_suite/icrc1/ledger:ledger_canister_icrc3_compatible_data_certificate",
             "//rs/universal_canister/impl:universal_canister.wasm.gz",
+            "@mainnet_ckbtc_ic-icrc1-ledger-v1//file",
+            "@mainnet_ckbtc_ic-icrc1-ledger-v2//file",
             "@mainnet_ckbtc_ic-icrc1-ledger//file",
+            "@mainnet_cketh_ic-icrc1-ledger-u256-v1//file",
+            "@mainnet_cketh_ic-icrc1-ledger-u256-v2//file",
             "@mainnet_cketh_ic-icrc1-ledger-u256//file",
             "@mainnet_ic-icrc1-ledger//file",
         ],
         env = {
             "CARGO_MANIFEST_DIR": "rs/ledger_suite/icrc1/ledger",
             "CKBTC_IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger//file)",
+            "CKBTC_IC_ICRC1_LEDGER_V1_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger-v1//file)",
+            "CKBTC_IC_ICRC1_LEDGER_V2_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger-v2//file)",
             "CKETH_IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256//file)",
+            "CKETH_IC_ICRC1_LEDGER_V1_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256-v1//file)",
+            "CKETH_IC_ICRC1_LEDGER_V2_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256-v2//file)",
             "IC_ICRC1_ARCHIVE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz)",
             "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)",
             "IC_ICRC1_LEDGER_ICRC3_COMPATIBLE_DATA_CERTIFICATE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/ledger:ledger_canister_icrc3_compatible_data_certificate)",

rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml

@@ -1,42 +1,42 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 6398923208
-      heap_increase: 227
+      instructions: 13798815361
+      heap_increase: 172
       stable_memory_increase: 128
     scopes:
       before_upgrade:
-        instructions: 5790054103
-        heap_increase: 71
+        instructions: 13630531777
+        heap_increase: 43
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 424320009
-        heap_increase: 27
+        instructions: 118601062
+        heap_increase: 0
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 184546072
+        instructions: 49679478
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 608867671
-        heap_increase: 156
+        instructions: 168282130
+        heap_increase: 129
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8755062
+      instructions: 8684974
       heap_increase: 258
-      stable_memory_increase: 129
+      stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8627524
+        instructions: 8606422
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 125131
+        instructions: 76145
         heap_increase: 129
-        stable_memory_increase: 129
+        stable_memory_increase: 128
       upgrade:
-        instructions: 8754373
+        instructions: 8684285
         heap_increase: 258
-        stable_memory_increase: 129
+        stable_memory_increase: 128
 version: 0.1.7

rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml

@@ -1,42 +1,42 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 5926738616
-      heap_increase: 214
+      instructions: 13237283790
+      heap_increase: 171
       stable_memory_increase: 128
     scopes:
       before_upgrade:
-        instructions: 5341274673
-        heap_increase: 64
+        instructions: 13068913917
+        heap_increase: 42
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 406888378
-        heap_increase: 21
+        instructions: 118797275
+        heap_increase: 0
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 178572258
+        instructions: 49569466
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 585462226
-        heap_increase: 150
+        instructions: 168368331
+        heap_increase: 129
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8758206
+      instructions: 8686052
       heap_increase: 258
-      stable_memory_increase: 129
+      stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8627876
+        instructions: 8606533
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 127923
+        instructions: 77112
         heap_increase: 129
-        stable_memory_increase: 129
+        stable_memory_increase: 128
       upgrade:
-        instructions: 8757517
+        instructions: 8685363
         heap_increase: 258
-        stable_memory_increase: 129
+        stable_memory_increase: 128
 version: 0.1.7

rs/ledger_suite/icrc1/ledger/ledger.did

@@ -107,8 +107,6 @@ type InitArgs = record {
     metadata : vec record { text; MetadataValue };
     initial_balances : vec record { Account; nat };
     feature_flags : opt FeatureFlags;
-    maximum_number_of_accounts : opt nat64;
-    accounts_overflow_trim_quantity : opt nat64;
     archive_options : record {
         num_blocks_to_archive : nat64;
         max_transactions_per_response : opt nat64;
@@ -144,7 +142,6 @@ type UpgradeArgs = record {
     change_fee_collector : opt ChangeFeeCollector;
     max_memo_length : opt nat16;
     feature_flags : opt FeatureFlags;
-    accounts_overflow_trim_quantity: opt nat64;
     change_archive_options : opt ChangeArchiveOptions;
 };
 

rs/ledger_suite/icrc1/ledger/src/benches/mod.rs

@@ -1,10 +1,8 @@
-use crate::{execute_transfer_not_async, post_upgrade, pre_upgrade, Access, Tokens};
+use crate::{balances_len, execute_transfer_not_async, post_upgrade, pre_upgrade, Tokens};
 use assert_matches::assert_matches;
 use candid::{Nat, Principal};
 use ic_canister_log::Sink;
-use ic_ledger_canister_core::ledger::{
-    blocks_to_archive, remove_archived_blocks, LedgerAccess, LedgerContext,
-};
+use ic_ledger_canister_core::ledger::{blocks_to_archive, remove_archived_blocks, LedgerAccess};
 use ic_ledger_core::block::BlockIndex;
 use icrc_ledger_types::icrc1::account::Account;
 use icrc_ledger_types::icrc1::transfer::TransferArg;
@@ -42,9 +40,7 @@ pub fn icrc1_transfer(
 }
 
 fn assert_has_num_balances(num_balances: u32) {
-    Access::with_ledger(|ledger| {
-        assert_eq!(ledger.balances().store.len() as u32, num_balances);
-    });
+    assert_eq!(balances_len() as u32, num_balances);
 }
 
 pub fn max_length_principal(index: u32) -> Principal {