feat: ICRC-ledger: FI-1439: Implement V4 for ICRC ledger - migrate ba…

…lances to stable structures (#2901) This version of the ledger migrates the balances stored by the ledger from heap to stable memory stable structures. This allows the ledger to store more data (heap memory is limited to 4GB) and reduces the number of instructions used during the upgrades - there is no need to serialize and deserialize the balances stored in heap memory. Since we are no longer limited by the heap size, the logic for trimming balances and related init/upgrade arguments (`maximum_number_of_accounts`, `accounts_overflow_trim_quantity`) were removed. The migration to stable structures is performed during the upgrade and, if 200B instruction limit is reached, using timer invocations after the upgrade. If timer invocations are used by the migration, the ledger is unavailable for transfers and approvals until migration is finished. The migration status can be checked by checking the value of the `ledger_stable_upgrade_migration_steps` metric and calling the `is_ledger_ready` endpoint. The metric will increase at roughly 1s intervals until the migration is finished. The canister logs will also contain some information about the number of migrated allowances and balances (e.g. "Migration partially done. Scheduling the next part. Number of elements migrated: allowances: 0 expirations: 0 balances: 7657..."). If migration is not progressing and the ledger is not ready, the ledger can still be downgraded to the previous version. After migration is finished and `is_ledger_ready` returns true, it is no longer possible to downgrade the ledger to the previous version . --------- Co-authored-by: Mathias Björkqvist <mathias.bjorkqvist@dfinity.org>
dfinity · Jan 7, 2025 · c741e34 · c741e34
1 parent 21c5e93
commit c741e34
Show file tree

Hide file tree

Showing 18 changed files with 494 additions and 168 deletions.
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel
@@ -26,9 +26,13 @@ canisters(
         "sns-wasm": "sns-wasm-canister.wasm.gz",
         "ck_btc_archive": "ic-icrc1-archive.wasm.gz",
         "ck_btc_ledger": "ic-icrc1-ledger.wasm.gz",
+        "ck_btc_ledger_v1": "ic-icrc1-ledger.wasm.gz",
+        "ck_btc_ledger_v2": "ic-icrc1-ledger.wasm.gz",
         "ck_btc_index": "ic-icrc1-index-ng.wasm.gz",
         "ck_eth_archive": "ic-icrc1-archive-u256.wasm.gz",
         "ck_eth_ledger": "ic-icrc1-ledger-u256.wasm.gz",
+        "ck_eth_ledger_v1": "ic-icrc1-ledger-u256.wasm.gz",
+        "ck_eth_ledger_v2": "ic-icrc1-ledger-u256.wasm.gz",
         "ck_eth_index": "ic-icrc1-index-ng-u256.wasm.gz",
         "sns_root": "sns-root-canister.wasm.gz",
         "sns_governance": "sns-governance-canister.wasm.gz",
@@ -52,9 +56,13 @@ canisters(
         "sns-wasm": "mainnet_nns_sns-wasm-canister",
         "ck_btc_archive": "mainnet_ckbtc_ic-icrc1-archive",
         "ck_btc_ledger": "mainnet_ckbtc_ic-icrc1-ledger",
+        "ck_btc_ledger_v1": "mainnet_ckbtc_ic-icrc1-ledger-v1",
+        "ck_btc_ledger_v2": "mainnet_ckbtc_ic-icrc1-ledger-v2",
         "ck_btc_index": "mainnet_ckbtc-index-ng",
         "ck_eth_archive": "mainnet_cketh_ic-icrc1-archive-u256",
         "ck_eth_ledger": "mainnet_cketh_ic-icrc1-ledger-u256",
+        "ck_eth_ledger_v1": "mainnet_cketh_ic-icrc1-ledger-u256-v1",
+        "ck_eth_ledger_v2": "mainnet_cketh_ic-icrc1-ledger-u256-v2",
         "ck_eth_index": "mainnet_cketh-index-ng",
         "sns_root": "mainnet_sns-root-canister",
         "sns_governance": "mainnet_sns-governance-canister",

diff --git a/mainnet-canister-revisions.json b/mainnet-canister-revisions.json
@@ -15,6 +15,14 @@
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "25071c2c55ad4571293e00d8e277f442aec7aed88109743ac52df3125209ff45"
   },
+  "ck_btc_ledger_v1": {
+    "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31",
+    "sha256": "a170bfdce5d66e751a3cc03747cb0f06b450af500e75e15976ec08a3f5691f4c"
+  },
+  "ck_btc_ledger_v2": {
+    "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743",
+    "sha256": "3d808fa63a3d8ebd4510c0400aa078e99a31afaa0515f0b68778f929ce4b2a46"
+  },
   "ck_eth_archive": {
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "2d25f7831894100d48aa9043c65e87c293487523f0958c15760027d004fbbda9"
@@ -27,6 +35,14 @@
     "rev": "2190613d3b5bcd9b74c382b22d151580b8ac271a",
     "sha256": "9637743e1215a4db376a62ee807a0986faf20833be2b332df09b3d5dbdd7339e"
   },
+  "ck_eth_ledger_v1": {
+    "rev": "d4ee25b0865e89d3eaac13a60f0016d5e3296b31",
+    "sha256": "e6072806ae22868ee09c07923d093b1b0b687dba540d22cfc1e1a5392bfcca46"
+  },
+  "ck_eth_ledger_v2": {
+    "rev": "e54d3fa34ded227c885d04e64505fa4b5d564743",
+    "sha256": "98a7b7391608dc4a554d6964bad24157b6aaf890a05bbaad3fcc92033d9c7b02"
+  },
   "cycles-minting": {
     "rev": "ee52ab3056cf5f39b09b08de70bdd20485c8b2dc",
     "sha256": "bbb8995cb749ba9e2c721ff507f5e5313f32e69b1adf3df20e3901ed56a70b42"

diff --git a/rs/ethereum/ledger-suite-orchestrator/src/scheduler/mod.rs b/rs/ethereum/ledger-suite-orchestrator/src/scheduler/mod.rs
@@ -932,8 +932,6 @@ fn icrc1_ledger_init_arg(
         ),
         max_memo_length: Some(MAX_MEMO_LENGTH),
         feature_flags: Some(ICRC2_FEATURE),
-        maximum_number_of_accounts: None,
-        accounts_overflow_trim_quantity: None,
     }
 }
 

diff --git a/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs b/rs/ledger_suite/common/ledger_canister_core/src/ledger.rs
@@ -208,6 +208,7 @@ pub enum TransferError<Tokens> {
 const APPROVE_PRUNE_LIMIT: usize = 100;
 
 /// Adds a new block with the specified transaction to the ledger.
+/// Trim balances if necessary.
 pub fn apply_transaction<L>(
     ledger: &mut L,
     transaction: L::Transaction,
@@ -217,6 +218,22 @@ pub fn apply_transaction<L>(
 where
     L: LedgerData,
     L::BalancesStore: InspectableBalancesStore,
+{
+    let result = apply_transaction_no_trimming(ledger, transaction, now, effective_fee);
+    trim_balances(ledger, now);
+    result
+}
+
+/// Adds a new block with the specified transaction to the ledger.
+/// Do not perform any balance trimming.
+pub fn apply_transaction_no_trimming<L>(
+    ledger: &mut L,
+    transaction: L::Transaction,
+    now: TimeStamp,
+    effective_fee: L::Tokens,
+) -> Result<(BlockIndex, HashOf<EncodedBlock>), TransferError<L::Tokens>>
+where
+    L: LedgerData,
 {
     let num_pruned = purge_old_transactions(ledger, now);
 
@@ -301,6 +318,16 @@ where
                 transaction_hash: tx_hash,
             });
     }
+
+    Ok((height, ledger.blockchain().last_hash.unwrap()))
+}
+
+/// Trim balances. Can be used e.g. if the ledger is low on heap memory.
+fn trim_balances<L>(ledger: &mut L, now: TimeStamp)
+where
+    L: LedgerData,
+    L::BalancesStore: InspectableBalancesStore,
+{
     let effective_max_number_of_accounts =
         ledger.max_number_of_accounts() + ledger.accounts_overflow_trim_quantity() - 1;
 
@@ -331,8 +358,6 @@ where
             ))
             .unwrap();
     }
-
-    Ok((height, ledger.blockchain().last_hash.unwrap()))
 }
 
 /// Finds the archive canister that contains the block with the specified height.

diff --git a/rs/ledger_suite/icp/ledger/tests/tests.rs b/rs/ledger_suite/icp/ledger/tests/tests.rs
@@ -1250,6 +1250,7 @@ fn test_upgrade_serialization() {
         upgrade_args,
         minter,
         false,
+        false,
     );
 }
 

diff --git a/rs/ledger_suite/icrc1/index-ng/tests/tests.rs b/rs/ledger_suite/icrc1/index-ng/tests/tests.rs
@@ -55,7 +55,6 @@ fn upgrade_ledger(
         change_fee_collector,
         max_memo_length: None,
         feature_flags: None,
-        accounts_overflow_trim_quantity: None,
         change_archive_options: None,
     }));
     env.upgrade_canister(ledger_id, ledger_wasm(), Encode!(&args).unwrap())

diff --git a/rs/ledger_suite/icrc1/ledger/BUILD.bazel b/rs/ledger_suite/icrc1/ledger/BUILD.bazel
@@ -278,14 +278,22 @@ rust_test(
             "//rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz",
             "//rs/ledger_suite/icrc1/ledger:ledger_canister_icrc3_compatible_data_certificate",
             "//rs/universal_canister/impl:universal_canister.wasm.gz",
+            "@mainnet_ckbtc_ic-icrc1-ledger-v1//file",
+            "@mainnet_ckbtc_ic-icrc1-ledger-v2//file",
             "@mainnet_ckbtc_ic-icrc1-ledger//file",
+            "@mainnet_cketh_ic-icrc1-ledger-u256-v1//file",
+            "@mainnet_cketh_ic-icrc1-ledger-u256-v2//file",
             "@mainnet_cketh_ic-icrc1-ledger-u256//file",
             "@mainnet_ic-icrc1-ledger//file",
         ],
         env = {
             "CARGO_MANIFEST_DIR": "rs/ledger_suite/icrc1/ledger",
             "CKBTC_IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger//file)",
+            "CKBTC_IC_ICRC1_LEDGER_V1_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger-v1//file)",
+            "CKBTC_IC_ICRC1_LEDGER_V2_VERSION_WASM_PATH": "$(rootpath @mainnet_ckbtc_ic-icrc1-ledger-v2//file)",
             "CKETH_IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256//file)",
+            "CKETH_IC_ICRC1_LEDGER_V1_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256-v1//file)",
+            "CKETH_IC_ICRC1_LEDGER_V2_VERSION_WASM_PATH": "$(rootpath @mainnet_cketh_ic-icrc1-ledger-u256-v2//file)",
             "IC_ICRC1_ARCHIVE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/archive:archive_canister" + name_suffix + ".wasm.gz)",
             "IC_ICRC1_LEDGER_DEPLOYED_VERSION_WASM_PATH": "$(rootpath @mainnet_ic-icrc1-ledger//file)",
             "IC_ICRC1_LEDGER_ICRC3_COMPATIBLE_DATA_CERTIFICATE_WASM_PATH": "$(rootpath //rs/ledger_suite/icrc1/ledger:ledger_canister_icrc3_compatible_data_certificate)",

diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u256.yml
@@ -1,42 +1,42 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 6398923208
-      heap_increase: 227
+      instructions: 13798815361
+      heap_increase: 172
       stable_memory_increase: 128
     scopes:
       before_upgrade:
-        instructions: 5790054103
-        heap_increase: 71
+        instructions: 13630531777
+        heap_increase: 43
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 424320009
-        heap_increase: 27
+        instructions: 118601062
+        heap_increase: 0
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 184546072
+        instructions: 49679478
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 608867671
-        heap_increase: 156
+        instructions: 168282130
+        heap_increase: 129
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8755062
+      instructions: 8684974
       heap_increase: 258
-      stable_memory_increase: 129
+      stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8627524
+        instructions: 8606422
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 125131
+        instructions: 76145
         heap_increase: 129
-        stable_memory_increase: 129
+        stable_memory_increase: 128
       upgrade:
-        instructions: 8754373
+        instructions: 8684285
         heap_increase: 258
-        stable_memory_increase: 129
+        stable_memory_increase: 128
 version: 0.1.7
diff --git a/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml b/rs/ledger_suite/icrc1/ledger/canbench_results/canbench_u64.yml
@@ -1,42 +1,42 @@
 benches:
   bench_icrc1_transfers:
     total:
-      instructions: 5926738616
-      heap_increase: 214
+      instructions: 13237283790
+      heap_increase: 171
       stable_memory_increase: 128
     scopes:
       before_upgrade:
-        instructions: 5341274673
-        heap_increase: 64
+        instructions: 13068913917
+        heap_increase: 42
         stable_memory_increase: 0
       post_upgrade:
-        instructions: 406888378
-        heap_increase: 21
+        instructions: 118797275
+        heap_increase: 0
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 178572258
+        instructions: 49569466
         heap_increase: 129
         stable_memory_increase: 128
       upgrade:
-        instructions: 585462226
-        heap_increase: 150
+        instructions: 168368331
+        heap_increase: 129
         stable_memory_increase: 128
   bench_upgrade_baseline:
     total:
-      instructions: 8758206
+      instructions: 8686052
       heap_increase: 258
-      stable_memory_increase: 129
+      stable_memory_increase: 128
     scopes:
       post_upgrade:
-        instructions: 8627876
+        instructions: 8606533
         heap_increase: 129
         stable_memory_increase: 0
       pre_upgrade:
-        instructions: 127923
+        instructions: 77112
         heap_increase: 129
-        stable_memory_increase: 129
+        stable_memory_increase: 128
       upgrade:
-        instructions: 8757517
+        instructions: 8685363
         heap_increase: 258
-        stable_memory_increase: 129
+        stable_memory_increase: 128
 version: 0.1.7
diff --git a/rs/ledger_suite/icrc1/ledger/ledger.did b/rs/ledger_suite/icrc1/ledger/ledger.did
@@ -107,8 +107,6 @@ type InitArgs = record {
     metadata : vec record { text; MetadataValue };
     initial_balances : vec record { Account; nat };
     feature_flags : opt FeatureFlags;
-    maximum_number_of_accounts : opt nat64;
-    accounts_overflow_trim_quantity : opt nat64;
     archive_options : record {
         num_blocks_to_archive : nat64;
         max_transactions_per_response : opt nat64;
@@ -144,7 +142,6 @@ type UpgradeArgs = record {
     change_fee_collector : opt ChangeFeeCollector;
     max_memo_length : opt nat16;
     feature_flags : opt FeatureFlags;
-    accounts_overflow_trim_quantity: opt nat64;
     change_archive_options : opt ChangeArchiveOptions;
 };
 

diff --git a/rs/ledger_suite/icrc1/ledger/src/benches/mod.rs b/rs/ledger_suite/icrc1/ledger/src/benches/mod.rs
@@ -1,10 +1,8 @@
-use crate::{execute_transfer_not_async, post_upgrade, pre_upgrade, Access, Tokens};
+use crate::{balances_len, execute_transfer_not_async, post_upgrade, pre_upgrade, Tokens};
 use assert_matches::assert_matches;
 use candid::{Nat, Principal};
 use ic_canister_log::Sink;
-use ic_ledger_canister_core::ledger::{
-    blocks_to_archive, remove_archived_blocks, LedgerAccess, LedgerContext,
-};
+use ic_ledger_canister_core::ledger::{blocks_to_archive, remove_archived_blocks, LedgerAccess};
 use ic_ledger_core::block::BlockIndex;
 use icrc_ledger_types::icrc1::account::Account;
 use icrc_ledger_types::icrc1::transfer::TransferArg;
@@ -42,9 +40,7 @@ pub fn icrc1_transfer(
 }
 
 fn assert_has_num_balances(num_balances: u32) {
-    Access::with_ledger(|ledger| {
-        assert_eq!(ledger.balances().store.len() as u32, num_balances);
-    });
+    assert_eq!(balances_len() as u32, num_balances);
 }
 
 pub fn max_length_principal(index: u32) -> Principal {
-Original file line number
+Diff line change
@@ Expand Up / @@ -1250,6 +1250,7 @@ fn test_upgrade_serialization() { @@
             upgrade_args,
             minter,
             false,
+            false,
         );
     }
@@ Expand Down @@