From 6c3858c9d4e877fe8ef09ae154e01a7a1cc6ce67 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 18 Jun 2024 13:56:50 +0200 Subject: [PATCH 01/63] Asc message execution - requery message bytecode after each message execution (#4710) * Requery bytecode * cargo fmt --- massa-execution-worker/src/context.rs | 12 ++++++------ massa-execution-worker/src/execution.rs | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index a58576577b9..e77060c5394 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -364,12 +364,12 @@ impl ExecutionContext { &mut self, max_gas: u64, async_msg_cst_gas_cost: u64, - ) -> Vec<(Option, AsyncMessage)> { - self.speculative_async_pool - .take_batch_to_execute(self.slot, max_gas, async_msg_cst_gas_cost) - .into_iter() - .map(|(_id, msg)| (self.get_bytecode(&msg.destination), msg)) - .collect() + ) -> Vec<(AsyncMessageId, AsyncMessage)> { + self.speculative_async_pool.take_batch_to_execute( + self.slot, + max_gas, + async_msg_cst_gas_cost, + ) } /// Create a new `ExecutionContext` for executing an active slot. diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 297538ff8e9..fcc1ccb5a07 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1271,7 +1271,9 @@ impl ExecutionState { // Try executing asynchronous messages. // Effects are cancelled on failure and the sender is reimbursed. - for (opt_bytecode, message) in messages { + for (_message_id, message) in messages { + let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); + match self.execute_async_message(message, opt_bytecode) { Ok(_message_return) => { cfg_if::cfg_if! { From c316e44c37ae060ae0c1d6981bb6f120d89ed2af Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 18 Jun 2024 09:36:59 +0200 Subject: [PATCH 02/63] Create MIP "MIP-0001-ASC-BugFix" --- massa-versioning/src/mips.rs | 28 ++++++++++++---------------- massa-versioning/src/versioning.rs | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 3c3e86592e2..66394b6bdd1 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -7,23 +7,19 @@ use massa_time::MassaTime; #[allow(unused_imports)] use crate::versioning::{MipComponent, MipInfo, MipState}; -pub fn get_mip_list() -> [(MipInfo, MipState); 0] { - // placeholder +pub fn get_mip_list() -> [(MipInfo, MipState); 1] { let mip_list = [ - /* - (MipInfo { - name: "MIP-0000".to_string(), - version: 0, - components: BTreeMap::from([ - (MipComponent::Address, 0), - (MipComponent::KeyPair, 0), - ]), - start: MassaTime::from_millis(0), - timeout: MassaTime::from_millis(0), - activation_delay: MassaTime::from_millis(0), - }, - MipState::new(MassaTime::from_millis(0))) - */ + ( + MipInfo { + name: "MIP-0001-ASC-BugFix".to_string(), + version: 1, + components: BTreeMap::from([(MipComponent::AscExecution, 1)]), + start: MassaTime::from_millis(0), // TODO: set when known + timeout: MassaTime::from_millis(0), // TODO: set when known + activation_delay: MassaTime::from_millis(3 * 24 * 60 * 60 * 1000), // TODO: set when known, 3 days as an example + }, + MipState::new(MassaTime::from_millis(0)), + ), // TODO: set when known, (when the MIP becomes defined, e.g. when merged to main branch) ]; // debug!("MIP list: {:?}", mip_list); diff --git a/massa-versioning/src/versioning.rs b/massa-versioning/src/versioning.rs index 2c73ef54941..d9937854218 100644 --- a/massa-versioning/src/versioning.rs +++ b/massa-versioning/src/versioning.rs @@ -45,6 +45,7 @@ pub enum MipComponent { Block, VM, FinalStateHashKind, + AscExecution, #[doc(hidden)] #[num_enum(default)] __Nonexhaustive, From ab29c86d72c4f72fca7e53376d598cfdfdcfe9ee Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 18 Jun 2024 14:09:09 +0200 Subject: [PATCH 03/63] Add versioning to asc execution context --- massa-execution-worker/src/context.rs | 20 ++++- massa-execution-worker/src/execution.rs | 111 +++++++++++++++++------- massa-versioning/src/mips.rs | 2 +- 3 files changed, 102 insertions(+), 31 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index e77060c5394..69bd9d06cef 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -360,7 +360,7 @@ impl ExecutionContext { /// A vector of `(Option, AsyncMessage)` pairs where: /// * `Option` is the bytecode to execute (or `None` if not found) /// * `AsyncMessage` is the asynchronous message to execute - pub(crate) fn take_async_batch( + pub(crate) fn take_async_batch_v0( &mut self, max_gas: u64, async_msg_cst_gas_cost: u64, @@ -370,6 +370,24 @@ impl ExecutionContext { max_gas, async_msg_cst_gas_cost, ) + ) -> Vec<(Option, AsyncMessage)> { + self.speculative_async_pool + .take_batch_to_execute(self.slot, max_gas, async_msg_cst_gas_cost) + .into_iter() + .map(|(_id, msg)| (self.get_bytecode(&msg.destination), msg)) + .collect() + } + + pub(crate) fn take_async_batch_v1( + &mut self, + max_gas: u64, + async_msg_cst_gas_cost: u64 + ) -> Vec<(AsyncMessageId, AsyncMessage)> { + self.speculative_async_pool.take_batch_to_execute( + self.slot, + max_gas, + async_msg_cst_gas_cost, + ) } /// Create a new `ExecutionContext` for executing an active slot. diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index fcc1ccb5a07..8a5380d72b2 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -44,7 +44,7 @@ use massa_module_cache::config::ModuleCacheConfig; use massa_module_cache::controller::ModuleCache; use massa_pos_exports::SelectorController; use massa_sc_runtime::{Interface, Response, VMError}; -use massa_versioning::versioning::MipStore; +use massa_versioning::versioning::{MipComponent, MipStore}; use massa_wallet::Wallet; use parking_lot::{Mutex, RwLock}; use std::collections::{BTreeMap, BTreeSet}; @@ -1104,7 +1104,9 @@ impl ExecutionState { &self, message: AsyncMessage, bytecode: Option, + asc_execution_version: u32 ) -> Result { + let mut result = AsyncMessageExecutionResult::new(); #[cfg(feature = "execution-info")] { @@ -1123,7 +1125,10 @@ impl ExecutionState { context.stack = vec![ ExecutionStackElement { address: message.sender, - coins: message.coins, + coins: match asc_execution_version { + 0 => message.coins, + _ => Default::default() + }, owned_addresses: vec![message.sender], operation_datastore: None, }, @@ -1260,37 +1265,85 @@ impl ExecutionState { self.mip_store.clone(), ); - // Get asynchronous messages to execute - let messages = execution_context.take_async_batch( - self.config.max_async_gas, - self.config.async_msg_cst_gas_cost, - ); + let ts = get_block_slot_timestamp( + self.config.thread_count, + self.config.t0, + self.config.genesis_timestamp, + self.final_cursor, + ) + .unwrap(); + + let asc_execution_version = self.final_state.read().get_mip_store().get_latest_component_version_at(&MipComponent::AscExecution, ts); + + match asc_execution_version { + 0 => { + // Get asynchronous messages to execute + let messages = execution_context.take_async_batch_v0( + self.config.max_async_gas, + self.config.async_msg_cst_gas_cost + ); - // Apply the created execution context for slot execution - *context_guard!(self) = execution_context; - - // Try executing asynchronous messages. - // Effects are cancelled on failure and the sender is reimbursed. - for (_message_id, message) in messages { - let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); - - match self.execute_async_message(message, opt_bytecode) { - Ok(_message_return) => { - cfg_if::cfg_if! { - if #[cfg(feature = "execution-trace")] { - // Safe to unwrap - slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); - } else if #[cfg(feature = "execution-info")] { - slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); - exec_info.async_messages.push(Ok(_message_return)); + // Apply the created execution context for slot execution + *context_guard!(self) = execution_context; + + // Try executing asynchronous messages. + // Effects are cancelled on failure and the sender is reimbursed. + for (opt_bytecode, message) in messages { + match self.execute_async_message(message, opt_bytecode, asc_execution_version) { + Ok(_message_return) => { + cfg_if::cfg_if! { + if #[cfg(feature = "execution-trace")] { + // Safe to unwrap + slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); + } else if #[cfg(feature = "execution-info")] { + slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); + exec_info.async_messages.push(Ok(_message_return)); + } + } + } + Err(err) => { + let msg = format!("failed executing async message: {}", err); + #[cfg(feature = "execution-info")] + exec_info.async_messages.push(Err(msg.clone())); + debug!(msg); } } } - Err(err) => { - let msg = format!("failed executing async message: {}", err); - #[cfg(feature = "execution-info")] - exec_info.async_messages.push(Err(msg.clone())); - debug!(msg); + }, + _ => { + // Get asynchronous messages to execute + let messages = execution_context.take_async_batch_v1( + self.config.max_async_gas, + self.config.async_msg_cst_gas_cost + ); + + // Apply the created execution context for slot execution + *context_guard!(self) = execution_context; + + // Try executing asynchronous messages. + // Effects are cancelled on failure and the sender is reimbursed. + for (_message_id, message) in messages { + let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); + + match self.execute_async_message(message, opt_bytecode, asc_execution_version) { + Ok(_message_return) => { + cfg_if::cfg_if! { + if #[cfg(feature = "execution-trace")] { + // Safe to unwrap + slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); + } else if #[cfg(feature = "execution-info")] { + slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); + exec_info.async_messages.push(Ok(_message_return)); + } + } + } + Err(err) => { + let msg = format!("failed executing async message: {}", err); + #[cfg(feature = "execution-info")] + exec_info.async_messages.push(Err(msg.clone())); + debug!(msg); + } + } } } } diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 66394b6bdd1..9f6bb205e69 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -14,7 +14,7 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { name: "MIP-0001-ASC-BugFix".to_string(), version: 1, components: BTreeMap::from([(MipComponent::AscExecution, 1)]), - start: MassaTime::from_millis(0), // TODO: set when known + start: MassaTime::from_millis(0), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); timeout: MassaTime::from_millis(0), // TODO: set when known activation_delay: MassaTime::from_millis(3 * 24 * 60 * 60 * 1000), // TODO: set when known, 3 days as an example }, From 93b2b3b7465c9d2a97645dac39024f2a3f563d28 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 18 Jun 2024 13:56:50 +0200 Subject: [PATCH 04/63] Asc message execution - requery message bytecode after each message execution (#4710) * Requery bytecode * cargo fmt --- massa-execution-worker/src/context.rs | 12 ++++++------ massa-execution-worker/src/execution.rs | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index a58576577b9..e77060c5394 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -364,12 +364,12 @@ impl ExecutionContext { &mut self, max_gas: u64, async_msg_cst_gas_cost: u64, - ) -> Vec<(Option, AsyncMessage)> { - self.speculative_async_pool - .take_batch_to_execute(self.slot, max_gas, async_msg_cst_gas_cost) - .into_iter() - .map(|(_id, msg)| (self.get_bytecode(&msg.destination), msg)) - .collect() + ) -> Vec<(AsyncMessageId, AsyncMessage)> { + self.speculative_async_pool.take_batch_to_execute( + self.slot, + max_gas, + async_msg_cst_gas_cost, + ) } /// Create a new `ExecutionContext` for executing an active slot. diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 297538ff8e9..fcc1ccb5a07 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1271,7 +1271,9 @@ impl ExecutionState { // Try executing asynchronous messages. // Effects are cancelled on failure and the sender is reimbursed. - for (opt_bytecode, message) in messages { + for (_message_id, message) in messages { + let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); + match self.execute_async_message(message, opt_bytecode) { Ok(_message_return) => { cfg_if::cfg_if! { From a7cc744a5506be71765b2df4a62607fa2ee78766 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 10:57:20 +0200 Subject: [PATCH 05/63] Improve versioning --- Cargo.lock | 39 ++-------------- Cargo.toml | 2 +- massa-execution-worker/src/context.rs | 49 ++++++++++++++++---- massa-execution-worker/src/execution.rs | 25 ++++------ massa-execution-worker/src/interface_impl.rs | 5 ++ massa-versioning/src/mips.rs | 4 +- massa-versioning/src/versioning.rs | 2 +- 7 files changed, 62 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b623ef87bb3..ac558d31e46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2051,15 +2051,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.0" @@ -2198,7 +2189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29110019693a4fa2dbda04876499d098fa16d70eba06b1e6e2b3f1b251419515" dependencies = [ "heck", - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2 1.0.71", "quote 1.0.33", "syn 1.0.109", @@ -2579,7 +2570,7 @@ dependencies = [ [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?rev=80352eb9f2a6b90a441cd64433d8874c33fb384f#80352eb9f2a6b90a441cd64433d8874c33fb384f" +source = "git+https://github.com/massalabs/massa-sc-runtime?rev=0bf920729a763300e8de9c73299296a50f20366e#0bf920729a763300e8de9c73299296a50f20366e" dependencies = [ "anyhow", "as-ffi-bindings", @@ -3743,7 +3734,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" dependencies = [ - "proc-macro-crate 2.0.0", + "proc-macro-crate", "proc-macro2 1.0.71", "quote 1.0.33", "syn 2.0.43", @@ -4142,15 +4133,6 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "proc-macro-crate" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" -dependencies = [ - "toml_edit 0.20.7", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -4241,7 +4223,7 @@ checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4262,7 +4244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2 1.0.71", "quote 1.0.33", "syn 2.0.43", @@ -5520,17 +5502,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "toml_edit" -version = "0.20.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" -dependencies = [ - "indexmap 2.1.0", - "toml_datetime", - "winnow", -] - [[package]] name = "toml_edit" version = "0.21.0" diff --git a/Cargo.toml b/Cargo.toml index feaaee062ea..211067dbaa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,7 +107,7 @@ massa_wallet = { path = "./massa-wallet" } # Massa projects dependencies massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "38950875a7aa406fedc4f0b8336864e5ff290f2c" } -massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "80352eb9f2a6b90a441cd64433d8874c33fb384f" } +massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "0bf920729a763300e8de9c73299296a50f20366e" } peernet = { git = "https://github.com/massalabs/PeerNet", "rev" = "04b05ddd320fbe76cc858115af7b5fc28bdb8310" } # Common dependencies diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 0fe07d61f4e..8e3e86ddf6d 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -40,7 +40,7 @@ use massa_module_cache::controller::ModuleCache; use massa_pos_exports::PoSChanges; use massa_serialization::Serializer; use massa_versioning::address_factory::{AddressArgs, AddressFactory}; -use massa_versioning::versioning::MipStore; +use massa_versioning::versioning::{MipComponent, MipStore}; use massa_versioning::versioning_factory::{FactoryStrategy, VersioningFactory}; use parking_lot::RwLock; use rand::SeedableRng; @@ -179,6 +179,9 @@ pub struct ExecutionContext { /// The gas remaining before the last subexecution. /// so *excluding* the gas used by the last sc call. pub gas_remaining_before_subexecution: Option, + + /// The version of the execution component + pub execution_component_version: u32, } impl ExecutionContext { @@ -201,6 +204,15 @@ impl ExecutionContext { mip_store: MipStore, execution_trail_hash: massa_hash::Hash, ) -> Self { + let slot = Slot::new(0, 0); + let ts = get_block_slot_timestamp( + config.thread_count, + config.t0, + config.genesis_timestamp, + slot + ) + .unwrap(); + ExecutionContext { speculative_ledger: SpeculativeLedger::new( final_state.clone(), @@ -227,7 +239,7 @@ impl ExecutionContext { active_history, ), creator_min_balance: Default::default(), - slot: Slot::new(0, 0), + slot, created_addr_index: Default::default(), created_event_index: Default::default(), created_message_index: Default::default(), @@ -240,9 +252,10 @@ impl ExecutionContext { origin_operation_id: Default::default(), module_cache, config, - address_factory: AddressFactory { mip_store }, + address_factory: AddressFactory { mip_store: mip_store.clone() }, execution_trail_hash, gas_remaining_before_subexecution: None, + execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), } } @@ -335,11 +348,20 @@ impl ExecutionContext { let execution_trail_hash = generate_execution_trail_hash(&prev_execution_trail_hash, &slot, None, true); + let ts = get_block_slot_timestamp( + config.thread_count, + config.t0, + config.genesis_timestamp, + slot + ) + .unwrap(); + // return readonly context ExecutionContext { slot, stack: call_stack, read_only: true, + execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), ..ExecutionContext::new( config, final_state, @@ -364,12 +386,12 @@ impl ExecutionContext { &mut self, max_gas: u64, async_msg_cst_gas_cost: u64, - ) -> Vec<(AsyncMessageId, AsyncMessage)> { - self.speculative_async_pool.take_batch_to_execute( - self.slot, - max_gas, - async_msg_cst_gas_cost, - ) + ) -> Vec<(Option, AsyncMessage)> { + self.speculative_async_pool + .take_batch_to_execute(self.slot, max_gas, async_msg_cst_gas_cost) + .into_iter() + .map(|(_id, msg)| (self.get_bytecode(&msg.destination), msg)) + .collect() } pub(crate) fn take_async_batch_v1( @@ -417,10 +439,19 @@ impl ExecutionContext { false, ); + let ts = get_block_slot_timestamp( + config.thread_count, + config.t0, + config.genesis_timestamp, + slot + ) + .unwrap(); + // return active slot execution context ExecutionContext { slot, opt_block_id, + execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), ..ExecutionContext::new( config, final_state, diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index f683d40cff8..e0481500f19 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -44,7 +44,7 @@ use massa_module_cache::config::ModuleCacheConfig; use massa_module_cache::controller::ModuleCache; use massa_pos_exports::SelectorController; use massa_sc_runtime::{Interface, Response, VMError}; -use massa_versioning::versioning::{MipComponent, MipStore}; +use massa_versioning::versioning::MipStore; use massa_wallet::Wallet; use parking_lot::{Mutex, RwLock}; use std::collections::{BTreeMap, BTreeSet}; @@ -1104,7 +1104,7 @@ impl ExecutionState { &self, message: AsyncMessage, bytecode: Option, - asc_execution_version: u32 + execution_version: u32 ) -> Result { let mut result = AsyncMessageExecutionResult::new(); @@ -1125,7 +1125,7 @@ impl ExecutionState { context.stack = vec![ ExecutionStackElement { address: message.sender, - coins: match asc_execution_version { + coins: match execution_version { 0 => message.coins, _ => Default::default() }, @@ -1234,7 +1234,7 @@ impl ExecutionState { } } } - + /// Executes a full slot (with or without a block inside) without causing any changes to the state, /// just yielding the execution output. /// @@ -1274,17 +1274,8 @@ impl ExecutionState { self.mip_store.clone(), ); - let ts = get_block_slot_timestamp( - self.config.thread_count, - self.config.t0, - self.config.genesis_timestamp, - self.final_cursor, - ) - .unwrap(); - - let asc_execution_version = self.final_state.read().get_mip_store().get_latest_component_version_at(&MipComponent::AscExecution, ts); - - match asc_execution_version { + let execution_version = execution_context.execution_component_version; + match execution_version { 0 => { // Get asynchronous messages to execute let messages = execution_context.take_async_batch_v0( @@ -1298,7 +1289,7 @@ impl ExecutionState { // Try executing asynchronous messages. // Effects are cancelled on failure and the sender is reimbursed. for (opt_bytecode, message) in messages { - match self.execute_async_message(message, opt_bytecode, asc_execution_version) { + match self.execute_async_message(message, opt_bytecode, execution_version) { Ok(_message_return) => { cfg_if::cfg_if! { if #[cfg(feature = "execution-trace")] { @@ -1334,7 +1325,7 @@ impl ExecutionState { for (_message_id, message) in messages { let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); - match self.execute_async_message(message, opt_bytecode, asc_execution_version) { + match self.execute_async_message(message, opt_bytecode, execution_version) { Ok(_message_return) => { cfg_if::cfg_if! { if #[cfg(feature = "execution-trace")] { diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index c1a2648f4dd..6627ffcec7d 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -226,6 +226,11 @@ impl Interface for InterfaceImpl { Ok(()) } + fn get_interface_version(&self) -> Result { + let context = context_guard!(self); + Ok(context.execution_component_version) + } + /// Initialize the call when bytecode calls a function from another bytecode /// This function transfers the coins passed as parameter, /// prepares the current execution context by pushing a new element on the top of the call stack, diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 9f6bb205e69..21d385bb392 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -11,9 +11,9 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { let mip_list = [ ( MipInfo { - name: "MIP-0001-ASC-BugFix".to_string(), + name: "MIP-0001-Execution-BugFix".to_string(), version: 1, - components: BTreeMap::from([(MipComponent::AscExecution, 1)]), + components: BTreeMap::from([(MipComponent::Execution, 1)]), start: MassaTime::from_millis(0), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); timeout: MassaTime::from_millis(0), // TODO: set when known activation_delay: MassaTime::from_millis(3 * 24 * 60 * 60 * 1000), // TODO: set when known, 3 days as an example diff --git a/massa-versioning/src/versioning.rs b/massa-versioning/src/versioning.rs index d9937854218..010e82ba09d 100644 --- a/massa-versioning/src/versioning.rs +++ b/massa-versioning/src/versioning.rs @@ -45,7 +45,7 @@ pub enum MipComponent { Block, VM, FinalStateHashKind, - AscExecution, + Execution, #[doc(hidden)] #[num_enum(default)] __Nonexhaustive, From 4f89e6ac72396c0547af32427f45248b4215184f Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 10:57:48 +0200 Subject: [PATCH 06/63] fmt --- massa-execution-worker/src/context.rs | 21 +++++++++++++-------- massa-execution-worker/src/execution.rs | 15 +++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 8e3e86ddf6d..9c5740355ce 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -209,7 +209,7 @@ impl ExecutionContext { config.thread_count, config.t0, config.genesis_timestamp, - slot + slot, ) .unwrap(); @@ -252,10 +252,13 @@ impl ExecutionContext { origin_operation_id: Default::default(), module_cache, config, - address_factory: AddressFactory { mip_store: mip_store.clone() }, + address_factory: AddressFactory { + mip_store: mip_store.clone(), + }, execution_trail_hash, gas_remaining_before_subexecution: None, - execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), + execution_component_version: mip_store + .get_latest_component_version_at(&MipComponent::Execution, ts), } } @@ -352,7 +355,7 @@ impl ExecutionContext { config.thread_count, config.t0, config.genesis_timestamp, - slot + slot, ) .unwrap(); @@ -361,7 +364,8 @@ impl ExecutionContext { slot, stack: call_stack, read_only: true, - execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), + execution_component_version: mip_store + .get_latest_component_version_at(&MipComponent::Execution, ts), ..ExecutionContext::new( config, final_state, @@ -397,7 +401,7 @@ impl ExecutionContext { pub(crate) fn take_async_batch_v1( &mut self, max_gas: u64, - async_msg_cst_gas_cost: u64 + async_msg_cst_gas_cost: u64, ) -> Vec<(AsyncMessageId, AsyncMessage)> { self.speculative_async_pool.take_batch_to_execute( self.slot, @@ -443,7 +447,7 @@ impl ExecutionContext { config.thread_count, config.t0, config.genesis_timestamp, - slot + slot, ) .unwrap(); @@ -451,7 +455,8 @@ impl ExecutionContext { ExecutionContext { slot, opt_block_id, - execution_component_version: mip_store.get_latest_component_version_at(&MipComponent::Execution, ts), + execution_component_version: mip_store + .get_latest_component_version_at(&MipComponent::Execution, ts), ..ExecutionContext::new( config, final_state, diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index e0481500f19..8c6ade7adb0 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1104,9 +1104,8 @@ impl ExecutionState { &self, message: AsyncMessage, bytecode: Option, - execution_version: u32 + execution_version: u32, ) -> Result { - let mut result = AsyncMessageExecutionResult::new(); #[cfg(feature = "execution-info")] { @@ -1127,7 +1126,7 @@ impl ExecutionState { address: message.sender, coins: match execution_version { 0 => message.coins, - _ => Default::default() + _ => Default::default(), }, owned_addresses: vec![message.sender], operation_datastore: None, @@ -1234,7 +1233,7 @@ impl ExecutionState { } } } - + /// Executes a full slot (with or without a block inside) without causing any changes to the state, /// just yielding the execution output. /// @@ -1280,7 +1279,7 @@ impl ExecutionState { // Get asynchronous messages to execute let messages = execution_context.take_async_batch_v0( self.config.max_async_gas, - self.config.async_msg_cst_gas_cost + self.config.async_msg_cst_gas_cost, ); // Apply the created execution context for slot execution @@ -1309,12 +1308,12 @@ impl ExecutionState { } } } - }, + } _ => { // Get asynchronous messages to execute let messages = execution_context.take_async_batch_v1( self.config.max_async_gas, - self.config.async_msg_cst_gas_cost + self.config.async_msg_cst_gas_cost, ); // Apply the created execution context for slot execution @@ -1324,7 +1323,7 @@ impl ExecutionState { // Effects are cancelled on failure and the sender is reimbursed. for (_message_id, message) in messages { let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); - + match self.execute_async_message(message, opt_bytecode, execution_version) { Ok(_message_return) => { cfg_if::cfg_if! { From f581540ec5fe7645e9bf350efab1a1b1cd325e97 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:06:19 +0200 Subject: [PATCH 07/63] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 680d261af63..63a8df54a72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, staging, trying] pull_request: - branches: [main, 'testnet_*'] + branches: [main, 'testnet_*', 'mainnet_*'] types: - opened - reopened From 72bd77cb3d7da6d762ee7c2efe3ff62c1e2e87a5 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:32:51 +0200 Subject: [PATCH 08/63] Add ledger changes versioning --- massa-execution-worker/src/context.rs | 96 ++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 9c5740355ce..c67b15fa2a0 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -928,14 +928,80 @@ impl ExecutionContext { result } - /// Finishes a slot and generates the execution output. - /// Settles emitted asynchronous messages, reimburse the senders of deleted messages. - /// Moves the output of the execution out of the context, - /// resetting some context fields in the process. - /// - /// This is used to get the output of an execution before discarding the context. - /// Note that we are not taking self by value to consume it because the context is shared. - pub fn settle_slot(&mut self, block_info: Option) -> ExecutionOutput { + fn settle_slot_v0(&mut self, block_info: Option) -> ExecutionOutput { + let slot = self.slot; + // execute the deferred credits coming from roll sells + let deferred_credits_transfers = self.execute_deferred_credits(&slot); + + // take the ledger changes first as they are needed for async messages and cache + let ledger_changes = self.speculative_ledger.take(); + + // settle emitted async messages and reimburse the senders of deleted messages + let deleted_messages = self + .speculative_async_pool + .settle_slot(&slot, &ledger_changes); + + let mut cancel_async_message_transfers = vec![]; + for (_msg_id, msg) in deleted_messages { + if let Some(t) = self.cancel_async_message(&msg) { + cancel_async_message_transfers.push(t) + } + } + + // update module cache + let bc_updates = ledger_changes.get_bytecode_updates(); + + { + let mut cache_write_lock = self.module_cache.write(); + for bytecode in bc_updates { + cache_write_lock.save_module(&bytecode.0); + } + } + // if the current slot is last in cycle check the production stats and act accordingly + let auto_sell_rolls = if self + .slot + .is_last_of_cycle(self.config.periods_per_cycle, self.config.thread_count) + { + self.speculative_roll_state.settle_production_stats( + &slot, + self.config.periods_per_cycle, + self.config.thread_count, + self.config.roll_price, + self.config.max_miss_ratio, + ) + } else { + vec![] + }; + + // generate the execution output + let state_changes = StateChanges { + ledger_changes, + async_pool_changes: self.speculative_async_pool.take(), + pos_changes: self.speculative_roll_state.take(), + executed_ops_changes: self.speculative_executed_ops.take(), + executed_denunciations_changes: self.speculative_executed_denunciations.take(), + execution_trail_hash_change: SetOrKeep::Set(self.execution_trail_hash), + }; + std::mem::take(&mut self.opt_block_id); + ExecutionOutput { + slot, + block_info, + state_changes, + events: std::mem::take(&mut self.events), + #[cfg(feature = "execution-trace")] + slot_trace: None, + #[cfg(feature = "dump-block")] + storage: None, + deferred_credits_execution: deferred_credits_transfers, + cancel_async_message_execution: cancel_async_message_transfers, + auto_sell_execution: auto_sell_rolls, + } + } + + fn settle_slot_with_fixed_ledger_change_handling( + &mut self, + block_info: Option, + ) -> ExecutionOutput { let slot = self.slot; // execute the deferred credits coming from roll sells @@ -1004,6 +1070,20 @@ impl ExecutionContext { } } + /// Finishes a slot and generates the execution output. + /// Settles emitted asynchronous messages, reimburse the senders of deleted messages. + /// Moves the output of the execution out of the context, + /// resetting some context fields in the process. + /// + /// This is used to get the output of an execution before discarding the context. + /// Note that we are not taking self by value to consume it because the context is shared. + pub fn settle_slot(&mut self, block_info: Option) -> ExecutionOutput { + match self.execution_component_version { + 0 => self.settle_slot_v0(block_info), + _ => self.settle_slot_with_fixed_ledger_change_handling(block_info), + } + } + /// Sets a bytecode for an address in the speculative ledger. /// Fail if the address is absent from the ledger. /// From 44a187443aa267016058d88bbec5fcf00b79bf7b Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:40:00 +0200 Subject: [PATCH 09/63] Add runtime module versioning --- massa-execution-worker/src/execution.rs | 33 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 8c6ade7adb0..5755b625308 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1179,17 +1179,28 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface - let Ok(module) = self - .module_cache - .write() - .load_module(&bytecode, message.max_gas) - else { - let err = - ExecutionError::RuntimeError("could not load module for async execution".into()); - let mut context = context_guard!(self); - context.reset_to_snapshot(context_snapshot, err.clone()); - context.cancel_async_message(&message); - return Err(err); + let module = match context_guard!(self).execution_component_version { + 0 => { + self + .module_cache + .write() + .load_module(&bytecode, message.max_gas)? + } + _ => { + let Ok(_module) = self + .module_cache + .write() + .load_module(&bytecode, message.max_gas) + else { + let err = + ExecutionError::RuntimeError("could not load module for async execution".into()); + let mut context = context_guard!(self); + context.reset_to_snapshot(context_snapshot, err.clone()); + context.cancel_async_message(&message); + return Err(err); + }; + _module + } }; let response = massa_sc_runtime::run_function( From 59a50526c8151fe183f5966d560cf0f98e853e23 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:47:31 +0200 Subject: [PATCH 10/63] Add send_message versioning --- massa-execution-worker/src/interface_impl.rs | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 6627ffcec7d..182b957b3f4 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1189,10 +1189,6 @@ impl Interface for InterfaceImpl { bail!("validity end thread exceeds the configuration thread count") } - if max_gas < self.config.gas_costs.max_instance_cost { - bail!("max gas is lower than the minimum instance cost") - } - let target_addr = Address::from_str(target_address)?; // check that the target address is an SC address @@ -1211,12 +1207,18 @@ impl Interface for InterfaceImpl { let mut execution_context = context_guard!(self); let emission_slot = execution_context.slot; - if Slot::new(validity_end.0, validity_end.1) < Slot::new(validity_start.0, validity_start.1) - { - bail!("validity end is earlier than the validity start") - } - if Slot::new(validity_end.0, validity_end.1) < emission_slot { - bail!("validity end is earlier than the current slot") + let execution_component_version = execution_context.execution_component_version; + if execution_component_version > 0 { + if max_gas < self.config.gas_costs.max_instance_cost { + bail!("max gas is lower than the minimum instance cost") + } + if Slot::new(validity_end.0, validity_end.1) < Slot::new(validity_start.0, validity_start.1) + { + bail!("validity end is earlier than the validity start") + } + if Slot::new(validity_end.0, validity_end.1) < emission_slot { + bail!("validity end is earlier than the current slot") + } } let emission_index = execution_context.created_message_index; From 361427fc28bf3037d0302afc6934d3993d71d5a3 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:49:54 +0200 Subject: [PATCH 11/63] Add address category versioning --- massa-execution-worker/src/execution.rs | 15 +++++++-------- massa-execution-worker/src/interface_impl.rs | 15 ++++++++++----- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 5755b625308..e222197bde8 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1180,20 +1180,19 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface let module = match context_guard!(self).execution_component_version { - 0 => { - self - .module_cache - .write() - .load_module(&bytecode, message.max_gas)? - } + 0 => self + .module_cache + .write() + .load_module(&bytecode, message.max_gas)?, _ => { let Ok(_module) = self .module_cache .write() .load_module(&bytecode, message.max_gas) else { - let err = - ExecutionError::RuntimeError("could not load module for async execution".into()); + let err = ExecutionError::RuntimeError( + "could not load module for async execution".into(), + ); let mut context = context_guard!(self); context.reset_to_snapshot(context_snapshot, err.clone()); context.cancel_async_message(&message); diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 182b957b3f4..f12e9652469 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1212,7 +1212,8 @@ impl Interface for InterfaceImpl { if max_gas < self.config.gas_costs.max_instance_cost { bail!("max gas is lower than the minimum instance cost") } - if Slot::new(validity_end.0, validity_end.1) < Slot::new(validity_start.0, validity_start.1) + if Slot::new(validity_end.0, validity_end.1) + < Slot::new(validity_start.0, validity_start.1) { bail!("validity end is earlier than the validity start") } @@ -1524,11 +1525,15 @@ impl Interface for InterfaceImpl { fn get_address_category_wasmv1(&self, to_check: &str) -> Result { let addr = Address::from_str(to_check)?; - match addr { - Address::User(_) => Ok(AddressCategory::UserAddress), - Address::SC(_) => Ok(AddressCategory::ScAddress), + let execution_component_version = context_guard!(self).execution_component_version; + + match (addr, execution_component_version) { + (Address::User(_), 0) => Ok(AddressCategory::ScAddress), + (Address::SC(_), 0) => Ok(AddressCategory::UserAddress), + (Address::User(_), _) => Ok(AddressCategory::UserAddress), + (Address::SC(_), _) => Ok(AddressCategory::ScAddress), #[allow(unreachable_patterns)] - _ => Ok(AddressCategory::Unspecified), + (_, _) => Ok(AddressCategory::Unspecified), } } From 99d8abee073a5f09763c83cfac7c02057b927d9e Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 18 Jul 2024 11:58:45 +0200 Subject: [PATCH 12/63] Add Fix eliminated msg versioning --- massa-execution-worker/src/context.rs | 14 ++++++++------ .../src/speculative_async_pool.rs | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index c67b15fa2a0..3a90208237c 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -937,9 +937,9 @@ impl ExecutionContext { let ledger_changes = self.speculative_ledger.take(); // settle emitted async messages and reimburse the senders of deleted messages - let deleted_messages = self - .speculative_async_pool - .settle_slot(&slot, &ledger_changes); + let deleted_messages = + self.speculative_async_pool + .settle_slot(&slot, &ledger_changes, false); let mut cancel_async_message_transfers = vec![]; for (_msg_id, msg) in deleted_messages { @@ -1008,9 +1008,11 @@ impl ExecutionContext { let deferred_credits_transfers = self.execute_deferred_credits(&slot); // settle emitted async messages and reimburse the senders of deleted messages - let deleted_messages = self - .speculative_async_pool - .settle_slot(&slot, &self.speculative_ledger.added_changes); + let deleted_messages = self.speculative_async_pool.settle_slot( + &slot, + &self.speculative_ledger.added_changes, + true, + ); let mut cancel_async_message_transfers = vec![]; for (_msg_id, msg) in deleted_messages { diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index 4413fd35609..347e3e7dd76 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -155,6 +155,7 @@ impl SpeculativeAsyncPool { &mut self, slot: &Slot, ledger_changes: &LedgerChanges, + fix_eliminated_msg: bool, ) -> Vec<(AsyncMessageId, AsyncMessage)> { // Update the messages_info: remove messages that should be removed // Filter out all messages for which the validity end is expired. @@ -223,12 +224,13 @@ impl SpeculativeAsyncPool { // Query eliminated messages let mut eliminated_msg = self.fetch_msgs(eliminated_infos.iter().map(|(id, _)| id).collect(), true); - - eliminated_msg.extend(eliminated_new_messages.iter().filter_map(|(k, v)| match v { - SetUpdateOrDelete::Set(v) => Some((*k, v.clone())), - SetUpdateOrDelete::Update(_v) => None, - SetUpdateOrDelete::Delete => None, - })); + if fix_eliminated_msg { + eliminated_msg.extend(eliminated_new_messages.iter().filter_map(|(k, v)| match v { + SetUpdateOrDelete::Set(v) => Some((*k, v.clone())), + SetUpdateOrDelete::Update(_v) => None, + SetUpdateOrDelete::Delete => None, + })); + } eliminated_msg } From 5df27005ea062038facfb0f37bd483648b278873 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 23 Jul 2024 08:39:06 +0200 Subject: [PATCH 13/63] add versioning test-exports mip --- massa-execution-worker/Cargo.toml | 2 ++ massa-execution-worker/src/tests/universe.rs | 6 ++-- massa-versioning/src/lib.rs | 2 +- massa-versioning/src/mips.rs | 35 ++++++++++++++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/massa-execution-worker/Cargo.toml b/massa-execution-worker/Cargo.toml index 2b18732fdb2..32a02e74d72 100644 --- a/massa-execution-worker/Cargo.toml +++ b/massa-execution-worker/Cargo.toml @@ -28,6 +28,7 @@ test-exports = [ "massa_pos_worker", "massa_metrics/test-exports", "massa_metrics/test-exports", + "massa_versioning/test-exports", "massa_db_worker", "tempfile", ] @@ -106,6 +107,7 @@ massa_metrics = { workspace = true, features = ["test-exports"] } massa_db_worker = { workspace = true } tempfile = { workspace = true } massa_test_framework = { workspace = true, "features" = ["test-exports"] } +massa_versioning = { workspace = true, "features" = ["test-exports"] } tokio = { workspace = true, features = ["sync"] } hex-literal = { workspace = true } mockall = { workspace = true } diff --git a/massa-execution-worker/src/tests/universe.rs b/massa-execution-worker/src/tests/universe.rs index bfffa6c80d9..71cd8407535 100644 --- a/massa-execution-worker/src/tests/universe.rs +++ b/massa-execution-worker/src/tests/universe.rs @@ -36,7 +36,7 @@ use massa_pos_exports::MockSelectorControllerWrapper; use massa_signature::KeyPair; use massa_storage::Storage; use massa_test_framework::TestUniverse; -use massa_versioning::versioning::{MipStatsConfig, MipStore}; +use massa_versioning::{mips::get_mip_list, versioning::{MipStatsConfig, MipStore}}; use massa_wallet::test_exports::create_test_wallet; use num::rational::Ratio; use parking_lot::RwLock; @@ -100,7 +100,9 @@ impl TestUniverse for ExecutionTestUniverse { block_count_considered: MIP_STORE_STATS_BLOCK_CONSIDERED, warn_announced_version_ratio: Ratio::new_raw(30, 100), }; - let mip_store = MipStore::try_from(([], mip_stats_config)).unwrap(); + let mip_list: [(massa_versioning::versioning::MipInfo, massa_versioning::versioning::MipState); 1] = get_mip_list(); + let mip_store = MipStore::try_from((mip_list, mip_stats_config)) + .expect("mip store creation failed"); let (tx, rx) = broadcast::channel(16); #[cfg(feature = "execution-trace")] let (tx_traces, rx_traces) = broadcast::channel(16); diff --git a/massa-versioning/src/lib.rs b/massa-versioning/src/lib.rs index 60af9a824d4..78e97dd9e0f 100644 --- a/massa-versioning/src/lib.rs +++ b/massa-versioning/src/lib.rs @@ -81,5 +81,5 @@ pub mod versioning_factory; pub mod versioning_ser_der; /// Test utils -#[cfg(test)] +#[cfg(any(test, feature = "test-exports"))] pub mod test_helpers; diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 21d385bb392..82846375302 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -7,7 +7,9 @@ use massa_time::MassaTime; #[allow(unused_imports)] use crate::versioning::{MipComponent, MipInfo, MipState}; +#[cfg(not(feature = "test-exports"))] pub fn get_mip_list() -> [(MipInfo, MipState); 1] { + let mip_list = [ ( MipInfo { @@ -26,3 +28,36 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { #[allow(clippy::let_and_return)] mip_list } + +#[cfg(feature = "test-exports")] +pub fn get_mip_list() -> [(MipInfo, MipState); 1] { + use crate::{test_helpers::versioning_helpers::advance_state_until, versioning::{Active, ComponentState}}; + + println!("Running with test-exports feature"); + + let mip_info_1 = MipInfo { + name: "MIP-0001-Execution-BugFix".to_string(), + version: 1, + components: BTreeMap::from([(MipComponent::Execution, 1)]), + start: MassaTime::from_millis(2), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); + timeout: MassaTime::from_millis(10), // TODO: set when known + activation_delay: MassaTime::from_millis(2), // TODO: set when known, 3 days as an example + }; + let mip_state_1 = advance_state_until( + ComponentState::Active( + Active { at: MassaTime::from_millis(3)} + ), + &mip_info_1 + ); + + let mip_list = [ + ( + mip_info_1, + mip_state_1, + ), + ]; + + println!("MIP list: {:?}", mip_list); + #[allow(clippy::let_and_return)] + mip_list +} From f848daa88f0188c87c27610f398ee326b9bdf6ac Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 23 Jul 2024 08:56:27 +0200 Subject: [PATCH 14/63] fmt --- massa-execution-worker/src/tests/universe.rs | 14 +++++++++---- massa-versioning/src/mips.rs | 21 +++++++++----------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/massa-execution-worker/src/tests/universe.rs b/massa-execution-worker/src/tests/universe.rs index 71cd8407535..3e7bc6bba35 100644 --- a/massa-execution-worker/src/tests/universe.rs +++ b/massa-execution-worker/src/tests/universe.rs @@ -36,7 +36,10 @@ use massa_pos_exports::MockSelectorControllerWrapper; use massa_signature::KeyPair; use massa_storage::Storage; use massa_test_framework::TestUniverse; -use massa_versioning::{mips::get_mip_list, versioning::{MipStatsConfig, MipStore}}; +use massa_versioning::{ + mips::get_mip_list, + versioning::{MipStatsConfig, MipStore}, +}; use massa_wallet::test_exports::create_test_wallet; use num::rational::Ratio; use parking_lot::RwLock; @@ -100,9 +103,12 @@ impl TestUniverse for ExecutionTestUniverse { block_count_considered: MIP_STORE_STATS_BLOCK_CONSIDERED, warn_announced_version_ratio: Ratio::new_raw(30, 100), }; - let mip_list: [(massa_versioning::versioning::MipInfo, massa_versioning::versioning::MipState); 1] = get_mip_list(); - let mip_store = MipStore::try_from((mip_list, mip_stats_config)) - .expect("mip store creation failed"); + let mip_list: [( + massa_versioning::versioning::MipInfo, + massa_versioning::versioning::MipState, + ); 1] = get_mip_list(); + let mip_store = + MipStore::try_from((mip_list, mip_stats_config)).expect("mip store creation failed"); let (tx, rx) = broadcast::channel(16); #[cfg(feature = "execution-trace")] let (tx_traces, rx_traces) = broadcast::channel(16); diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 82846375302..3c33322addd 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -9,7 +9,6 @@ use crate::versioning::{MipComponent, MipInfo, MipState}; #[cfg(not(feature = "test-exports"))] pub fn get_mip_list() -> [(MipInfo, MipState); 1] { - let mip_list = [ ( MipInfo { @@ -31,7 +30,10 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { #[cfg(feature = "test-exports")] pub fn get_mip_list() -> [(MipInfo, MipState); 1] { - use crate::{test_helpers::versioning_helpers::advance_state_until, versioning::{Active, ComponentState}}; + use crate::{ + test_helpers::versioning_helpers::advance_state_until, + versioning::{Active, ComponentState}, + }; println!("Running with test-exports feature"); @@ -44,18 +46,13 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { activation_delay: MassaTime::from_millis(2), // TODO: set when known, 3 days as an example }; let mip_state_1 = advance_state_until( - ComponentState::Active( - Active { at: MassaTime::from_millis(3)} - ), - &mip_info_1 + ComponentState::Active(Active { + at: MassaTime::from_millis(3), + }), + &mip_info_1, ); - let mip_list = [ - ( - mip_info_1, - mip_state_1, - ), - ]; + let mip_list = [(mip_info_1, mip_state_1)]; println!("MIP list: {:?}", mip_list); #[allow(clippy::let_and_return)] From 5aa1fb83beaab04ad2bc81aee8676bb50ef64290 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 1 Aug 2024 10:47:37 +0200 Subject: [PATCH 15/63] Add ledger delete_entry versioning --- massa-bootstrap/src/tests/universe_server.rs | 6 +- .../src/speculative_async_pool.rs | 72 ++++++++++++++----- massa-final-state/src/final_state.rs | 21 ++++-- massa-ledger-exports/src/controller.rs | 7 +- massa-ledger-worker/src/ledger.rs | 14 +++- massa-ledger-worker/src/ledger_db.rs | 23 ++++-- massa-versioning/src/mips.rs | 7 +- massa-versioning/src/versioning.rs | 1 + 8 files changed, 114 insertions(+), 37 deletions(-) diff --git a/massa-bootstrap/src/tests/universe_server.rs b/massa-bootstrap/src/tests/universe_server.rs index 8ca827e1ddf..3ec34d77e84 100644 --- a/massa-bootstrap/src/tests/universe_server.rs +++ b/massa-bootstrap/src/tests/universe_server.rs @@ -202,7 +202,7 @@ impl BootstrapServerTestUniverseBuilder { let mut batch = DBBatch::default(); let versioning_batch = DBBatch::default(); self.final_ledger - .apply_changes_to_batch(ledger_changes, &mut batch); + .apply_changes_to_batch(ledger_changes, &mut batch, 1); self.controllers .database .write() @@ -216,7 +216,7 @@ impl BootstrapServerTestUniverseBuilder { let mut batch = DBBatch::default(); let versioning_batch = DBBatch::default(); self.final_ledger - .apply_changes_to_batch(ledger_changes, &mut batch); + .apply_changes_to_batch(ledger_changes, &mut batch, 1); self.controllers .database .write() @@ -232,7 +232,7 @@ impl BootstrapServerTestUniverseBuilder { let mut batch = DBBatch::default(); let versioning_batch = DBBatch::default(); self.final_ledger - .apply_changes_to_batch(ledger_changes, &mut batch); + .apply_changes_to_batch(ledger_changes, &mut batch, 1); self.controllers .database .write() diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index c10cf96589f..b666d37b760 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -126,7 +126,11 @@ impl SpeculativeAsyncPool { // Note: SecureShareOperation.get_validity_range(...) returns RangeInclusive // so to be consistent here, use >= & <= checks if available_gas >= corrected_max_gas - && Self::is_message_ready_to_execute(&slot, &message_info.validity_start, &message_info.validity_end) + && Self::is_message_ready_to_execute( + &slot, + &message_info.validity_start, + &message_info.validity_end, + ) && message_info.can_be_executed { available_gas -= corrected_max_gas; @@ -165,7 +169,6 @@ impl SpeculativeAsyncPool { let mut eliminated_infos = Vec::new(); self.message_infos.retain(|id, info| { - if Self::is_message_expired(slot, &info.validity_end) { eliminated_infos.push((*id, info.clone())); false @@ -326,11 +329,14 @@ impl SpeculativeAsyncPool { /// Return true if a message (given its validity_start & validity end) is ready to execute /// Must be consistent with is_message_expired - fn is_message_ready_to_execute(slot: &Slot, message_validity_start: &Slot, message_validity_end: &Slot) -> bool { + fn is_message_ready_to_execute( + slot: &Slot, + message_validity_start: &Slot, + message_validity_end: &Slot, + ) -> bool { // Note: SecureShareOperation.get_validity_range(...) returns RangeInclusive // (for operation validity) so apply the same rule for message validity - slot >= message_validity_start - && slot <= message_validity_end + slot >= message_validity_start && slot <= message_validity_end } } @@ -343,24 +349,52 @@ fn is_triggered(filter: &AsyncMessageTrigger, ledger_changes: &LedgerChanges) -> mod tests { use super::*; - // Test if is_message_expired & is_message_ready_to_execute are consistent + // Test if is_message_expired & is_message_ready_to_execute are consistent #[test] fn test_validity() { let slot1 = Slot::new(6, 0); let slot2 = Slot::new(9, 0); let slot_validity_start = Slot::new(4, 0); let slot_validity_end = Slot::new(8, 0); - - assert!(!SpeculativeAsyncPool::is_message_expired(&slot1, &slot_validity_end)); - assert!(SpeculativeAsyncPool::is_message_ready_to_execute(&slot1, &slot_validity_start, &slot_validity_end)); - - assert!(!SpeculativeAsyncPool::is_message_expired(&slot_validity_start, &slot_validity_end)); - assert!(SpeculativeAsyncPool::is_message_ready_to_execute(&slot_validity_start, &slot_validity_start, &slot_validity_end)); - - assert!(!SpeculativeAsyncPool::is_message_expired(&slot_validity_end, &slot_validity_end)); - assert!(SpeculativeAsyncPool::is_message_ready_to_execute(&slot_validity_end, &slot_validity_start, &slot_validity_end)); - - assert!(SpeculativeAsyncPool::is_message_expired(&slot2, &slot_validity_end)); - assert!(!SpeculativeAsyncPool::is_message_ready_to_execute(&slot2, &slot_validity_start, &slot_validity_end)); + + assert!(!SpeculativeAsyncPool::is_message_expired( + &slot1, + &slot_validity_end + )); + assert!(SpeculativeAsyncPool::is_message_ready_to_execute( + &slot1, + &slot_validity_start, + &slot_validity_end + )); + + assert!(!SpeculativeAsyncPool::is_message_expired( + &slot_validity_start, + &slot_validity_end + )); + assert!(SpeculativeAsyncPool::is_message_ready_to_execute( + &slot_validity_start, + &slot_validity_start, + &slot_validity_end + )); + + assert!(!SpeculativeAsyncPool::is_message_expired( + &slot_validity_end, + &slot_validity_end + )); + assert!(SpeculativeAsyncPool::is_message_ready_to_execute( + &slot_validity_end, + &slot_validity_start, + &slot_validity_end + )); + + assert!(SpeculativeAsyncPool::is_message_expired( + &slot2, + &slot_validity_end + )); + assert!(!SpeculativeAsyncPool::is_message_ready_to_execute( + &slot2, + &slot_validity_start, + &slot_validity_end + )); } -} \ No newline at end of file +} diff --git a/massa-final-state/src/final_state.rs b/massa-final-state/src/final_state.rs index 0396a663764..8bc2b5e333e 100644 --- a/massa-final-state/src/final_state.rs +++ b/massa-final-state/src/final_state.rs @@ -25,7 +25,7 @@ use massa_models::operation::OperationId; use massa_models::slot::Slot; use massa_models::timeslots::get_block_slot_timestamp; use massa_pos_exports::{PoSFinalState, SelectorController}; -use massa_versioning::versioning::MipStore; +use massa_versioning::versioning::{MipComponent, MipStore}; use tracing::{debug, info, warn}; /// Represents a final state `(ledger, async pool, executed_ops, executed_de and the state of the PoS)` @@ -428,7 +428,17 @@ impl FinalState { // check slot consistency let next_slot = cur_slot.get_next_slot(self.config.thread_count)?; - // .expect("overflow in execution state slot"); + let ts = get_block_slot_timestamp( + self.config.thread_count, + self.config.t0, + self.config.genesis_timestamp, + slot, + ) + .unwrap(); + + let final_state_component_version = self + .get_mip_store() + .get_latest_component_version_at(&MipComponent::FinalState, ts); if slot != next_slot { return Err(anyhow!( @@ -449,8 +459,11 @@ impl FinalState { // do not panic above, it might just mean that the lookback cycle is not available // bootstrap again instead - self.ledger - .apply_changes_to_batch(changes.ledger_changes, &mut db_batch); + self.ledger.apply_changes_to_batch( + changes.ledger_changes, + &mut db_batch, + final_state_component_version, + ); self.executed_ops .apply_changes_to_batch(changes.executed_ops_changes, slot, &mut db_batch); diff --git a/massa-ledger-exports/src/controller.rs b/massa-ledger-exports/src/controller.rs index 8bf77715017..24a37a31f8e 100644 --- a/massa-ledger-exports/src/controller.rs +++ b/massa-ledger-exports/src/controller.rs @@ -51,7 +51,12 @@ pub trait LedgerController: Send + Sync { /// USED FOR BOOTSTRAP ONLY fn reset(&mut self); - fn apply_changes_to_batch(&mut self, changes: LedgerChanges, ledger_batch: &mut DBBatch); + fn apply_changes_to_batch( + &mut self, + changes: LedgerChanges, + ledger_batch: &mut DBBatch, + final_state_component_version: u32, + ); /// Deserializes the key and value, useful after bootstrap fn is_key_value_valid(&self, serialized_key: &[u8], serialized_value: &[u8]) -> bool; diff --git a/massa-ledger-worker/src/ledger.rs b/massa-ledger-worker/src/ledger.rs index 94684a089a8..e74c3d675b9 100644 --- a/massa-ledger-worker/src/ledger.rs +++ b/massa-ledger-worker/src/ledger.rs @@ -150,9 +150,17 @@ impl LedgerController for FinalLedger { } /// Allows applying `LedgerChanges` to the final ledger - fn apply_changes_to_batch(&mut self, changes: LedgerChanges, ledger_batch: &mut DBBatch) { - self.sorted_ledger - .apply_changes_to_batch(changes, ledger_batch); + fn apply_changes_to_batch( + &mut self, + changes: LedgerChanges, + ledger_batch: &mut DBBatch, + final_state_component_version: u32, + ) { + self.sorted_ledger.apply_changes_to_batch( + changes, + ledger_batch, + final_state_component_version, + ); } /// Deserializes the key and value, useful after bootstrap diff --git a/massa-ledger-worker/src/ledger_db.rs b/massa-ledger-worker/src/ledger_db.rs index a95a3199614..6bbf05d1e9b 100644 --- a/massa-ledger-worker/src/ledger_db.rs +++ b/massa-ledger-worker/src/ledger_db.rs @@ -125,7 +125,12 @@ impl LedgerDB { /// # Arguments /// * changes: ledger changes to be applied /// * batch: the batch to apply the changes to - pub fn apply_changes_to_batch(&self, changes: LedgerChanges, batch: &mut DBBatch) { + pub fn apply_changes_to_batch( + &self, + changes: LedgerChanges, + batch: &mut DBBatch, + final_state_component_version: u32, + ) { // for all incoming changes for (addr, change) in changes.0 { match change { @@ -143,7 +148,7 @@ impl LedgerDB { // the incoming change deletes a ledger entry SetUpdateOrDelete::Delete => { // delete the entry, if it exists - self.delete_entry(&addr, batch); + self.delete_entry(&addr, batch, final_state_component_version); } } } @@ -432,7 +437,12 @@ impl LedgerDB { /// /// # Arguments /// * batch: the given operation batch to update - fn delete_entry(&self, addr: &Address, batch: &mut DBBatch) { + fn delete_entry( + &self, + addr: &Address, + batch: &mut DBBatch, + final_state_component_version: u32, + ) { let db = self.db.read(); // version @@ -464,7 +474,10 @@ impl LedgerDB { STATE_CF, MassaIteratorMode::From(&key_prefix, MassaDirection::Forward), ) - .take_while(|(key, _)| key < &end_prefix(&key_prefix).unwrap()) + .take_while(|(key, _)| match final_state_component_version { + 0 => key <= &end_prefix(&key_prefix).unwrap(), + _ => key < &end_prefix(&key_prefix).unwrap(), + }) { db.delete_key(batch, serialized_key.to_vec()); } @@ -658,7 +671,7 @@ mod tests { // delete entry let mut batch = DBBatch::new(); - ledger_db.delete_entry(&addr, &mut batch); + ledger_db.delete_entry(&addr, &mut batch, 1); ledger_db .db .write() diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index 3c33322addd..f161ef326dd 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -14,7 +14,10 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { MipInfo { name: "MIP-0001-Execution-BugFix".to_string(), version: 1, - components: BTreeMap::from([(MipComponent::Execution, 1)]), + components: BTreeMap::from([ + (MipComponent::Execution, 1), + (MipComponent::FinalState, 1), + ]), start: MassaTime::from_millis(0), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); timeout: MassaTime::from_millis(0), // TODO: set when known activation_delay: MassaTime::from_millis(3 * 24 * 60 * 60 * 1000), // TODO: set when known, 3 days as an example @@ -40,7 +43,7 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { let mip_info_1 = MipInfo { name: "MIP-0001-Execution-BugFix".to_string(), version: 1, - components: BTreeMap::from([(MipComponent::Execution, 1)]), + components: BTreeMap::from([(MipComponent::Execution, 1), (MipComponent::FinalState, 1)]), start: MassaTime::from_millis(2), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); timeout: MassaTime::from_millis(10), // TODO: set when known activation_delay: MassaTime::from_millis(2), // TODO: set when known, 3 days as an example diff --git a/massa-versioning/src/versioning.rs b/massa-versioning/src/versioning.rs index 010e82ba09d..409ec7f81ea 100644 --- a/massa-versioning/src/versioning.rs +++ b/massa-versioning/src/versioning.rs @@ -46,6 +46,7 @@ pub enum MipComponent { VM, FinalStateHashKind, Execution, + FinalState, #[doc(hidden)] #[num_enum(default)] __Nonexhaustive, From 0eb5ffc9c1cee1473e5e7f7c8c7d92fbdfb48972 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 1 Aug 2024 11:32:16 +0200 Subject: [PATCH 16/63] Add versioning for Consistent expiry period --- .../src/speculative_async_pool.rs | 68 +++++++++++++++---- .../src/tests/scenarios_mandatories.rs | 18 ++++- 2 files changed, 71 insertions(+), 15 deletions(-) diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index b666d37b760..f5d1518a4da 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -10,7 +10,12 @@ use massa_async_pool::{ }; use massa_final_state::FinalStateController; use massa_ledger_exports::{Applicable, LedgerChanges, SetUpdateOrDelete}; -use massa_models::slot::Slot; +use massa_models::{ + config::{GENESIS_TIMESTAMP, T0, THREAD_COUNT}, + slot::Slot, + timeslots::get_block_slot_timestamp, +}; +use massa_versioning::versioning::MipComponent; use parking_lot::RwLock; use std::{ collections::{BTreeMap, HashMap}, @@ -130,6 +135,7 @@ impl SpeculativeAsyncPool { &slot, &message_info.validity_start, &message_info.validity_end, + self.get_execution_component_version(&slot), ) && message_info.can_be_executed { @@ -168,8 +174,9 @@ impl SpeculativeAsyncPool { // Note: that the validity_end bound is included in the validity interval of the message. let mut eliminated_infos = Vec::new(); + let execution_component_version = self.get_execution_component_version(slot); self.message_infos.retain(|id, info| { - if Self::is_message_expired(slot, &info.validity_end) { + if Self::is_message_expired(slot, &info.validity_end, execution_component_version) { eliminated_infos.push((*id, info.clone())); false } else { @@ -177,10 +184,15 @@ impl SpeculativeAsyncPool { } }); + let execution_component_version = self.get_execution_component_version(&slot); let mut eliminated_new_messages = Vec::new(); self.pool_changes.0.retain(|k, v| match v { SetUpdateOrDelete::Set(message) => { - if Self::is_message_expired(slot, &message.validity_end) { + if Self::is_message_expired( + slot, + &message.validity_end, + execution_component_version, + ) { eliminated_new_messages.push((*k, v.clone())); false } else { @@ -319,12 +331,28 @@ impl SpeculativeAsyncPool { msgs } + fn get_execution_component_version(&self, slot: &Slot) -> u32 { + let ts = get_block_slot_timestamp(THREAD_COUNT, T0, *GENESIS_TIMESTAMP, *slot).unwrap(); + + self.final_state + .read() + .get_mip_store() + .get_latest_component_version_at(&MipComponent::Execution, ts) + } + /// Return true if a message (given its validity end) is expired /// Must be consistent with is_message_valid - fn is_message_expired(slot: &Slot, message_validity_end: &Slot) -> bool { + fn is_message_expired( + slot: &Slot, + message_validity_end: &Slot, + execution_component_version: u32, + ) -> bool { // Note: SecureShareOperation.get_validity_range(...) returns RangeInclusive // (for operation validity) so apply the same rule for message validity - *slot > *message_validity_end + match execution_component_version { + 0 => *slot >= *message_validity_end, + _ => *slot > *message_validity_end, + } } /// Return true if a message (given its validity_start & validity end) is ready to execute @@ -333,10 +361,14 @@ impl SpeculativeAsyncPool { slot: &Slot, message_validity_start: &Slot, message_validity_end: &Slot, + execution_component_version: u32, ) -> bool { // Note: SecureShareOperation.get_validity_range(...) returns RangeInclusive // (for operation validity) so apply the same rule for message validity - slot >= message_validity_start && slot <= message_validity_end + match execution_component_version { + 0 => slot >= message_validity_start && slot < message_validity_end, + _ => slot >= message_validity_start && slot <= message_validity_end, + } } } @@ -359,42 +391,50 @@ mod tests { assert!(!SpeculativeAsyncPool::is_message_expired( &slot1, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(SpeculativeAsyncPool::is_message_ready_to_execute( &slot1, &slot_validity_start, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(!SpeculativeAsyncPool::is_message_expired( &slot_validity_start, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(SpeculativeAsyncPool::is_message_ready_to_execute( &slot_validity_start, &slot_validity_start, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(!SpeculativeAsyncPool::is_message_expired( &slot_validity_end, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(SpeculativeAsyncPool::is_message_ready_to_execute( &slot_validity_end, &slot_validity_start, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(SpeculativeAsyncPool::is_message_expired( &slot2, - &slot_validity_end + &slot_validity_end, + 1 )); assert!(!SpeculativeAsyncPool::is_message_ready_to_execute( &slot2, &slot_validity_start, - &slot_validity_end + &slot_validity_end, + 1 )); } } diff --git a/massa-execution-worker/src/tests/scenarios_mandatories.rs b/massa-execution-worker/src/tests/scenarios_mandatories.rs index 154ea28d3cb..ec49c51cced 100644 --- a/massa-execution-worker/src/tests/scenarios_mandatories.rs +++ b/massa-execution-worker/src/tests/scenarios_mandatories.rs @@ -15,7 +15,8 @@ use massa_ledger_exports::{ }; use massa_models::bytecode::Bytecode; use massa_models::config::{ - CHAINID, ENDORSEMENT_COUNT, LEDGER_ENTRY_DATASTORE_BASE_SIZE, THREAD_COUNT, + CHAINID, ENDORSEMENT_COUNT, LEDGER_ENTRY_DATASTORE_BASE_SIZE, MIP_STORE_STATS_BLOCK_CONSIDERED, + THREAD_COUNT, }; use massa_models::prehash::PreHashMap; use massa_models::test_exports::gen_endorsements_for_denunciation; @@ -31,6 +32,8 @@ use massa_pos_exports::{ }; use massa_signature::KeyPair; use massa_test_framework::{TestUniverse, WaitPoint}; +use massa_versioning::mips::get_mip_list; +use massa_versioning::versioning::{MipStatsConfig, MipStore}; use mockall::predicate; use num::rational::Ratio; use parking_lot::RwLock; @@ -100,6 +103,19 @@ fn final_state_boilerplate( .expect_get_pos_state() .return_const(pos_final_state); + let mip_stats_config = MipStatsConfig { + block_count_considered: MIP_STORE_STATS_BLOCK_CONSIDERED, + warn_announced_version_ratio: Ratio::new_raw(30, 100), + }; + let mip_list = get_mip_list(); + let mip_store = + MipStore::try_from((mip_list, mip_stats_config)).expect("mip store creation failed"); + + mock_final_state + .write() + .expect_get_mip_store() + .return_const(mip_store); + let async_pool = custom_async_pool.unwrap_or(AsyncPool::new(AsyncPoolConfig::default(), db.clone())); mock_final_state From 6b19df6cc2bc135bce105ae3f8bbb4c9da6203b9 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 1 Aug 2024 12:30:38 +0200 Subject: [PATCH 17/63] Update speculative_async_pool.rs --- massa-execution-worker/src/speculative_async_pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index f5d1518a4da..9cc835a5672 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -184,7 +184,6 @@ impl SpeculativeAsyncPool { } }); - let execution_component_version = self.get_execution_component_version(&slot); let mut eliminated_new_messages = Vec::new(); self.pool_changes.0.retain(|k, v| match v { SetUpdateOrDelete::Set(message) => { From f78ee71451579193726e0d655e27a4d48e6eaa00 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 1 Aug 2024 14:33:46 +0200 Subject: [PATCH 18/63] Add versioning to fees fix --- massa-execution-worker/src/execution.rs | 224 ++++++++++++++++-------- 1 file changed, 154 insertions(+), 70 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index b8f8cc0275f..fefb7c532d2 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1540,89 +1540,173 @@ impl ExecutionState { // Update speculative rolls state production stats context.update_production_stats(&block_creator_addr, *slot, Some(*block_id)); - // Divide the total block credits into parts + remainder - let block_credit_part_count = 3 * (1 + self.config.endorsement_count); - let block_credit_part = block_credits - .checked_div_u64(block_credit_part_count) - .expect("critical: block_credits checked_div factor is 0"); - let remainder = block_credits - .checked_rem_u64(block_credit_part_count) - .expect("critical: block_credits checked_rem factor is 0"); - - // Give 3 parts + remainder to the block producer to stimulate block production - // even in the absence of endorsements. - let mut block_producer_credit = block_credit_part - .saturating_mul_u64(3) - .saturating_add(remainder); - - for endorsement_creator in endorsement_creators { - // Credit the creator of the block with 1 part to stimulate endorsement inclusion of endorsements, - // and dissuade from emitting the block too early (before the endorsements have propageted). - block_producer_credit = block_producer_credit.saturating_add(block_credit_part); - - // Credit creator of the endorsement with 1 part to stimulate the production of endorsements. - // This also motivates endorsers to not publish their endorsements too early (will not endorse the right block), - // and to not publish too late (will not be included in the block). - match context.transfer_coins( - None, - Some(endorsement_creator), - block_credit_part, - false, - ) { - Ok(_) => { - #[cfg(feature = "execution-info")] - exec_info - .endorsement_creator_rewards - .insert(endorsement_creator, block_credit_part); + match execution_version { + 0 => { + // Credit endorsement producers and endorsed block producers + let mut remaining_credit = block_credits; + let block_credit_part = block_credits + .checked_div_u64(3 * (1 + (self.config.endorsement_count))) + .expect("critical: block_credits checked_div factor is 0"); + + for endorsement_creator in endorsement_creators { + // credit creator of the endorsement with coins + match context.transfer_coins( + None, + Some(endorsement_creator), + block_credit_part, + false, + ) { + Ok(_) => { + remaining_credit = + remaining_credit.saturating_sub(block_credit_part); + + #[cfg(feature = "execution-info")] + exec_info + .endorsement_creator_rewards + .insert(endorsement_creator, block_credit_part); + } + Err(err) => { + debug!( + "failed to credit {} coins to endorsement creator {} for an endorsed block execution: {}", + block_credit_part, endorsement_creator, err + ) + } + } + + // credit creator of the endorsed block with coins + match context.transfer_coins( + None, + Some(endorsement_target_creator), + block_credit_part, + false, + ) { + Ok(_) => { + remaining_credit = + remaining_credit.saturating_sub(block_credit_part); + #[cfg(feature = "execution-info")] + { + exec_info.endorsement_target_reward = + Some((endorsement_target_creator, block_credit_part)); + } + } + Err(err) => { + debug!( + "failed to credit {} coins to endorsement target creator {} on block execution: {}", + block_credit_part, endorsement_target_creator, err + ) + } + } } - Err(err) => { + + // Credit block creator with remaining_credit + if let Err(err) = context.transfer_coins( + None, + Some(block_creator_addr), + remaining_credit, + false, + ) { debug!( - "failed to credit {} coins to endorsement creator {} for an endorsed block execution: {}", - block_credit_part, endorsement_creator, err + "failed to credit {} coins to block creator {} on block execution: {}", + remaining_credit, block_creator_addr, err ) + } else { + #[cfg(feature = "execution-info")] + { + exec_info.block_producer_reward = + Some((block_creator_addr, remaining_credit)); + } } } + _ => { + // Divide the total block credits into parts + remainder + let block_credit_part_count = 3 * (1 + self.config.endorsement_count); + let block_credit_part = block_credits + .checked_div_u64(block_credit_part_count) + .expect("critical: block_credits checked_div factor is 0"); + let remainder = block_credits + .checked_rem_u64(block_credit_part_count) + .expect("critical: block_credits checked_rem factor is 0"); + + // Give 3 parts + remainder to the block producer to stimulate block production + // even in the absence of endorsements. + let mut block_producer_credit = block_credit_part + .saturating_mul_u64(3) + .saturating_add(remainder); + + for endorsement_creator in endorsement_creators { + // Credit the creator of the block with 1 part to stimulate endorsement inclusion of endorsements, + // and dissuade from emitting the block too early (before the endorsements have propageted). + block_producer_credit = + block_producer_credit.saturating_add(block_credit_part); + + // Credit creator of the endorsement with 1 part to stimulate the production of endorsements. + // This also motivates endorsers to not publish their endorsements too early (will not endorse the right block), + // and to not publish too late (will not be included in the block). + match context.transfer_coins( + None, + Some(endorsement_creator), + block_credit_part, + false, + ) { + Ok(_) => { + #[cfg(feature = "execution-info")] + exec_info + .endorsement_creator_rewards + .insert(endorsement_creator, block_credit_part); + } + Err(err) => { + debug!( + "failed to credit {} coins to endorsement creator {} for an endorsed block execution: {}", + block_credit_part, endorsement_creator, err + ) + } + } - // Credit the creator of the endorsed block with 1 part. - // This is done to incentivize block producers to be endorsed, - // typically by not publishing their blocks too late. - match context.transfer_coins( - None, - Some(endorsement_target_creator), - block_credit_part, - false, - ) { - Ok(_) => { - #[cfg(feature = "execution-info")] - { - exec_info.endorsement_target_reward = - Some((endorsement_target_creator, block_credit_part)); + // Credit the creator of the endorsed block with 1 part. + // This is done to incentivize block producers to be endorsed, + // typically by not publishing their blocks too late. + match context.transfer_coins( + None, + Some(endorsement_target_creator), + block_credit_part, + false, + ) { + Ok(_) => { + #[cfg(feature = "execution-info")] + { + exec_info.endorsement_target_reward = + Some((endorsement_target_creator, block_credit_part)); + } + } + Err(err) => { + debug!( + "failed to credit {} coins to endorsement target creator {} on block execution: {}", + block_credit_part, endorsement_target_creator, err + ) + } } } - Err(err) => { + + // Credit block producer + if let Err(err) = context.transfer_coins( + None, + Some(block_creator_addr), + block_producer_credit, + false, + ) { debug!( - "failed to credit {} coins to endorsement target creator {} on block execution: {}", - block_credit_part, endorsement_target_creator, err + "failed to credit {} coins to block creator {} on block execution: {}", + block_producer_credit, block_creator_addr, err ) + } else { + #[cfg(feature = "execution-info")] + { + exec_info.block_producer_reward = + Some((block_creator_addr, block_producer_credit)); + } } } } - - // Credit block producer - if let Err(err) = - context.transfer_coins(None, Some(block_creator_addr), block_producer_credit, false) - { - debug!( - "failed to credit {} coins to block creator {} on block execution: {}", - block_producer_credit, block_creator_addr, err - ) - } else { - #[cfg(feature = "execution-info")] - { - exec_info.block_producer_reward = - Some((block_creator_addr, block_producer_credit)); - } - } } else { // the slot is a miss, check who was supposed to be the creator and update production stats let producer_addr = selector From 4ca00b0b0a77f02bcbf863eb96198eb7203e2e62 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 2 Aug 2024 16:36:26 +0200 Subject: [PATCH 19/63] Add versioning for Fix amount remaining to slash 2 --- massa-execution-worker/src/context.rs | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index f1cb880d5a7..8e8e958def9 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -821,6 +821,76 @@ impl ExecutionContext { &mut self, denounced_addr: &Address, roll_count: u64, + ) -> Result { + let execution_component_version = self.execution_component_version; + + match execution_component_version { + 0 => self.try_slash_rolls_v0(denounced_addr, roll_count), + _ => self.try_slash_rolls_v1(denounced_addr, roll_count), + } + } + + pub fn try_slash_rolls_v0( + &mut self, + denounced_addr: &Address, + roll_count: u64, + ) -> Result { + // try to slash as many roll as available + let slashed_rolls = self + .speculative_roll_state + .try_slash_rolls(denounced_addr, roll_count); + // convert slashed rolls to coins (as deferred credits => coins) + let mut slashed_coins = self + .config + .roll_price + .checked_mul_u64(slashed_rolls.unwrap_or_default()) + .ok_or_else(|| { + ExecutionError::RuntimeError(format!( + "Cannot multiply roll price by {}", + roll_count + )) + })?; + + // what remains to slash (then will try to slash as many deferred credits as avail/what remains to be slashed) + let amount_remaining_to_slash = self + .config + .roll_price + .checked_mul_u64(roll_count) + .ok_or_else(|| { + ExecutionError::RuntimeError(format!( + "Cannot multiply roll price by {}", + roll_count + )) + })? + .saturating_sub(slashed_coins); + + if amount_remaining_to_slash > Amount::zero() { + // There is still an amount to slash for this denunciation so we need to slash + // in deferred credits + let slashed_coins_in_deferred_credits = self + .speculative_roll_state + .try_slash_deferred_credits(&self.slot, denounced_addr, &amount_remaining_to_slash); + + slashed_coins = slashed_coins.saturating_add(slashed_coins_in_deferred_credits); + + let amount_remaining_to_slash_2 = + slashed_coins.saturating_sub(slashed_coins_in_deferred_credits); + if amount_remaining_to_slash_2 > Amount::zero() { + // Use saturating_mul_u64 to avoid an error (for just a warn!(..)) + warn!("Slashed {} coins (by selling rolls) and {} coins from deferred credits of address: {} but cumulative amount is lower than expected: {} coins", + slashed_coins, slashed_coins_in_deferred_credits, denounced_addr, + self.config.roll_price.saturating_mul_u64(roll_count) + ); + } + } + + Ok(slashed_coins) + } + + pub fn try_slash_rolls_v1( + &mut self, + denounced_addr: &Address, + roll_count: u64, ) -> Result { // try to slash as many roll as available let slashed_rolls = self From de4bf075e01b9fc3129f81812e12e72e6ffeac9f Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 2 Aug 2024 17:48:38 +0200 Subject: [PATCH 20/63] Add versioning for LedgerChanges::Delete --- massa-ledger-worker/src/ledger_db.rs | 34 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/massa-ledger-worker/src/ledger_db.rs b/massa-ledger-worker/src/ledger_db.rs index 4a5896d5ba3..5ed7e804774 100644 --- a/massa-ledger-worker/src/ledger_db.rs +++ b/massa-ledger-worker/src/ledger_db.rs @@ -111,7 +111,7 @@ impl LedgerDB { let mut batch = DBBatch::new(); for (address, entry) in initial_ledger { - self.put_entry(&address, entry, &mut batch); + self.put_entry(&address, entry, &mut batch, 0); } self.db.write().write_batch( @@ -138,7 +138,7 @@ impl LedgerDB { // the incoming change sets a ledger entry to a new one SetUpdateOrDelete::Set(new_entry) => { // inserts/overwrites the entry with the incoming one - self.put_entry(&addr, new_entry, batch); + self.put_entry(&addr, new_entry, batch, final_state_component_version); } // the incoming change updates an existing ledger entry SetUpdateOrDelete::Update(entry_update) => { @@ -149,7 +149,7 @@ impl LedgerDB { // the incoming change deletes a ledger entry SetUpdateOrDelete::Delete => { // delete the entry, if it exists - self.delete_entry(&addr, batch, final_state_component_version); + self.delete_entry(&addr, batch); } } } @@ -289,11 +289,22 @@ impl LedgerDB { /// * `addr`: associated address /// * `ledger_entry`: complete entry to be added /// * `batch`: the given operation batch to update - fn put_entry(&self, addr: &Address, ledger_entry: LedgerEntry, batch: &mut DBBatch) { + fn put_entry( + &self, + addr: &Address, + ledger_entry: LedgerEntry, + batch: &mut DBBatch, + final_state_component_version: u32, + ) { let db = self.db.read(); - // Ensures any potential previous entry is fully deleted. - delete_datastore_entries(addr, &db, batch); + match final_state_component_version { + 0 => {} + _ => { + // Ensures any potential previous entry is fully deleted. + delete_datastore_entries(addr, &db, batch); + } + } // Version //TODO: Get version number from parameters @@ -441,12 +452,7 @@ impl LedgerDB { /// /// # Arguments /// * batch: the given operation batch to update - fn delete_entry( - &self, - addr: &Address, - batch: &mut DBBatch, - final_state_component_version: u32, - ) { + fn delete_entry(&self, addr: &Address, batch: &mut DBBatch) { let db = self.db.read(); // version @@ -636,7 +642,7 @@ mod tests { let ledger_db = LedgerDB::new(db.clone(), 32, 255, 1000); let mut batch = DBBatch::new(); - ledger_db.put_entry(&addr, entry, &mut batch); + ledger_db.put_entry(&addr, entry, &mut batch, 0); ledger_db.update_entry(&addr, entry_update, &mut batch); ledger_db .db @@ -683,7 +689,7 @@ mod tests { // delete entry let mut batch = DBBatch::new(); - ledger_db.delete_entry(&addr, &mut batch, 1); + ledger_db.delete_entry(&addr, &mut batch); ledger_db .db .write() From f6d2f971ca4ca33f7e2433de5f71b23349cd522a Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 7 Nov 2024 14:51:30 +0100 Subject: [PATCH 21/63] cargo fmt + check pass --- massa-execution-worker/src/context.rs | 2 +- massa-execution-worker/src/interface_impl.rs | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index f6b3e3d0548..a3b12c84ce6 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -187,7 +187,7 @@ pub struct ExecutionContext { /// The version of the execution component pub execution_component_version: u32, - + /// recursion counter, incremented for each new nested call pub recursion_counter: u16, } diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 02114a971c6..8b6e50bd2ab 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -233,7 +233,7 @@ impl Interface for InterfaceImpl { let context = context_guard!(self); Ok(context.execution_component_version) } - + fn increment_recursion_counter(&self) -> Result<()> { let mut context = context_guard!(self); @@ -1798,11 +1798,6 @@ impl Interface for InterfaceImpl { } } } - - /// Interface version to sync with the runtime for its versioning - fn get_interface_version(&self) -> Result { - Ok(0) - } } #[cfg(test)] From 641b99e707a96a41fe5da8d01e4f96fdb988f799 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 7 Nov 2024 15:16:33 +0100 Subject: [PATCH 22/63] Add Condom Middleware versioning --- massa-execution-worker/src/context.rs | 5 +- massa-execution-worker/src/execution.rs | 53 +++++++++++++++----- massa-execution-worker/src/interface_impl.rs | 21 +++++++- massa-module-cache/src/controller.rs | 24 +++++---- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index a3b12c84ce6..7fafcaefbf7 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -38,6 +38,7 @@ use massa_models::{ }; use massa_module_cache::controller::ModuleCache; use massa_pos_exports::PoSChanges; +use massa_sc_runtime::CondomLimits; use massa_serialization::Serializer; use massa_versioning::address_factory::{AddressArgs, AddressFactory}; use massa_versioning::versioning::{MipComponent, MipStore}; @@ -1040,7 +1041,7 @@ impl ExecutionContext { { let mut cache_write_lock = self.module_cache.write(); for bytecode in bc_updates { - cache_write_lock.save_module(&bytecode.0); + cache_write_lock.save_module(&bytecode.0, CondomLimits::default()); } } // if the current slot is last in cycle check the production stats and act accordingly @@ -1112,7 +1113,7 @@ impl ExecutionContext { { let mut cache_write_lock = self.module_cache.write(); for bytecode in bc_updates { - cache_write_lock.save_module(&bytecode.0); + cache_write_lock.save_module(&bytecode.0, self.config.condom_limits.clone()); } } diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 433cc592920..9495abda854 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -43,7 +43,7 @@ use massa_models::{amount::Amount, slot::Slot}; use massa_module_cache::config::ModuleCacheConfig; use massa_module_cache::controller::ModuleCache; use massa_pos_exports::SelectorController; -use massa_sc_runtime::{Interface, Response, VMError}; +use massa_sc_runtime::{CondomLimits, Interface, Response, VMError}; use massa_versioning::versioning::MipStore; use massa_wallet::Wallet; use parking_lot::{Mutex, RwLock}; @@ -945,10 +945,12 @@ impl ExecutionState { _ => panic!("unexpected operation type"), }; + let execution_component_version; { // acquire write access to the context let mut context = context_guard!(self); - + + execution_component_version = context.execution_component_version; // Set the call stack to a single element: // * the execution will happen in the context of the address of the operation's sender // * the context will give the operation's sender write access to its own ledger entry @@ -962,18 +964,23 @@ impl ExecutionState { }]; }; + let condom_limits = match execution_component_version { + 0 => Default::default(), + _ => self.config.condom_limits.clone(), + }; + // load the tmp module let module = self .module_cache .read() - .load_tmp_module(bytecode, *max_gas)?; + .load_tmp_module(bytecode, *max_gas, condom_limits.clone())?; // run the VM let _res = massa_sc_runtime::run_main( &*self.execution_interface, module, *max_gas, self.config.gas_costs.clone(), - self.config.condom_limits.clone(), + condom_limits, ) .map_err(|error| ExecutionError::VMError { context: "ExecuteSC".to_string(), @@ -1018,10 +1025,13 @@ impl ExecutionState { // prepare the current slot context for executing the operation let bytecode; + let execution_component_version; { // acquire write access to the context let mut context = context_guard!(self); + execution_component_version = context.execution_component_version; + // Set the call stack // This needs to be defined before anything can fail, so that the emitted event contains the right stack context.stack = vec![ @@ -1066,7 +1076,13 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface - let module = self.module_cache.write().load_module(&bytecode, max_gas)?; + + let condom_limits = match execution_component_version { + 0 => Default::default(), + _ => self.config.condom_limits.clone(), + }; + + let module = self.module_cache.write().load_module(&bytecode, max_gas, condom_limits.clone())?; let response = massa_sc_runtime::run_function( &*self.execution_interface, module, @@ -1074,7 +1090,7 @@ impl ExecutionState { param, max_gas, self.config.gas_costs.clone(), - self.config.condom_limits.clone(), + condom_limits, ); match response { Ok(Response { init_gas_cost, .. }) @@ -1188,12 +1204,12 @@ impl ExecutionState { 0 => self .module_cache .write() - .load_module(&bytecode, message.max_gas)?, + .load_module(&bytecode, message.max_gas, CondomLimits::default())?, _ => { let Ok(_module) = self .module_cache .write() - .load_module(&bytecode, message.max_gas) + .load_module(&bytecode, message.max_gas, self.config.condom_limits.clone()) else { let err = ExecutionError::RuntimeError( "could not load module for async execution".into(), @@ -1943,6 +1959,8 @@ impl ExecutionState { // run the interpreter according to the target type let exec_response = match req.target { ReadOnlyExecutionTarget::BytecodeExecution(bytecode) => { + + let execution_component_version = execution_context.execution_component_version; { let mut context = context_guard!(self); *context = execution_context; @@ -1955,11 +1973,16 @@ impl ExecutionState { } } + let condom_limits = match execution_component_version { + 0 => CondomLimits::default(), + _ => self.config.condom_limits.clone(), + }; + // load the tmp module let module = self .module_cache .read() - .load_tmp_module(&bytecode, req.max_gas)?; + .load_tmp_module(&bytecode, req.max_gas, condom_limits.clone())?; // run the VM massa_sc_runtime::run_main( @@ -1967,7 +1990,7 @@ impl ExecutionState { module, req.max_gas, self.config.gas_costs.clone(), - self.config.condom_limits.clone(), + condom_limits, ) .map_err(|error| ExecutionError::VMError { context: "ReadOnlyExecutionTarget::BytecodeExecution".to_string(), @@ -1986,6 +2009,7 @@ impl ExecutionState { .unwrap_or_default() .0; + let execution_component_version = execution_context.execution_component_version; { let mut context = context_guard!(self); *context = execution_context; @@ -2008,12 +2032,17 @@ impl ExecutionState { } } + let condom_limits = match execution_component_version { + 0 => CondomLimits::default(), + _ => self.config.condom_limits.clone(), + }; + // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface let module = self .module_cache .write() - .load_module(&bytecode, req.max_gas)?; + .load_module(&bytecode, req.max_gas, condom_limits.clone())?; let response = massa_sc_runtime::run_function( &*self.execution_interface, @@ -2022,7 +2051,7 @@ impl ExecutionState { ¶meter, req.max_gas, self.config.gas_costs.clone(), - self.config.condom_limits.clone(), + condom_limits, ); match response { diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 8b6e50bd2ab..d3e8609c12c 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -21,6 +21,7 @@ use massa_models::{ use massa_proto_rs::massa::model::v1::{ AddressCategory, ComparisonResult, NativeAmount, NativeTime, }; +use massa_sc_runtime::CondomLimits; use massa_sc_runtime::RuntimeModule; use massa_sc_runtime::{Interface, InterfaceClone}; use massa_signature::PublicKey; @@ -334,10 +335,18 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` CL compiled module & the remaining gas after loading the module fn get_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { + + let execution_component_version = self.get_interface_version()?; + + let condom_limits = match execution_component_version { + 0 => CondomLimits::default(), + _ => self.config.condom_limits.clone(), + }; + Ok((context_guard!(self)) .module_cache .write() - .load_module(bytecode, gas_limit)?) + .load_module(bytecode, gas_limit, condom_limits)?) } /// Compile and return a temporary module @@ -345,10 +354,18 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` SP compiled module & the remaining gas after loading the module fn get_tmp_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { + + let execution_component_version = self.get_interface_version()?; + + let condom_limits = match execution_component_version { + 0 => CondomLimits::default(), + _ => self.config.condom_limits.clone(), + }; + Ok((context_guard!(self)) .module_cache .write() - .load_tmp_module(bytecode, gas_limit)?) + .load_tmp_module(bytecode, gas_limit, condom_limits)?) } /// Gets the balance of the current address address (top of the stack). diff --git a/massa-module-cache/src/controller.rs b/massa-module-cache/src/controller.rs index 18d12ea3935..86ec59d4e0f 100644 --- a/massa-module-cache/src/controller.rs +++ b/massa-module-cache/src/controller.rs @@ -1,6 +1,6 @@ use massa_hash::Hash; use massa_models::prehash::BuildHashMapper; -use massa_sc_runtime::{Compiler, RuntimeModule}; +use massa_sc_runtime::{Compiler, CondomLimits, RuntimeModule}; use schnellru::{ByLength, LruMap}; use tracing::debug; @@ -40,12 +40,12 @@ impl ModuleCache { } /// Internal function to compile and build `ModuleInfo` - fn compile_cached(&mut self, bytecode: &[u8], hash: Hash) -> ModuleInfo { + fn compile_cached(&mut self, bytecode: &[u8], hash: Hash, condom_limits: CondomLimits) -> ModuleInfo { match RuntimeModule::new( bytecode, self.cfg.gas_costs.clone(), Compiler::CL, - self.cfg.condom_limits.clone(), + condom_limits, ) { Ok(module) => { debug!("compilation of module {} succeeded", hash); @@ -60,12 +60,12 @@ impl ModuleCache { } /// Save a new or an already existing module in the cache - pub fn save_module(&mut self, bytecode: &[u8]) { + pub fn save_module(&mut self, bytecode: &[u8], condom_limits: CondomLimits) { let hash = Hash::compute_from(bytecode); if let Some(hd_module_info) = self.hd_cache.get( hash, self.cfg.gas_costs.clone(), - self.cfg.condom_limits.clone(), + condom_limits.clone(), ) { debug!("save_module: {} present in hd", hash); self.lru_cache.insert(hash, hd_module_info); @@ -74,7 +74,7 @@ impl ModuleCache { self.hd_cache.insert(hash, lru_module_info); } else { debug!("save_module: {} missing", hash); - let module_info = self.compile_cached(bytecode, hash); + let module_info = self.compile_cached(bytecode, hash, condom_limits); self.hd_cache.insert(hash, module_info.clone()); self.lru_cache.insert(hash, module_info); } @@ -100,7 +100,7 @@ impl ModuleCache { /// * `ModuleInfo::Invalid` if the module is invalid /// * `ModuleInfo::Module` if the module is valid and has no delta /// * `ModuleInfo::ModuleAndDelta` if the module is valid and has a delta - fn load_module_info(&mut self, bytecode: &[u8]) -> ModuleInfo { + fn load_module_info(&mut self, bytecode: &[u8], condom_limits: CondomLimits) -> ModuleInfo { if bytecode.is_empty() { let error_msg = "load_module: bytecode is absent".to_string(); debug!(error_msg); @@ -122,14 +122,14 @@ impl ModuleCache { } else if let Some(hd_module_info) = self.hd_cache.get( hash, self.cfg.gas_costs.clone(), - self.cfg.condom_limits.clone(), + condom_limits.clone(), ) { debug!("load_module: {} missing in lru but present in hd", hash); self.lru_cache.insert(hash, hd_module_info.clone()); hd_module_info } else { debug!("load_module: {} missing", hash); - let module_info = self.compile_cached(bytecode, hash); + let module_info = self.compile_cached(bytecode, hash, condom_limits); self.hd_cache.insert(hash, module_info.clone()); self.lru_cache.insert(hash, module_info.clone()); module_info @@ -144,6 +144,7 @@ impl ModuleCache { &mut self, bytecode: &[u8], execution_gas: u64, + condom_limits: CondomLimits, ) -> Result { // Do not actually debit the instance creation cost from the provided gas // This is only supposed to be a check @@ -155,7 +156,7 @@ impl ModuleCache { )))?; // TODO: interesting but unimportant optim // remove max_instance_cost hard check if module is cached and has a delta - let module_info = self.load_module_info(bytecode); + let module_info = self.load_module_info(bytecode, condom_limits); let module = match module_info { ModuleInfo::Invalid(err) => { let err_msg = format!("invalid module: {}", err); @@ -184,6 +185,7 @@ impl ModuleCache { &self, bytecode: &[u8], limit: u64, + condom_limits: CondomLimits, ) -> Result { debug!("load_tmp_module"); // Do not actually debit the instance creation cost from the provided gas @@ -198,7 +200,7 @@ impl ModuleCache { bytecode, self.cfg.gas_costs.clone(), Compiler::SP, - self.cfg.condom_limits.clone(), + condom_limits, )?; Ok(module) } From 9ddf5ae522fa037a14b99c1e848ab296ab013ae0 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 7 Nov 2024 15:42:04 +0100 Subject: [PATCH 23/63] fmt --- massa-execution-worker/src/execution.rs | 55 +++++++++++--------- massa-execution-worker/src/interface_impl.rs | 11 ++-- massa-module-cache/src/controller.rs | 25 +++++---- 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 9495abda854..c15401a2ae3 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -949,7 +949,7 @@ impl ExecutionState { { // acquire write access to the context let mut context = context_guard!(self); - + execution_component_version = context.execution_component_version; // Set the call stack to a single element: // * the execution will happen in the context of the address of the operation's sender @@ -970,10 +970,10 @@ impl ExecutionState { }; // load the tmp module - let module = self - .module_cache - .read() - .load_tmp_module(bytecode, *max_gas, condom_limits.clone())?; + let module = + self.module_cache + .read() + .load_tmp_module(bytecode, *max_gas, condom_limits.clone())?; // run the VM let _res = massa_sc_runtime::run_main( &*self.execution_interface, @@ -1031,7 +1031,7 @@ impl ExecutionState { let mut context = context_guard!(self); execution_component_version = context.execution_component_version; - + // Set the call stack // This needs to be defined before anything can fail, so that the emitted event contains the right stack context.stack = vec![ @@ -1082,7 +1082,10 @@ impl ExecutionState { _ => self.config.condom_limits.clone(), }; - let module = self.module_cache.write().load_module(&bytecode, max_gas, condom_limits.clone())?; + let module = + self.module_cache + .write() + .load_module(&bytecode, max_gas, condom_limits.clone())?; let response = massa_sc_runtime::run_function( &*self.execution_interface, module, @@ -1201,16 +1204,17 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface let module = match context_guard!(self).execution_component_version { - 0 => self - .module_cache - .write() - .load_module(&bytecode, message.max_gas, CondomLimits::default())?, + 0 => self.module_cache.write().load_module( + &bytecode, + message.max_gas, + CondomLimits::default(), + )?, _ => { - let Ok(_module) = self - .module_cache - .write() - .load_module(&bytecode, message.max_gas, self.config.condom_limits.clone()) - else { + let Ok(_module) = self.module_cache.write().load_module( + &bytecode, + message.max_gas, + self.config.condom_limits.clone(), + ) else { let err = ExecutionError::RuntimeError( "could not load module for async execution".into(), ); @@ -1959,7 +1963,6 @@ impl ExecutionState { // run the interpreter according to the target type let exec_response = match req.target { ReadOnlyExecutionTarget::BytecodeExecution(bytecode) => { - let execution_component_version = execution_context.execution_component_version; { let mut context = context_guard!(self); @@ -1979,10 +1982,11 @@ impl ExecutionState { }; // load the tmp module - let module = self - .module_cache - .read() - .load_tmp_module(&bytecode, req.max_gas, condom_limits.clone())?; + let module = self.module_cache.read().load_tmp_module( + &bytecode, + req.max_gas, + condom_limits.clone(), + )?; // run the VM massa_sc_runtime::run_main( @@ -2039,10 +2043,11 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface - let module = self - .module_cache - .write() - .load_module(&bytecode, req.max_gas, condom_limits.clone())?; + let module = self.module_cache.write().load_module( + &bytecode, + req.max_gas, + condom_limits.clone(), + )?; let response = massa_sc_runtime::run_function( &*self.execution_interface, diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index d3e8609c12c..5d09cc2e40b 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -335,7 +335,6 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` CL compiled module & the remaining gas after loading the module fn get_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { - let execution_component_version = self.get_interface_version()?; let condom_limits = match execution_component_version { @@ -343,10 +342,11 @@ impl Interface for InterfaceImpl { _ => self.config.condom_limits.clone(), }; - Ok((context_guard!(self)) - .module_cache - .write() - .load_module(bytecode, gas_limit, condom_limits)?) + Ok((context_guard!(self)).module_cache.write().load_module( + bytecode, + gas_limit, + condom_limits, + )?) } /// Compile and return a temporary module @@ -354,7 +354,6 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` SP compiled module & the remaining gas after loading the module fn get_tmp_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { - let execution_component_version = self.get_interface_version()?; let condom_limits = match execution_component_version { diff --git a/massa-module-cache/src/controller.rs b/massa-module-cache/src/controller.rs index 86ec59d4e0f..ca877615410 100644 --- a/massa-module-cache/src/controller.rs +++ b/massa-module-cache/src/controller.rs @@ -40,7 +40,12 @@ impl ModuleCache { } /// Internal function to compile and build `ModuleInfo` - fn compile_cached(&mut self, bytecode: &[u8], hash: Hash, condom_limits: CondomLimits) -> ModuleInfo { + fn compile_cached( + &mut self, + bytecode: &[u8], + hash: Hash, + condom_limits: CondomLimits, + ) -> ModuleInfo { match RuntimeModule::new( bytecode, self.cfg.gas_costs.clone(), @@ -62,11 +67,10 @@ impl ModuleCache { /// Save a new or an already existing module in the cache pub fn save_module(&mut self, bytecode: &[u8], condom_limits: CondomLimits) { let hash = Hash::compute_from(bytecode); - if let Some(hd_module_info) = self.hd_cache.get( - hash, - self.cfg.gas_costs.clone(), - condom_limits.clone(), - ) { + if let Some(hd_module_info) = + self.hd_cache + .get(hash, self.cfg.gas_costs.clone(), condom_limits.clone()) + { debug!("save_module: {} present in hd", hash); self.lru_cache.insert(hash, hd_module_info); } else if let Some(lru_module_info) = self.lru_cache.get(hash) { @@ -119,11 +123,10 @@ impl ModuleCache { if let Some(lru_module_info) = self.lru_cache.get(hash) { debug!("load_module: {} present in lru", hash); lru_module_info - } else if let Some(hd_module_info) = self.hd_cache.get( - hash, - self.cfg.gas_costs.clone(), - condom_limits.clone(), - ) { + } else if let Some(hd_module_info) = + self.hd_cache + .get(hash, self.cfg.gas_costs.clone(), condom_limits.clone()) + { debug!("load_module: {} missing in lru but present in hd", hash); self.lru_cache.insert(hash, hd_module_info.clone()); hd_module_info From d40efaa9c78cc493e4f55a0423e08f17903d30e6 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 7 Nov 2024 15:51:45 +0100 Subject: [PATCH 24/63] Add versioning to max_recursive_calls_depth --- massa-execution-worker/src/interface_impl.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 5d09cc2e40b..5bf2e2b39a7 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -240,7 +240,11 @@ impl Interface for InterfaceImpl { context.recursion_counter += 1; - if context.recursion_counter > self.config.max_recursive_calls_depth { + let execution_component_version = self.get_interface_version()?; + + if execution_component_version > 0 + && context.recursion_counter > self.config.max_recursive_calls_depth + { bail!("recursion depth limit reached"); } From 7ed039ad7dbea5998415c3075fc2a9a5dca0f1fd Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 7 Nov 2024 19:16:29 +0100 Subject: [PATCH 25/63] Fix deadlock --- massa-execution-worker/src/interface_impl.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 5bf2e2b39a7..b4bd4e63236 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -236,12 +236,12 @@ impl Interface for InterfaceImpl { } fn increment_recursion_counter(&self) -> Result<()> { + let execution_component_version = self.get_interface_version()?; + let mut context = context_guard!(self); context.recursion_counter += 1; - let execution_component_version = self.get_interface_version()?; - if execution_component_version > 0 && context.recursion_counter > self.config.max_recursive_calls_depth { From 84c0e4c7a636a6a7abda0f40a4b8a781198c5764 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 8 Nov 2024 08:23:06 +0100 Subject: [PATCH 26/63] Add versioning to Fix potential ledger keys boundaries issue --- massa-ledger-worker/src/ledger_db.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/massa-ledger-worker/src/ledger_db.rs b/massa-ledger-worker/src/ledger_db.rs index 5ed7e804774..4214e3caba8 100644 --- a/massa-ledger-worker/src/ledger_db.rs +++ b/massa-ledger-worker/src/ledger_db.rs @@ -149,7 +149,7 @@ impl LedgerDB { // the incoming change deletes a ledger entry SetUpdateOrDelete::Delete => { // delete the entry, if it exists - self.delete_entry(&addr, batch); + self.delete_entry(&addr, batch, final_state_component_version); } } } @@ -302,7 +302,7 @@ impl LedgerDB { 0 => {} _ => { // Ensures any potential previous entry is fully deleted. - delete_datastore_entries(addr, &db, batch); + delete_datastore_entries(addr, &db, batch, final_state_component_version); } } @@ -452,7 +452,12 @@ impl LedgerDB { /// /// # Arguments /// * batch: the given operation batch to update - fn delete_entry(&self, addr: &Address, batch: &mut DBBatch) { + fn delete_entry( + &self, + addr: &Address, + batch: &mut DBBatch, + final_state_component_version: u32, + ) { let db = self.db.read(); // version @@ -476,7 +481,7 @@ impl LedgerDB { .expect(KEY_SER_ERROR); db.delete_key(batch, serialized_key); - delete_datastore_entries(addr, &db, batch); + delete_datastore_entries(addr, &db, batch, final_state_component_version); } } @@ -487,6 +492,7 @@ fn delete_datastore_entries( addr: &Address, db: &RwLockReadGuard>, batch: &mut std::collections::BTreeMap, Option>>, + final_state_component_version: u32, ) { // datastore let key_prefix = datastore_prefix_from_address(addr, &[]); @@ -496,7 +502,10 @@ fn delete_datastore_entries( STATE_CF, MassaIteratorMode::From(&key_prefix, MassaDirection::Forward), ) - .take_while(|(key, _)| key < &end_prefix(&key_prefix).unwrap()) + .take_while(|(key, _)| match final_state_component_version { + 0 => key <= &end_prefix(&key_prefix).unwrap(), + _ => key < &end_prefix(&key_prefix).unwrap(), + }) { db.delete_key(batch, serialized_key.to_vec()); } @@ -689,7 +698,7 @@ mod tests { // delete entry let mut batch = DBBatch::new(); - ledger_db.delete_entry(&addr, &mut batch); + ledger_db.delete_entry(&addr, &mut batch, 1); ledger_db .db .write() From 1acd921f86d37599bf9b6ebc29b897b50ace599e Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 12:57:56 +0100 Subject: [PATCH 27/63] Add versioning to gas costs --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48f44238e90..f260f012428 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2796,7 +2796,7 @@ dependencies = [ [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?branch=next_breaking_update#095d6253bf6c31e725f056c79b4df9994f39380e" +source = "git+https://github.com/massalabs/massa-sc-runtime?branch=next_breaking_update#aa8e91a640c962027a83441c432a5f40a080758f" dependencies = [ "anyhow", "as-ffi-bindings", @@ -4465,8 +4465,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.12.0", + "heck 0.5.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -4486,7 +4486,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.12.0", + "itertools 0.10.5", "proc-macro2 1.0.86", "quote 1.0.37", "syn 2.0.75", From a02f301e744b9ee5622590b07c93199281162c3f Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 13:20:43 +0100 Subject: [PATCH 28/63] Add versioning to Add additional verification for v & s value in evm_signature_verify --- massa-execution-worker/src/interface_impl.rs | 143 +++++++++++-------- 1 file changed, 86 insertions(+), 57 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index b4bd4e63236..2948c91261d 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -927,13 +927,18 @@ impl Interface for InterfaceImpl { signature_: &[u8], public_key_: &[u8], ) -> Result { + let execution_component_version = self.get_interface_version()?; + // check the signature length if signature_.len() != 65 { return Err(anyhow!("invalid signature length in evm_signature_verify")); } // parse the public key - let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; + let public_key = match execution_component_version { + 0 => libsecp256k1::PublicKey::parse_slice(public_key_, Some(libsecp256k1::PublicKeyFormat::Raw))?, + _ => libsecp256k1::PublicKey::parse_slice(public_key_, None)? + }; // build the message let prefix = format!("\x19Ethereum Signed Message:\n{}", message_.len()); @@ -947,34 +952,36 @@ impl Interface for InterfaceImpl { // s is the signature proof for R.x (32 bytes) // v is a recovery parameter used to ease the signature verification (1 byte) // see test_evm_verify for an example of its usage - let recovery_id: u8 = libsecp256k1::RecoveryId::parse_rpc(signature_[64])?.into(); - // Note: parse_rpc returns p - 27 and allow for 27, 28, 29, 30 - // restrict to only 27 & 28 (=> 0 & 1) - if recovery_id != 0 && recovery_id != 1 { - // Note: - // The v value in an EVM signature serves as a recovery ID, - // aiding in the recovery of the public key from the signature. - // Typically, v should be either 27 or 28 - // (or sometimes 0 or 1, depending on the implementation). - // Ensuring that v is within the expected range is crucial - // for correctly recovering the public key. - // the Ethereum yellow paper specifies only 27 and 28, requiring additional checks. - return Err(anyhow!( - "invalid recovery id value (v = {recovery_id}) in evm_signature_verify" - )); - } - let signature = libsecp256k1::Signature::parse_standard_slice(&signature_[..64])?; - // Note: - // The s value in an EVM signature should be in the lower half of the elliptic curve - // in order to prevent malleability attacks. - // If s is in the high-order range, it can be converted to its low-order equivalent, - // which should be enforced during signature verification. - if signature.s.is_high() { - return Err(anyhow!( - "High-Order s Value are prohibited in evm_get_pubkey_from_signature" - )); + if execution_component_version != 0 { + let recovery_id: u8 = libsecp256k1::RecoveryId::parse_rpc(signature_[64])?.into(); + // Note: parse_rpc returns p - 27 and allow for 27, 28, 29, 30 + // restrict to only 27 & 28 (=> 0 & 1) + if recovery_id != 0 && recovery_id != 1 { + // Note: + // The v value in an EVM signature serves as a recovery ID, + // aiding in the recovery of the public key from the signature. + // Typically, v should be either 27 or 28 + // (or sometimes 0 or 1, depending on the implementation). + // Ensuring that v is within the expected range is crucial + // for correctly recovering the public key. + // the Ethereum yellow paper specifies only 27 and 28, requiring additional checks. + return Err(anyhow!( + "invalid recovery id value (v = {recovery_id}) in evm_signature_verify" + )); + } + + // Note: + // The s value in an EVM signature should be in the lower half of the elliptic curve + // in order to prevent malleability attacks. + // If s is in the high-order range, it can be converted to its low-order equivalent, + // which should be enforced during signature verification. + if signature.s.is_high() { + return Err(anyhow!( + "High-Order s Value are prohibited in evm_get_pubkey_from_signature" + )); + } } // verify the signature @@ -1007,6 +1014,8 @@ impl Interface for InterfaceImpl { /// Get a raw secp256k1 public key from an EVM signature and the signed hash. fn evm_get_pubkey_from_signature(&self, hash_: &[u8], signature_: &[u8]) -> Result> { + let execution_component_version = self.get_interface_version()?; + // check the signature length if signature_.len() != 65 { return Err(anyhow!( @@ -1014,37 +1023,57 @@ impl Interface for InterfaceImpl { )); } - // parse the message - let message = libsecp256k1::Message::parse_slice(hash_)?; - - // parse the signature as being (r, s, v) use only r and s - let signature = libsecp256k1::Signature::parse_standard_slice(&signature_[..64])?; - - // Note: - // See evm_signature_verify explanation - if signature.s.is_high() { - return Err(anyhow!( - "High-Order s Value are prohibited in evm_get_pubkey_from_signature" - )); - } - - // parse v as a recovery id - let recovery_id = libsecp256k1::RecoveryId::parse_rpc(signature_[64])?; - - let recovery_id_: u8 = recovery_id.into(); - if recovery_id_ != 0 && recovery_id_ != 1 { - // Note: - // See evm_signature_verify explanation - return Err(anyhow!( - "invalid recovery id value (v = {recovery_id_}) in evm_get_pubkey_from_signature" - )); + match execution_component_version { + 0 => { + // parse the message + let message = libsecp256k1::Message::parse_slice(hash_).unwrap(); + + // parse the signature as being (r, s, v) use only r and s + let signature = libsecp256k1::Signature::parse_standard_slice(&signature_[..64]).unwrap(); + + // parse v as a recovery id + let recovery_id = libsecp256k1::RecoveryId::parse_rpc(signature_[64]).unwrap(); + + // recover the public key + let recovered = libsecp256k1::recover(&message, &signature, &recovery_id).unwrap(); + + // return its serialized value + Ok(recovered.serialize().to_vec()) + }, + _ => { + // parse the message + let message = libsecp256k1::Message::parse_slice(hash_)?; + + // parse the signature as being (r, s, v) use only r and s + let signature = libsecp256k1::Signature::parse_standard_slice(&signature_[..64])?; + + // Note: + // See evm_signature_verify explanation + if signature.s.is_high() { + return Err(anyhow!( + "High-Order s Value are prohibited in evm_get_pubkey_from_signature" + )); + } + + // parse v as a recovery id + let recovery_id = libsecp256k1::RecoveryId::parse_rpc(signature_[64])?; + + let recovery_id_: u8 = recovery_id.into(); + if recovery_id_ != 0 && recovery_id_ != 1 { + // Note: + // See evm_signature_verify explanation + return Err(anyhow!( + "invalid recovery id value (v = {recovery_id_}) in evm_get_pubkey_from_signature" + )); + } + + // recover the public key + let recovered = libsecp256k1::recover(&message, &signature, &recovery_id)?; + + // return its serialized value + Ok(recovered.serialize().to_vec()) + } } - - // recover the public key - let recovered = libsecp256k1::recover(&message, &signature, &recovery_id)?; - - // return its serialized value - Ok(recovered.serialize().to_vec()) } // Return true if the address is a User address, false if it is an SC address. From 8468653a3c6731f74d270423aabd67c5e43156fd Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 13:22:32 +0100 Subject: [PATCH 29/63] Update runtime dep --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f260f012428..6f74ab93622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2796,7 +2796,7 @@ dependencies = [ [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?branch=next_breaking_update#aa8e91a640c962027a83441c432a5f40a080758f" +source = "git+https://github.com/massalabs/massa-sc-runtime?branch=next_breaking_update#a87f3a33a1b9038a0fc6cf52c212156dcbc1246d" dependencies = [ "anyhow", "as-ffi-bindings", @@ -4465,8 +4465,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck 0.5.0", - "itertools 0.10.5", + "heck 0.4.1", + "itertools 0.12.0", "log", "multimap", "once_cell", @@ -4486,7 +4486,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.0", "proc-macro2 1.0.86", "quote 1.0.37", "syn 2.0.75", From 56c53ad2f7e4df4158db1f344620e80a03328321 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 13:27:59 +0100 Subject: [PATCH 30/63] Add comment relative to versioning of Update executed denunciations in StateChanges.apply method --- massa-execution-worker/src/interface_impl.rs | 14 +++++++++----- massa-final-state/src/state_changes.rs | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 2948c91261d..593d964dcd5 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -936,8 +936,11 @@ impl Interface for InterfaceImpl { // parse the public key let public_key = match execution_component_version { - 0 => libsecp256k1::PublicKey::parse_slice(public_key_, Some(libsecp256k1::PublicKeyFormat::Raw))?, - _ => libsecp256k1::PublicKey::parse_slice(public_key_, None)? + 0 => libsecp256k1::PublicKey::parse_slice( + public_key_, + Some(libsecp256k1::PublicKeyFormat::Raw), + )?, + _ => libsecp256k1::PublicKey::parse_slice(public_key_, None)?, }; // build the message @@ -971,7 +974,7 @@ impl Interface for InterfaceImpl { "invalid recovery id value (v = {recovery_id}) in evm_signature_verify" )); } - + // Note: // The s value in an EVM signature should be in the lower half of the elliptic curve // in order to prevent malleability attacks. @@ -1029,7 +1032,8 @@ impl Interface for InterfaceImpl { let message = libsecp256k1::Message::parse_slice(hash_).unwrap(); // parse the signature as being (r, s, v) use only r and s - let signature = libsecp256k1::Signature::parse_standard_slice(&signature_[..64]).unwrap(); + let signature = + libsecp256k1::Signature::parse_standard_slice(&signature_[..64]).unwrap(); // parse v as a recovery id let recovery_id = libsecp256k1::RecoveryId::parse_rpc(signature_[64]).unwrap(); @@ -1039,7 +1043,7 @@ impl Interface for InterfaceImpl { // return its serialized value Ok(recovered.serialize().to_vec()) - }, + } _ => { // parse the message let message = libsecp256k1::Message::parse_slice(hash_)?; diff --git a/massa-final-state/src/state_changes.rs b/massa-final-state/src/state_changes.rs index c524c7882d1..3a67aa7e8a4 100644 --- a/massa-final-state/src/state_changes.rs +++ b/massa-final-state/src/state_changes.rs @@ -217,6 +217,7 @@ impl StateChanges { self.pos_changes.extend(changes.pos_changes); self.executed_ops_changes .extend(changes.executed_ops_changes); + // Note: no need to version the changes, as this function is not used in the codebase self.executed_denunciations_changes .extend(changes.executed_denunciations_changes); self.execution_trail_hash_change From a862f23003df1b4f40f7257fff0800e8bec9e7b4 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 14:16:33 +0100 Subject: [PATCH 31/63] Add versioning to Limit event msg size & event number per operation --- massa-execution-exports/src/settings.rs | 4 +- .../src/test_exports/config.rs | 3 +- massa-execution-worker/src/interface_impl.rs | 50 +++++++++++++++---- massa-execution-worker/src/tests/interface.rs | 6 +-- massa-models/src/config/constants.rs | 4 +- massa-node/src/main.rs | 10 ++-- 6 files changed, 58 insertions(+), 19 deletions(-) diff --git a/massa-execution-exports/src/settings.rs b/massa-execution-exports/src/settings.rs index 9a3cd4cd4d7..a2ef08f720d 100644 --- a/massa-execution-exports/src/settings.rs +++ b/massa-execution-exports/src/settings.rs @@ -91,7 +91,9 @@ pub struct ExecutionConfig { /// slot execution outputs channel capacity pub broadcast_slot_execution_output_channel_capacity: usize, /// max size of event data, in bytes - pub max_event_size: usize, + pub max_event_size_v0: usize, + /// max size of event data, in bytes + pub max_event_size_v1: usize, /// chain id pub chain_id: u64, /// whether slot execution traces broadcast is enabled diff --git a/massa-execution-exports/src/test_exports/config.rs b/massa-execution-exports/src/test_exports/config.rs index 7b644abab83..09aee2abf4b 100644 --- a/massa-execution-exports/src/test_exports/config.rs +++ b/massa-execution-exports/src/test_exports/config.rs @@ -68,7 +68,8 @@ impl Default for ExecutionConfig { denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS, broadcast_enabled: true, broadcast_slot_execution_output_channel_capacity: 5000, - max_event_size: 512, + max_event_size_v0: 50_000, + max_event_size_v1: 512, max_event_per_operation: 25, max_function_length: 1000, max_parameter_length: 1000, diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 593d964dcd5..0724f3eccfe 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1204,7 +1204,13 @@ impl Interface for InterfaceImpl { /// /// [DeprecatedByNewRuntime] Replaced by `get_current_slot` fn generate_event(&self, data: String) -> Result<()> { - if data.len() > self.config.max_event_size { + let execution_component_version = self.get_interface_version()?; + let max_event_size = match execution_component_version { + 0 => self.config.max_event_size_v0, + _ => self.config.max_event_size_v1, + }; + + if data.len() > max_event_size { bail!("Event data size is too large"); }; @@ -1220,14 +1226,15 @@ impl Interface for InterfaceImpl { is_final: None, is_error: None, }; - let event_per_op = context - .events - .get_filtered_sc_output_events_iter(&event_filter) - .count(); - if event_per_op >= self.config.max_event_per_operation { - bail!("Too many event for this operation"); + if execution_component_version > 0 { + let event_per_op = context + .events + .get_filtered_sc_output_events_iter(&event_filter) + .count(); + if event_per_op >= self.config.max_event_per_operation { + bail!("Too many event for this operation"); + } } - context.event_emit(event); Ok(()) } @@ -1237,13 +1244,38 @@ impl Interface for InterfaceImpl { /// # Arguments: /// data: the bytes_array data that is the payload of the event fn generate_event_wasmv1(&self, data: Vec) -> Result<()> { - if data.len() > self.config.max_event_size { + let execution_component_version = self.get_interface_version()?; + let max_event_size = match execution_component_version { + 0 => self.config.max_event_size_v0, + _ => self.config.max_event_size_v1, + }; + + if data.len() > max_event_size { bail!("Event data size is too large"); }; let data_str = String::from_utf8(data.clone()).unwrap_or(format!("{:?}", data)); let mut context = context_guard!(self); let event = context.event_create(data_str, false); + + let event_filter = EventFilter { + start: None, + end: None, + emitter_address: None, + original_caller_address: None, + original_operation_id: event.context.origin_operation_id, + is_final: None, + is_error: None, + }; + if execution_component_version > 0 { + let event_per_op = context + .events + .get_filtered_sc_output_events_iter(&event_filter) + .count(); + if event_per_op >= self.config.max_event_per_operation { + bail!("Too many event for this operation"); + } + } context.event_emit(event); Ok(()) diff --git a/massa-execution-worker/src/tests/interface.rs b/massa-execution-worker/src/tests/interface.rs index fd167f6ab92..520a7410142 100644 --- a/massa-execution-worker/src/tests/interface.rs +++ b/massa-execution-worker/src/tests/interface.rs @@ -128,7 +128,7 @@ fn test_emit_event_too_large() { // emit 2 events and check that the 2nd event is rejected (because the msg is too large) let config = ExecutionConfig { - max_event_size: 10, + max_event_size_v0: 10, ..Default::default() }; @@ -138,9 +138,9 @@ fn test_emit_event_too_large() { Some(config.clone()), ); - let res = interface.generate_event("a".repeat(config.max_event_size).to_string()); + let res = interface.generate_event("a".repeat(config.max_event_size_v0).to_string()); assert!(res.is_ok()); - let res_2 = interface.generate_event("b".repeat(config.max_event_size + 1).to_string()); + let res_2 = interface.generate_event("b".repeat(config.max_event_size_v0 + 1).to_string()); assert!(res_2.is_err()); println!("res_2: {:?}", res_2); if let Err(e) = res_2 { diff --git a/massa-models/src/config/constants.rs b/massa-models/src/config/constants.rs index a3c40654c46..86ad6dcd243 100644 --- a/massa-models/src/config/constants.rs +++ b/massa-models/src/config/constants.rs @@ -309,7 +309,9 @@ pub const ASYNC_MSG_CST_GAS_COST: u64 = 750_000; /// Gas used by a base operation (transaction, roll buy, roll sell) pub const BASE_OPERATION_GAS_COST: u64 = 800_000; // approx MAX_GAS_PER_BLOCK / MAX_OPERATIONS_PER_BLOCK /// Maximum event size in bytes -pub const MAX_EVENT_DATA_SIZE: usize = 512; +pub const MAX_EVENT_DATA_SIZE_V0: usize = 50_000; +/// Maximum event size in bytes +pub const MAX_EVENT_DATA_SIZE_V1: usize = 512; /// Maximum event number that can be emitted for an operation pub const MAX_EVENT_PER_OPERATION: usize = 25; /// Maximum number of recursion for calls diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 91385776f2c..6345808755a 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -88,9 +88,10 @@ use massa_models::config::constants::{ use massa_models::config::{ BASE_OPERATION_GAS_COST, CHAINID, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS, MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE, - MAX_EVENT_DATA_SIZE, MAX_EVENT_PER_OPERATION, MAX_MESSAGE_SIZE, MAX_RECURSIVE_CALLS_DEPTH, - MAX_RUNTIME_MODULE_CUSTOM_SECTION_DATA_LEN, MAX_RUNTIME_MODULE_CUSTOM_SECTION_LEN, - MAX_RUNTIME_MODULE_EXPORTS, MAX_RUNTIME_MODULE_FUNCTIONS, MAX_RUNTIME_MODULE_FUNCTION_NAME_LEN, + MAX_EVENT_DATA_SIZE_V0, MAX_EVENT_DATA_SIZE_V1, MAX_EVENT_PER_OPERATION, MAX_MESSAGE_SIZE, + MAX_RECURSIVE_CALLS_DEPTH, MAX_RUNTIME_MODULE_CUSTOM_SECTION_DATA_LEN, + MAX_RUNTIME_MODULE_CUSTOM_SECTION_LEN, MAX_RUNTIME_MODULE_EXPORTS, + MAX_RUNTIME_MODULE_FUNCTIONS, MAX_RUNTIME_MODULE_FUNCTION_NAME_LEN, MAX_RUNTIME_MODULE_GLOBAL_INITIALIZER, MAX_RUNTIME_MODULE_IMPORTS, MAX_RUNTIME_MODULE_MEMORIES, MAX_RUNTIME_MODULE_NAME_LEN, MAX_RUNTIME_MODULE_PASSIVE_DATA, MAX_RUNTIME_MODULE_PASSIVE_ELEMENT, MAX_RUNTIME_MODULE_SIGNATURE_LEN, MAX_RUNTIME_MODULE_TABLE, @@ -528,7 +529,8 @@ async fn launch( broadcast_slot_execution_output_channel_capacity: SETTINGS .execution .broadcast_slot_execution_output_channel_capacity, - max_event_size: MAX_EVENT_DATA_SIZE, + max_event_size_v0: MAX_EVENT_DATA_SIZE_V0, + max_event_size_v1: MAX_EVENT_DATA_SIZE_V1, max_function_length: MAX_FUNCTION_NAME_LENGTH, max_parameter_length: MAX_PARAMETERS_SIZE, chain_id: *CHAINID, From d3c3bf0571bd7268c641c88e57220cacaa0f86dd Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 14:53:18 +0100 Subject: [PATCH 32/63] Fix versioning for interface implementation tests --- massa-execution-worker/src/interface_impl.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 0724f3eccfe..5a7df26c9b3 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -99,7 +99,10 @@ impl InterfaceImpl { use massa_module_cache::{config::ModuleCacheConfig, controller::ModuleCache}; use massa_pos_exports::SelectorConfig; use massa_pos_worker::start_selector_worker; - use massa_versioning::versioning::{MipStatsConfig, MipStore}; + use massa_versioning::{ + mips::get_mip_list, + versioning::{MipStatsConfig, MipStore}, + }; use parking_lot::RwLock; use tempfile::TempDir; @@ -141,8 +144,8 @@ impl InterfaceImpl { block_count_considered: MIP_STORE_STATS_BLOCK_CONSIDERED, warn_announced_version_ratio: Ratio::new_raw(30, 100), }; - let mip_store = - MipStore::try_from(([], mip_stats_config)).expect("Cannot create an empty MIP store"); + let mip_store = MipStore::try_from((get_mip_list(), mip_stats_config)) + .expect("Cannot create an empty MIP store"); let mut execution_context = ExecutionContext::new( config.clone(), From 315b8508990caaddd162b25ed61ab69d9d8a5c1f Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Fri, 15 Nov 2024 16:01:55 +0100 Subject: [PATCH 33/63] Fix test for max_event size --- massa-execution-worker/src/tests/interface.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/massa-execution-worker/src/tests/interface.rs b/massa-execution-worker/src/tests/interface.rs index 520a7410142..0fe73f62f05 100644 --- a/massa-execution-worker/src/tests/interface.rs +++ b/massa-execution-worker/src/tests/interface.rs @@ -128,7 +128,7 @@ fn test_emit_event_too_large() { // emit 2 events and check that the 2nd event is rejected (because the msg is too large) let config = ExecutionConfig { - max_event_size_v0: 10, + max_event_size_v1: 10, ..Default::default() }; @@ -138,9 +138,9 @@ fn test_emit_event_too_large() { Some(config.clone()), ); - let res = interface.generate_event("a".repeat(config.max_event_size_v0).to_string()); + let res = interface.generate_event("a".repeat(config.max_event_size_v1).to_string()); assert!(res.is_ok()); - let res_2 = interface.generate_event("b".repeat(config.max_event_size_v0 + 1).to_string()); + let res_2 = interface.generate_event("b".repeat(config.max_event_size_v1 + 1).to_string()); assert!(res_2.is_err()); println!("res_2: {:?}", res_2); if let Err(e) = res_2 { From ce0e14bcc7724035c52856f9a8d12437752b100b Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 19 Nov 2024 12:23:04 +0100 Subject: [PATCH 34/63] Add versioning to Early set operation id --- massa-execution-worker/src/execution.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index c15401a2ae3..7cec1025122 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -441,6 +441,7 @@ impl ExecutionState { // lock execution context let mut context = context_guard!(self); + let execution_component_version = context.execution_component_version; // ignore the operation if it was already executed if context.is_op_executed(&operation_id) { @@ -462,7 +463,9 @@ impl ExecutionState { // set the context origin operation ID // Note: set operation ID early as if context.transfer_coins fails, event_create will use // operation ID in the event message - context.origin_operation_id = Some(operation_id); + if execution_component_version >= 1 { + context.origin_operation_id = Some(operation_id); + } // debit the fee from the operation sender if let Err(err) = @@ -483,6 +486,10 @@ impl ExecutionState { // set the creator address context.creator_address = Some(operation.content_creator_address); + if execution_component_version == 0 { + context.origin_operation_id = Some(operation_id); + } + Ok(context_snapshot) } From 82ed83785c93fb6434416bb99d1fd935ebba9188 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 28 Nov 2024 11:56:01 +0100 Subject: [PATCH 35/63] Review comment: propagate error message and better match --- massa-execution-worker/src/execution.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 7cec1025122..43cc90f7c0b 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1217,20 +1217,23 @@ impl ExecutionState { CondomLimits::default(), )?, _ => { - let Ok(_module) = self.module_cache.write().load_module( + match self.module_cache.write().load_module( &bytecode, message.max_gas, self.config.condom_limits.clone(), - ) else { - let err = ExecutionError::RuntimeError( - "could not load module for async execution".into(), - ); - let mut context = context_guard!(self); - context.reset_to_snapshot(context_snapshot, err.clone()); - context.cancel_async_message(&message); - return Err(err); - }; - _module + ) { + Ok(module) => module, + Err(err) => { + let err = ExecutionError::RuntimeError(format!( + "could not load module for async execution: {}", + err + )); + let mut context = context_guard!(self); + context.reset_to_snapshot(context_snapshot, err.clone()); + context.cancel_async_message(&message); + return Err(err); + } + } } }; From 95b720d4323e615cc3cb90f1df34e2643e0b5d42 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 28 Nov 2024 11:56:21 +0100 Subject: [PATCH 36/63] Review comment: clean out match --- massa-ledger-worker/src/ledger_db.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/massa-ledger-worker/src/ledger_db.rs b/massa-ledger-worker/src/ledger_db.rs index 4214e3caba8..1b78f691935 100644 --- a/massa-ledger-worker/src/ledger_db.rs +++ b/massa-ledger-worker/src/ledger_db.rs @@ -298,12 +298,9 @@ impl LedgerDB { ) { let db = self.db.read(); - match final_state_component_version { - 0 => {} - _ => { - // Ensures any potential previous entry is fully deleted. - delete_datastore_entries(addr, &db, batch, final_state_component_version); - } + if final_state_component_version > 0 { + // Ensures any potential previous entry is fully deleted. + delete_datastore_entries(addr, &db, batch, final_state_component_version); } // Version From fb37498e2b1af2cc79d6d6f183f89d6df50d18e7 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 28 Nov 2024 12:49:15 +0100 Subject: [PATCH 37/63] Review comment: Add helper function to query versioned condom_limits in context --- massa-execution-worker/src/context.rs | 8 +++++ massa-execution-worker/src/execution.rs | 32 ++++-------------- massa-execution-worker/src/interface_impl.rs | 35 +++++++++----------- 3 files changed, 29 insertions(+), 46 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 7fafcaefbf7..8c50a7ac7e2 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -1330,6 +1330,14 @@ impl ExecutionContext { ))), } } + + /// Get the condom limits to pass to the VM depending on the current execution component version + pub fn get_condom_limits(&self) -> CondomLimits { + match self.execution_component_version { + 0 => Default::default(), + _ => self.config.condom_limits.clone(), + } + } } /// Generate the execution trail hash diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 43cc90f7c0b..1025c7d6bca 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -952,12 +952,12 @@ impl ExecutionState { _ => panic!("unexpected operation type"), }; - let execution_component_version; + let condom_limits; { // acquire write access to the context let mut context = context_guard!(self); - execution_component_version = context.execution_component_version; + condom_limits = context.get_condom_limits(); // Set the call stack to a single element: // * the execution will happen in the context of the address of the operation's sender // * the context will give the operation's sender write access to its own ledger entry @@ -971,11 +971,6 @@ impl ExecutionState { }]; }; - let condom_limits = match execution_component_version { - 0 => Default::default(), - _ => self.config.condom_limits.clone(), - }; - // load the tmp module let module = self.module_cache @@ -1032,12 +1027,12 @@ impl ExecutionState { // prepare the current slot context for executing the operation let bytecode; - let execution_component_version; + let condom_limits; { // acquire write access to the context let mut context = context_guard!(self); - execution_component_version = context.execution_component_version; + condom_limits = context.get_condom_limits(); // Set the call stack // This needs to be defined before anything can fail, so that the emitted event contains the right stack @@ -1084,11 +1079,6 @@ impl ExecutionState { // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface - let condom_limits = match execution_component_version { - 0 => Default::default(), - _ => self.config.condom_limits.clone(), - }; - let module = self.module_cache .write() @@ -1973,7 +1963,7 @@ impl ExecutionState { // run the interpreter according to the target type let exec_response = match req.target { ReadOnlyExecutionTarget::BytecodeExecution(bytecode) => { - let execution_component_version = execution_context.execution_component_version; + let condom_limits = execution_context.get_condom_limits(); { let mut context = context_guard!(self); *context = execution_context; @@ -1986,11 +1976,6 @@ impl ExecutionState { } } - let condom_limits = match execution_component_version { - 0 => CondomLimits::default(), - _ => self.config.condom_limits.clone(), - }; - // load the tmp module let module = self.module_cache.read().load_tmp_module( &bytecode, @@ -2023,7 +2008,7 @@ impl ExecutionState { .unwrap_or_default() .0; - let execution_component_version = execution_context.execution_component_version; + let condom_limits = execution_context.get_condom_limits(); { let mut context = context_guard!(self); *context = execution_context; @@ -2046,11 +2031,6 @@ impl ExecutionState { } } - let condom_limits = match execution_component_version { - 0 => CondomLimits::default(), - _ => self.config.condom_limits.clone(), - }; - // load and execute the compiled module // IMPORTANT: do not keep a lock here as `run_function` uses the `get_module` interface let module = self.module_cache.write().load_module( diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 5a7df26c9b3..c7ef1bf7433 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -21,7 +21,6 @@ use massa_models::{ use massa_proto_rs::massa::model::v1::{ AddressCategory, ComparisonResult, NativeAmount, NativeTime, }; -use massa_sc_runtime::CondomLimits; use massa_sc_runtime::RuntimeModule; use massa_sc_runtime::{Interface, InterfaceClone}; use massa_signature::PublicKey; @@ -342,18 +341,15 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` CL compiled module & the remaining gas after loading the module fn get_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { - let execution_component_version = self.get_interface_version()?; + let context = context_guard!(self); + let condom_limits = context.get_condom_limits(); - let condom_limits = match execution_component_version { - 0 => CondomLimits::default(), - _ => self.config.condom_limits.clone(), - }; + let ret = context + .module_cache + .write() + .load_module(bytecode, gas_limit, condom_limits)?; - Ok((context_guard!(self)).module_cache.write().load_module( - bytecode, - gas_limit, - condom_limits, - )?) + Ok(ret) } /// Compile and return a temporary module @@ -361,17 +357,16 @@ impl Interface for InterfaceImpl { /// # Returns /// A `massa-sc-runtime` SP compiled module & the remaining gas after loading the module fn get_tmp_module(&self, bytecode: &[u8], gas_limit: u64) -> Result { - let execution_component_version = self.get_interface_version()?; + let context = context_guard!(self); + let condom_limits = context.get_condom_limits(); - let condom_limits = match execution_component_version { - 0 => CondomLimits::default(), - _ => self.config.condom_limits.clone(), - }; + let ret = + context + .module_cache + .write() + .load_tmp_module(bytecode, gas_limit, condom_limits)?; - Ok((context_guard!(self)) - .module_cache - .write() - .load_tmp_module(bytecode, gas_limit, condom_limits)?) + Ok(ret) } /// Gets the balance of the current address address (top of the stack). From b3e9ba549d57af56b0f4bc87920dd41f1aae0fdc Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 28 Nov 2024 14:45:23 +0100 Subject: [PATCH 38/63] Add comment --- massa-execution-worker/src/interface_impl.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index c7ef1bf7433..cc41a3302a0 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1695,6 +1695,9 @@ impl Interface for InterfaceImpl { let addr = Address::from_str(to_check)?; let execution_component_version = context_guard!(self).execution_component_version; + // Fixed behavior for this ABI in https://github.com/massalabs/massa/pull/4728 + // We keep the previous (bugged) code if the execution component version is 0 + // to avoid a breaking change match (addr, execution_component_version) { (Address::User(_), 0) => Ok(AddressCategory::ScAddress), (Address::SC(_), 0) => Ok(AddressCategory::UserAddress), From 907b86ff0a949dee7ec862edf5a0a8929213e90c Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 28 Nov 2024 15:26:12 +0100 Subject: [PATCH 39/63] Add dummy info for MIP_info --- massa-versioning/src/mips.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/massa-versioning/src/mips.rs b/massa-versioning/src/mips.rs index f161ef326dd..a8408616477 100644 --- a/massa-versioning/src/mips.rs +++ b/massa-versioning/src/mips.rs @@ -18,11 +18,11 @@ pub fn get_mip_list() -> [(MipInfo, MipState); 1] { (MipComponent::Execution, 1), (MipComponent::FinalState, 1), ]), - start: MassaTime::from_millis(0), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); - timeout: MassaTime::from_millis(0), // TODO: set when known + start: MassaTime::from_utc_ymd_hms(2024, 11, 28, 2, 0, 0).unwrap(), // TODO: set when known, ex: MassaTime::from_utc_ymd_hms(2024, 7, 10, 15, 0, 0).unwrap(); + timeout: MassaTime::from_utc_ymd_hms(2025, 11, 28, 2, 0, 0).unwrap(), // TODO: set when known activation_delay: MassaTime::from_millis(3 * 24 * 60 * 60 * 1000), // TODO: set when known, 3 days as an example }, - MipState::new(MassaTime::from_millis(0)), + MipState::new(MassaTime::from_utc_ymd_hms(2024, 11, 28, 0, 0, 0).unwrap()), ), // TODO: set when known, (when the MIP becomes defined, e.g. when merged to main branch) ]; From c46bf0c106fc153335c19da98170585b2aa7a7e4 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Wed, 4 Dec 2024 09:29:17 +0100 Subject: [PATCH 40/63] Change .unwrap() to .expect() for get_block_timestamp --- massa-execution-worker/src/context.rs | 6 +++--- massa-execution-worker/src/speculative_async_pool.rs | 3 ++- massa-final-state/src/final_state.rs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 8c50a7ac7e2..2cf1a594d00 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -221,7 +221,7 @@ impl ExecutionContext { config.genesis_timestamp, slot, ) - .unwrap(); + .expect("Time overflow when getting block slot timestamp for MIP"); ExecutionContext { speculative_ledger: SpeculativeLedger::new( @@ -370,7 +370,7 @@ impl ExecutionContext { config.genesis_timestamp, slot, ) - .unwrap(); + .expect("Time overflow when getting block slot timestamp for MIP"); // return readonly context ExecutionContext { @@ -462,7 +462,7 @@ impl ExecutionContext { config.genesis_timestamp, slot, ) - .unwrap(); + .expect("Time overflow when getting block slot timestamp for MIP"); // return active slot execution context ExecutionContext { diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index f1ed748b995..c850a263e9c 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -331,7 +331,8 @@ impl SpeculativeAsyncPool { } fn get_execution_component_version(&self, slot: &Slot) -> u32 { - let ts = get_block_slot_timestamp(THREAD_COUNT, T0, *GENESIS_TIMESTAMP, *slot).unwrap(); + let ts = get_block_slot_timestamp(THREAD_COUNT, T0, *GENESIS_TIMESTAMP, *slot) + .expect("Time overflow when getting block slot timestamp for MIP"); self.final_state .read() diff --git a/massa-final-state/src/final_state.rs b/massa-final-state/src/final_state.rs index 8bc2b5e333e..5d749f69ba3 100644 --- a/massa-final-state/src/final_state.rs +++ b/massa-final-state/src/final_state.rs @@ -434,7 +434,7 @@ impl FinalState { self.config.genesis_timestamp, slot, ) - .unwrap(); + .expect("Time overflow when getting block slot timestamp for MIP"); let final_state_component_version = self .get_mip_store() From 6b03ad8f64c4725bc085a703309d7590e676fea5 Mon Sep 17 00:00:00 2001 From: modship Date: Thu, 5 Dec 2024 17:50:19 +0100 Subject: [PATCH 41/63] Disable deferred calls abi if exec_comp = 0 --- massa-execution-worker/src/context.rs | 1 + massa-execution-worker/src/execution.rs | 92 ++++++++++++-------- massa-execution-worker/src/interface_impl.rs | 46 +++++++--- 3 files changed, 90 insertions(+), 49 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index e8e91a35717..d08c6fce666 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -1087,6 +1087,7 @@ impl ExecutionContext { executed_ops_changes: self.speculative_executed_ops.take(), executed_denunciations_changes: self.speculative_executed_denunciations.take(), execution_trail_hash_change: SetOrKeep::Set(self.execution_trail_hash), + deferred_call_changes: self.speculative_deferred_calls.take(), }; std::mem::take(&mut self.opt_block_id); ExecutionOutput { diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index d40aae1b142..c1103563b69 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1356,6 +1356,7 @@ impl ExecutionState { let module = self.module_cache.write().load_module( &bytecode, call.get_effective_gas(self.config.deferred_calls_config.call_cst_gas_cost), + self.config.condom_limits.clone(), )?; let response = massa_sc_runtime::run_function( &*self.execution_interface, @@ -1454,39 +1455,57 @@ impl ExecutionState { self.mip_store.clone(), ); - // Deferred calls - let calls = execution_context.deferred_calls_advance_slot(*slot); + let execution_version = execution_context.execution_component_version; - // Apply the created execution context for slot execution - *context_guard!(self) = execution_context; + let mut deferred_calls_slot_gas = 0; - for (id, call) in calls.slot_calls { - let cancelled = call.cancelled; - match self.execute_deferred_call(&id, call) { - Ok(_exec) => { - if cancelled { - continue; - } - info!("executed deferred call: {:?}", id); - cfg_if::cfg_if! { - if #[cfg(feature = "execution-trace")] { - // Safe to unwrap - slot_trace.deferred_call_stacks.push(_exec.traces.unwrap().0); - } else if #[cfg(feature = "execution-info")] { - slot_trace.deferred_call_stacks.push(_exec.traces.clone().unwrap().0); - exec_info.deferred_calls_messages.push(Ok(_exec)); + // deferred calls execution + + match execution_version { + 0 => { + // Apply the created execution context for slot execution + *context_guard!(self) = execution_context; + } + _ => { + // Deferred calls + let calls = execution_context.deferred_calls_advance_slot(*slot); + + deferred_calls_slot_gas = calls.effective_slot_gas; + + // Apply the created execution context for slot execution + *context_guard!(self) = execution_context; + + for (id, call) in calls.slot_calls { + let cancelled = call.cancelled; + match self.execute_deferred_call(&id, call) { + Ok(_exec) => { + if cancelled { + continue; + } + info!("executed deferred call: {:?}", id); + cfg_if::cfg_if! { + if #[cfg(feature = "execution-trace")] { + // Safe to unwrap + slot_trace.deferred_call_stacks.push(_exec.traces.unwrap().0); + } else if #[cfg(feature = "execution-info")] { + slot_trace.deferred_call_stacks.push(_exec.traces.clone().unwrap().0); + exec_info.deferred_calls_messages.push(Ok(_exec)); + } + } + } + Err(err) => { + let msg = format!("failed executing deferred call: {}", err); + #[cfg(feature = "execution-info")] + exec_info.deferred_calls_messages.push(Err(msg.clone())); + dbg!(msg); } } } - Err(err) => { - let msg = format!("failed executing deferred call: {}", err); - #[cfg(feature = "execution-info")] - exec_info.deferred_calls_messages.push(Err(msg.clone())); - dbg!(msg); - } } } + // Block execution + let mut block_info: Option = None; // Set block gas (max_gas_per_block - gas used by deferred calls) let mut remaining_block_gas = self.config.max_gas_per_block; @@ -1844,18 +1863,16 @@ impl ExecutionState { context_guard!(self).update_production_stats(&producer_addr, *slot, None); } - let execution_version = execution_context.execution_component_version; + // Async msg execution + match execution_version { 0 => { // Get asynchronous messages to execute - let messages = execution_context.take_async_batch_v0( + let messages = context_guard!(self).take_async_batch_v0( self.config.max_async_gas, self.config.async_msg_cst_gas_cost, ); - // Apply the created execution context for slot execution - *context_guard!(self) = execution_context; - // Try executing asynchronous messages. // Effects are cancelled on failure and the sender is reimbursed. for (opt_bytecode, message) in messages { @@ -1882,14 +1899,19 @@ impl ExecutionState { } _ => { // Get asynchronous messages to execute - let messages = execution_context.take_async_batch_v1( - self.config.max_async_gas, + // The gas available for async messages is the remaining block gas + async remaining gas (max_async - gas used by deferred calls) + let async_msg_gas_available = self + .config + .max_async_gas + .saturating_sub(deferred_calls_slot_gas) + .saturating_add(remaining_block_gas); + + // Get asynchronous messages to execute + let messages = context_guard!(self).take_async_batch_v1( + async_msg_gas_available, self.config.async_msg_cst_gas_cost, ); - // Apply the created execution context for slot execution - *context_guard!(self) = execution_context; - // Try executing asynchronous messages. // Effects are cancelled on failure and the sender is reimbursed. for (_message_id, message) in messages { diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 4e148236bc2..45215853fbb 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1538,24 +1538,30 @@ impl Interface for InterfaceImpl { params_size: u64, ) -> Result<(bool, u64)> { // write-lock context - let context = context_guard!(self); - let current_slot = context.slot; + match context.execution_component_version { + 0 => { + bail!("Deferred calls are not supported in this execution component version") + } + _ => { + let current_slot = context.slot; - let target_slot = Slot::new(target_slot.0, target_slot.1); + let target_slot = Slot::new(target_slot.0, target_slot.1); - let gas_request = - gas_limit.saturating_add(self.config.deferred_calls_config.call_cst_gas_cost); + let gas_request = + gas_limit.saturating_add(self.config.deferred_calls_config.call_cst_gas_cost); - match context.deferred_calls_compute_call_fee( - target_slot, - gas_request, - current_slot, - params_size, - ) { - Ok(fee) => Ok((true, fee.to_raw())), - Err(_) => Ok((false, 0)), + match context.deferred_calls_compute_call_fee( + target_slot, + gas_request, + current_slot, + params_size, + ) { + Ok(fee) => Ok((true, fee.to_raw())), + Err(_) => Ok((false, 0)), + } + } } } @@ -1580,6 +1586,10 @@ impl Interface for InterfaceImpl { params: &[u8], coins: u64, ) -> Result { + if context_guard!(self).execution_component_version == 0 { + bail!("Deferred calls are not supported in this execution component version") + } + // This function spends coins + deferred_call_quote(target_slot, max_gas).unwrap() from the caller, fails if the balance is insufficient or if the quote would return None. let target_addr = Address::from_str(target_addr)?; @@ -1641,8 +1651,12 @@ impl Interface for InterfaceImpl { /// true if the call exists, false otherwise fn deferred_call_exists(&self, id: &str) -> Result { // write-lock context - let call_id = DeferredCallId::from_str(id)?; let context = context_guard!(self); + if context.execution_component_version == 0 { + bail!("Deferred calls are not supported in this execution component version") + } + + let call_id = DeferredCallId::from_str(id)?; Ok(context.deferred_call_exists(&call_id)) } @@ -1655,6 +1669,10 @@ impl Interface for InterfaceImpl { let mut context = context_guard!(self); + if context.execution_component_version == 0 { + bail!("Deferred calls are not supported in this execution component version") + } + // Can only be called by the creator of the deferred call. let caller = context.get_current_address()?; From 2c3806237a4c18ac94309e92b9a8eab15af37bd2 Mon Sep 17 00:00:00 2001 From: modship Date: Mon, 9 Dec 2024 14:30:07 +0100 Subject: [PATCH 42/63] deferred calls const gas cost --- massa-node/base_config/gas_costs/abi_gas_costs.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/massa-node/base_config/gas_costs/abi_gas_costs.json b/massa-node/base_config/gas_costs/abi_gas_costs.json index 5c1df9f465f..5ed2283dd5e 100644 --- a/massa-node/base_config/gas_costs/abi_gas_costs.json +++ b/massa-node/base_config/gas_costs/abi_gas_costs.json @@ -12,10 +12,10 @@ "assembly_script_date_now": 71, "assembly_script_delete_data": 196, "assembly_script_delete_data_for": 220, - "assembly_script_deferred_call_cancel": 11, - "assembly_script_deferred_call_exists": 11, - "assembly_script_get_deferred_call_quote": 11, - "assembly_script_deferred_call_register": 11, + "assembly_script_deferred_call_cancel": 833, + "assembly_script_deferred_call_exists": 1316, + "assembly_script_get_deferred_call_quote": 244, + "assembly_script_deferred_call_register": 530, "assembly_script_function_exists": 575, "assembly_script_generate_event": 172, "assembly_script_get_balance": 149, @@ -129,6 +129,10 @@ "abi_unsafe_random": 402, "abi_verify_signature": 1192, "abi_chain_id": 301, + "abi_deferred_call_cancel": 750, + "abi_deferred_call_exists": 443, + "abi_deferred_call_register": 745, + "abi_get_deferred_call_quote": 416, "assembly_script_console_log": 171, "assembly_script_console_info": 171, "assembly_script_console_debug": 171, From cb30010b25b230ead210e7b6eb2a74f96483fc5a Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 10 Dec 2024 09:51:15 +0100 Subject: [PATCH 43/63] Add versioning for fix async message updates and add unit test --- massa-execution-worker/src/active_history.rs | 24 ++++++++----- .../src/speculative_async_pool.rs | 34 +++++++++++++++---- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/massa-execution-worker/src/active_history.rs b/massa-execution-worker/src/active_history.rs index 905c6cd829f..4e647816ca9 100644 --- a/massa-execution-worker/src/active_history.rs +++ b/massa-execution-worker/src/active_history.rs @@ -68,6 +68,7 @@ impl ActiveHistory { &self, message_id: &AsyncMessageId, mut current_updates: AsyncMessageUpdate, + execution_compound_version: u32, ) -> HistorySearchResult> { for history_element in self.0.iter().rev() { match history_element @@ -81,11 +82,16 @@ impl ActiveHistory { msg.apply(current_updates); return HistorySearchResult::Present(SetUpdateOrDelete::Set(msg)); } - Some(SetUpdateOrDelete::Update(msg_update)) => { - let mut combined_message_update = msg_update.clone(); - combined_message_update.apply(current_updates); - current_updates = combined_message_update; - } + Some(SetUpdateOrDelete::Update(msg_update)) => match execution_compound_version { + 0 => { + current_updates.apply(msg_update.clone()); + } + _ => { + let mut combined_message_update = msg_update.clone(); + combined_message_update.apply(current_updates); + current_updates = combined_message_update; + } + }, Some(SetUpdateOrDelete::Delete) => return HistorySearchResult::Absent, _ => (), } @@ -487,14 +493,14 @@ mod test { // Test fetch_message with message_id (expect HistorySearchResult::Absent) { let current_updates = AsyncMessageUpdate::default(); - let fetched = active_history.fetch_message(&message_id, current_updates); + let fetched = active_history.fetch_message(&message_id, current_updates, 1); assert_matches!(fetched, HistorySearchResult::Absent); } // Test fetch_message with message_id_2 (expect HistorySearchResult::Set) { let current_updates = AsyncMessageUpdate::default(); - let fetched = active_history.fetch_message(&message_id_2, current_updates); + let fetched = active_history.fetch_message(&message_id_2, current_updates, 1); if let HistorySearchResult::Present(SetUpdateOrDelete::Set(msg)) = fetched { assert_eq!(msg, msg_2); @@ -526,7 +532,7 @@ mod test { trigger: Default::default(), can_be_executed: Default::default(), }; - let fetched = active_history.fetch_message(&message_id_2, current_updates); + let fetched = active_history.fetch_message(&message_id_2, current_updates, 1); if let HistorySearchResult::Present(SetUpdateOrDelete::Set(msg)) = fetched { assert_ne!(msg, msg_2); @@ -542,7 +548,7 @@ mod test { // Test fetch_message with message_id_3 (expect HistorySearchResult::Present) { let current_updates = AsyncMessageUpdate::default(); - let fetched = active_history.fetch_message(&message_id_3, current_updates); + let fetched = active_history.fetch_message(&message_id_3, current_updates, 1); if let HistorySearchResult::Present(SetUpdateOrDelete::Update(msg_up)) = fetched { // Note that there should be to async message update & we ensure that the latest diff --git a/massa-execution-worker/src/speculative_async_pool.rs b/massa-execution-worker/src/speculative_async_pool.rs index fa352735e22..39daf21bc14 100644 --- a/massa-execution-worker/src/speculative_async_pool.rs +++ b/massa-execution-worker/src/speculative_async_pool.rs @@ -146,7 +146,11 @@ impl SpeculativeAsyncPool { } } - let taken = self.fetch_msgs(wanted_messages, true); + let taken = self.fetch_msgs( + wanted_messages, + true, + self.get_execution_component_version(&slot), + ); for (message_id, _) in taken.iter() { self.message_infos.remove(message_id); @@ -170,12 +174,13 @@ impl SpeculativeAsyncPool { ledger_changes: &LedgerChanges, fix_eliminated_msg: bool, ) -> Vec<(AsyncMessageId, AsyncMessage)> { + let execution_component_version = self.get_execution_component_version(slot); + // Update the messages_info: remove messages that should be removed // Filter out all messages for which the validity end is expired. // Note: that the validity_end bound is included in the validity interval of the message. let mut eliminated_infos = Vec::new(); - let execution_component_version = self.get_execution_component_version(slot); self.message_infos.retain(|id, info| { if Self::is_message_expired(slot, &info.validity_end, execution_component_version) { eliminated_infos.push((*id, info.clone())); @@ -232,16 +237,22 @@ impl SpeculativeAsyncPool { } // Query triggered messages - let triggered_msg = - self.fetch_msgs(triggered_info.iter().map(|(id, _)| id).collect(), false); + let triggered_msg = self.fetch_msgs( + triggered_info.iter().map(|(id, _)| id).collect(), + false, + execution_component_version, + ); for (msg_id, _msg) in triggered_msg.iter() { self.pool_changes.push_activate(*msg_id); } // Query eliminated messages - let mut eliminated_msg = - self.fetch_msgs(eliminated_infos.iter().map(|(id, _)| id).collect(), true); + let mut eliminated_msg = self.fetch_msgs( + eliminated_infos.iter().map(|(id, _)| id).collect(), + true, + execution_component_version, + ); if fix_eliminated_msg { eliminated_msg.extend(eliminated_new_messages.iter().filter_map(|(k, v)| match v { SetUpdateOrDelete::Set(v) => Some((*k, v.clone())), @@ -256,6 +267,7 @@ impl SpeculativeAsyncPool { &mut self, mut wanted_ids: Vec<&AsyncMessageId>, delete_existing: bool, + execution_component_version: u32, ) -> Vec<(AsyncMessageId, AsyncMessage)> { let mut msgs = Vec::new(); @@ -290,6 +302,7 @@ impl SpeculativeAsyncPool { match self.active_history.read().fetch_message( message_id, current_changes.get(message_id).cloned().unwrap_or_default(), + execution_component_version, ) { Present(SetUpdateOrDelete::Set(mut msg)) => { msg.apply(current_changes.get(message_id).cloned().unwrap_or_default()); @@ -301,7 +314,14 @@ impl SpeculativeAsyncPool { } Present(SetUpdateOrDelete::Update(msg_update)) => { current_changes.entry(message_id).and_modify(|e| { - *e = msg_update.clone(); + match execution_component_version { + 0 => { + e.apply(msg_update.clone()); + } + _ => { + *e = msg_update.clone(); + } + } }); return true; } From 820ed1ef835327fa00851028c8de48f5ffea186e Mon Sep 17 00:00:00 2001 From: modship Date: Wed, 11 Dec 2024 11:44:41 +0100 Subject: [PATCH 44/63] Api : deferred call endpoint versioning --- massa-api/src/public.rs | 30 +++++++++++++++++++++ massa-execution-exports/src/mapping_grpc.rs | 19 +++++++++++++ massa-grpc/src/public.rs | 4 ++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index 391120833db..e28d638fe0e 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -1160,6 +1160,16 @@ impl MassaRpcServer for API { &self, req: Vec, ) -> RpcResult> { + let current_network_version = self + .0 + .keypair_factory + .mip_store + .get_network_version_current(); + + if current_network_version < 1 { + return Err(ApiError::NotFound.into()); + } + if req.len() as u64 > self.0.api_settings.max_arguments { return Err(ApiError::BadRequest("too many arguments".into()).into()); } @@ -1205,6 +1215,16 @@ impl MassaRpcServer for API { &self, arg: Vec, ) -> RpcResult> { + let current_network_version = self + .0 + .keypair_factory + .mip_store + .get_network_version_current(); + + if current_network_version < 1 { + return Err(ApiError::NotFound.into()); + } + if arg.len() as u64 > self.0.api_settings.max_arguments { return Err(ApiError::BadRequest("too many arguments".into()).into()); } @@ -1245,6 +1265,16 @@ impl MassaRpcServer for API { &self, slots: Vec, ) -> RpcResult> { + let current_network_version = self + .0 + .keypair_factory + .mip_store + .get_network_version_current(); + + if current_network_version < 1 { + return Err(ApiError::NotFound.into()); + } + if slots.len() as u64 > self.0.api_settings.max_arguments { return Err(ApiError::BadRequest("too many arguments".into()).into()); } diff --git a/massa-execution-exports/src/mapping_grpc.rs b/massa-execution-exports/src/mapping_grpc.rs index 425d8d2942f..0be923d81d4 100644 --- a/massa-execution-exports/src/mapping_grpc.rs +++ b/massa-execution-exports/src/mapping_grpc.rs @@ -21,6 +21,7 @@ use massa_proto_rs::massa::model::v1 as grpc_model; /// Convert a `grpc_api::ScExecutionEventsRequest` to a `ScExecutionEventsRequest` pub fn to_querystate_filter( query: grpc_api::ExecutionQueryRequestItem, + network_version: u32, ) -> Result { if let Some(item) = query.request_item { match item { @@ -135,6 +136,12 @@ pub fn to_querystate_filter( Ok(ExecutionQueryRequestItem::Events(event_filter)) } exec::RequestItem::DeferredCallQuote(value) => { + if network_version < 1 { + return Err(ModelsError::InvalidVersionError( + "deferred call quote is not supported in this network version".to_string(), + )); + } + Ok(ExecutionQueryRequestItem::DeferredCallQuote { target_slot: value .target_slot @@ -147,10 +154,22 @@ pub fn to_querystate_filter( }) } exec::RequestItem::DeferredCallInfo(info) => { + if network_version < 1 { + return Err(ModelsError::InvalidVersionError( + "deferred call quote is not supported in this network version".to_string(), + )); + } + let id = DeferredCallId::from_str(&info.call_id)?; Ok(ExecutionQueryRequestItem::DeferredCallInfo(id)) } exec::RequestItem::DeferredCallsBySlot(value) => { + if network_version < 1 { + return Err(ModelsError::InvalidVersionError( + "deferred call quote is not supported in this network version".to_string(), + )); + } + Ok(ExecutionQueryRequestItem::DeferredCallsBySlot( value .slot diff --git a/massa-grpc/src/public.rs b/massa-grpc/src/public.rs index 9da185f5d7b..6dd98bed3c1 100644 --- a/massa-grpc/src/public.rs +++ b/massa-grpc/src/public.rs @@ -992,11 +992,13 @@ pub(crate) fn query_state( grpc: &MassaPublicGrpc, request: tonic::Request, ) -> Result { + let current_network_version = grpc.keypair_factory.mip_store.get_network_version_current(); + let queries = request .into_inner() .queries .into_iter() - .map(to_querystate_filter) + .map(|q| to_querystate_filter(q, current_network_version)) .collect::, _>>()?; if queries.is_empty() { From fc07bc555282ff471b9a620e7b1073df07bb9c02 Mon Sep 17 00:00:00 2001 From: modship Date: Thu, 12 Dec 2024 10:32:54 +0100 Subject: [PATCH 45/63] Metrics: add network version metrics --- massa-metrics/src/lib.rs | 39 ++++++++++++++++++++++++++++++ massa-node/src/main.rs | 1 + massa-node/src/survey.rs | 7 ++++++ massa-versioning/src/versioning.rs | 6 +++++ 4 files changed, 53 insertions(+) diff --git a/massa-metrics/src/lib.rs b/massa-metrics/src/lib.rs index 6e8c6909df5..4cef4a0b64f 100644 --- a/massa-metrics/src/lib.rs +++ b/massa-metrics/src/lib.rs @@ -199,6 +199,9 @@ pub struct MassaMetrics { // peer bandwidth (bytes sent, bytes received) peers_bandwidth: Arc>>, + // network versions votes + network_versions_votes: Arc>>, + pub tick_delay: Duration, // deferred calls metrics @@ -557,6 +560,7 @@ impl MassaMetrics { final_cursor_thread, final_cursor_period, peers_bandwidth: Arc::new(RwLock::new(HashMap::new())), + network_versions_votes: Arc::new(RwLock::new(HashMap::new())), tick_delay, deferred_calls_executed, deferred_calls_failed, @@ -756,6 +760,41 @@ impl MassaMetrics { self.deferred_calls_failed.inc(); } + // Update the network version vote metrics + pub fn update_network_version_vote(&self, data: HashMap) { + if self.enabled { + let mut write = self.network_versions_votes.write().unwrap(); + + { + let missing_version = write + .keys() + .filter(|key| !data.contains_key(key)) + .cloned() + .collect::>(); + + for key in missing_version { + if let Some(counter) = write.remove(&key) { + if let Err(e) = prometheus::unregister(Box::new(counter)) { + warn!("Failed to unregister network_version_vote_{} : {}", key, e); + } + } + } + } + + for (version, count) in data.into_iter() { + if let Some(actual_counter) = write.get_mut(&version) { + actual_counter.set(count as i64); + } else { + let label = format!("network_version_votes_{}", version); + let counter = IntGauge::new(label, "").unwrap(); + counter.set(count as i64); + let _ = prometheus::register(Box::new(counter.clone())); + write.insert(version, counter); + } + } + } + } + /// Update the bandwidth metrics for all peers /// HashMap pub fn update_peers_tx_rx(&self, data: HashMap) { diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 6c203198b7d..b735ca16134 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -1101,6 +1101,7 @@ async fn launch( api_config.periods_per_cycle, api_config.last_start_period, ), + mip_store, ); #[cfg(feature = "deadlock_detection")] diff --git a/massa-node/src/survey.rs b/massa-node/src/survey.rs index 26018f3cda8..a4dc7c5accd 100644 --- a/massa-node/src/survey.rs +++ b/massa-node/src/survey.rs @@ -8,6 +8,7 @@ use massa_metrics::MassaMetrics; use massa_models::{address::Address, slot::Slot, timeslots::get_latest_block_slot_at_timestamp}; use massa_pool_exports::PoolController; use massa_time::MassaTime; +use massa_versioning::versioning::MipStore; use tracing::info; // use std::time::Duration; use tracing::warn; @@ -45,6 +46,7 @@ impl MassaSurvey { pool_controller: Box, massa_metrics: MassaMetrics, config: (u8, MassaTime, MassaTime, u64, u64), + mip_store: MipStore, ) -> MassaSurveyStopper { if massa_metrics.is_enabled() { #[cfg(all(not(feature = "sandbox"), not(test)))] @@ -130,6 +132,11 @@ impl MassaSurvey { .get(); massa_metrics.set_available_processors(count); } + + { + let network_stats= mip_store.0.read().get_network_versions_stats(); + massa_metrics.update_network_version_vote(network_stats); + } } } }) { diff --git a/massa-versioning/src/versioning.rs b/massa-versioning/src/versioning.rs index 3d63c3623d8..e26096e2458 100644 --- a/massa-versioning/src/versioning.rs +++ b/massa-versioning/src/versioning.rs @@ -1312,6 +1312,12 @@ impl MipStoreRaw { Ok(()) } + // get network versions stats + // + pub fn get_network_versions_stats(&self) -> HashMap { + self.stats.network_version_counters.clone() + } + // Final state pub fn is_key_value_valid(&self, serialized_key: &[u8], serialized_value: &[u8]) -> bool { self._is_key_value_valid(serialized_key, serialized_value) From ef95780f309d47c0570e14dd81103f39066b81d5 Mon Sep 17 00:00:00 2001 From: modship Date: Thu, 12 Dec 2024 17:20:05 +0100 Subject: [PATCH 46/63] fix empty help msg --- massa-metrics/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-metrics/src/lib.rs b/massa-metrics/src/lib.rs index 4cef4a0b64f..98da5ace180 100644 --- a/massa-metrics/src/lib.rs +++ b/massa-metrics/src/lib.rs @@ -786,7 +786,7 @@ impl MassaMetrics { actual_counter.set(count as i64); } else { let label = format!("network_version_votes_{}", version); - let counter = IntGauge::new(label, "").unwrap(); + let counter = IntGauge::new(label, "vote counter for network version").unwrap(); counter.set(count as i64); let _ = prometheus::register(Box::new(counter.clone())); write.insert(version, counter); From 37edce7501a7716b536d2e35eb33cc610b1e0bf8 Mon Sep 17 00:00:00 2001 From: modship Date: Mon, 16 Dec 2024 15:05:48 +0100 Subject: [PATCH 47/63] Rollback deferred calls abi bail --- massa-execution-worker/src/interface_impl.rs | 43 ++++++-------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 45215853fbb..97b9ef45780 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1539,29 +1539,21 @@ impl Interface for InterfaceImpl { ) -> Result<(bool, u64)> { // write-lock context let context = context_guard!(self); + let current_slot = context.slot; - match context.execution_component_version { - 0 => { - bail!("Deferred calls are not supported in this execution component version") - } - _ => { - let current_slot = context.slot; + let target_slot = Slot::new(target_slot.0, target_slot.1); - let target_slot = Slot::new(target_slot.0, target_slot.1); + let gas_request = + gas_limit.saturating_add(self.config.deferred_calls_config.call_cst_gas_cost); - let gas_request = - gas_limit.saturating_add(self.config.deferred_calls_config.call_cst_gas_cost); - - match context.deferred_calls_compute_call_fee( - target_slot, - gas_request, - current_slot, - params_size, - ) { - Ok(fee) => Ok((true, fee.to_raw())), - Err(_) => Ok((false, 0)), - } - } + match context.deferred_calls_compute_call_fee( + target_slot, + gas_request, + current_slot, + params_size, + ) { + Ok(fee) => Ok((true, fee.to_raw())), + Err(_) => Ok((false, 0)), } } @@ -1586,10 +1578,6 @@ impl Interface for InterfaceImpl { params: &[u8], coins: u64, ) -> Result { - if context_guard!(self).execution_component_version == 0 { - bail!("Deferred calls are not supported in this execution component version") - } - // This function spends coins + deferred_call_quote(target_slot, max_gas).unwrap() from the caller, fails if the balance is insufficient or if the quote would return None. let target_addr = Address::from_str(target_addr)?; @@ -1652,9 +1640,6 @@ impl Interface for InterfaceImpl { fn deferred_call_exists(&self, id: &str) -> Result { // write-lock context let context = context_guard!(self); - if context.execution_component_version == 0 { - bail!("Deferred calls are not supported in this execution component version") - } let call_id = DeferredCallId::from_str(id)?; Ok(context.deferred_call_exists(&call_id)) @@ -1669,10 +1654,6 @@ impl Interface for InterfaceImpl { let mut context = context_guard!(self); - if context.execution_component_version == 0 { - bail!("Deferred calls are not supported in this execution component version") - } - // Can only be called by the creator of the deferred call. let caller = context.get_current_address()?; From eeda2f50f78b36360fc15c3201d67e072a20889c Mon Sep 17 00:00:00 2001 From: modship Date: Mon, 16 Dec 2024 15:40:52 +0100 Subject: [PATCH 48/63] update runtime --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba35600ad67..df6feccc143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2798,7 +2798,7 @@ dependencies = [ [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?rev=82dd714d38cbcd863ae0bb215c06442917bb6404#82dd714d38cbcd863ae0bb215c06442917bb6404" +source = "git+https://github.com/massalabs/massa-sc-runtime?rev=0727dc254db2d1fe769334a6e68b6605d2d8eee1#0727dc254db2d1fe769334a6e68b6605d2d8eee1" dependencies = [ "anyhow", "as-ffi-bindings", diff --git a/Cargo.toml b/Cargo.toml index 0390ccbe859..d29ab06a48b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,7 +110,7 @@ massa_wallet = { path = "./massa-wallet" } # massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", branch = "deferred_calls" } # massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "branch" = "deferred_calls" } massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "b5267178eaf266ec724691d7de163e4c34343416" } -massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "82dd714d38cbcd863ae0bb215c06442917bb6404" } +massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "0727dc254db2d1fe769334a6e68b6605d2d8eee1" } peernet = { git = "https://github.com/massalabs/PeerNet", "rev" = "04b05ddd320fbe76cc858115af7b5fc28bdb8310" } From faa8521c7676cbeb360d2789bf7127b7ba34276d Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 16 Dec 2024 15:56:25 +0100 Subject: [PATCH 49/63] Allow transfers to SC addresses (#4789) * Allow transfers to SC * Add unit test for 4 scenarios --- massa-execution-worker/src/context.rs | 24 +--- .../src/speculative_ledger.rs | 5 + .../src/tests/scenarios_mandatories.rs | 125 ++++++++++++++---- 3 files changed, 113 insertions(+), 41 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index d08c6fce666..501a0e624e4 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -750,24 +750,12 @@ impl ExecutionContext { ) -> Result<(), ExecutionError> { if let Some(from_addr) = &from_addr { // check access rights - if check_rights { - // ensure we can't spend from an address on which we have no write access - if !self.has_write_rights_on(from_addr) { - return Err(ExecutionError::RuntimeError(format!( - "spending from address {} is not allowed in this context", - from_addr - ))); - } - - // ensure we can't transfer towards SC addresses on which we have no write access - if let Some(to_addr) = &to_addr { - if matches!(to_addr, Address::SC(..)) && !self.has_write_rights_on(to_addr) { - return Err(ExecutionError::RuntimeError(format!( - "crediting SC address {} is not allowed without write access to it", - to_addr - ))); - } - } + // ensure we can't spend from an address on which we have no write access + if check_rights && !self.has_write_rights_on(from_addr) { + return Err(ExecutionError::RuntimeError(format!( + "spending from address {} is not allowed in this context", + from_addr + ))); } } diff --git a/massa-execution-worker/src/speculative_ledger.rs b/massa-execution-worker/src/speculative_ledger.rs index 654219da8c1..15f22c2a344 100644 --- a/massa-execution-worker/src/speculative_ledger.rs +++ b/massa-execution-worker/src/speculative_ledger.rs @@ -177,6 +177,11 @@ impl SpeculativeLedger { )) })?; changes.set_balance(to_addr, new_balance); + } else if matches!(to_addr, Address::SC(..)) { + return Err(ExecutionError::RuntimeError(format!( + "cannot transfer coins to non-existing smart contract address {}", + to_addr + ))); } else if let Some(remaining_coins) = amount.checked_sub(self.storage_costs_constants.ledger_entry_base_cost) { diff --git a/massa-execution-worker/src/tests/scenarios_mandatories.rs b/massa-execution-worker/src/tests/scenarios_mandatories.rs index 9d010d8abe1..ad82b331c00 100644 --- a/massa-execution-worker/src/tests/scenarios_mandatories.rs +++ b/massa-execution-worker/src/tests/scenarios_mandatories.rs @@ -2582,10 +2582,43 @@ fn send_and_receive_transaction() { // setup the period duration let exec_cfg = ExecutionConfig::default(); let mut foreign_controllers = ExecutionForeignControllers::new_with_mocks(); + + // Set various addresses to test the execution's behavior + let existing_user_recipient_address = + Address::from_public_key(&KeyPair::generate(0).unwrap().get_public_key()); + let non_existing_user_recipient_address = + Address::from_public_key(&KeyPair::generate(0).unwrap().get_public_key()); + let existing_sc_recipient_address = + Address::from_str("AS1Bc3kZ6LhPLJvXV4vcVJLFRExRFbkPWD7rCg9aAdQ1NGzRwgnu").unwrap(); + let non_existing_sc_recipient_address = + Address::from_str("AS1aEhosr1ebJJZ7cEMpSVKbY6xp1p4DdXabGb8fdkKKJ6WphGnR").unwrap(); + let non_existing_recipient_addresses = [ + non_existing_user_recipient_address, + non_existing_sc_recipient_address, + ]; + let recipient_addresses = vec![ + existing_user_recipient_address, + existing_sc_recipient_address, + non_existing_user_recipient_address, + non_existing_sc_recipient_address, + ]; + + foreign_controllers + .ledger_controller + .set_expectations(|ledger_controller| { + ledger_controller + .expect_get_balance() + .returning(move |address| { + if non_existing_recipient_addresses.contains(address) { + None + } else { + Some(Amount::from_str("100").unwrap()) + } + }); + }); let finalized_waitpoint = WaitPoint::new(); let finalized_waitpoint_trigger_handle = finalized_waitpoint.get_trigger_handle(); - let recipient_address = - Address::from_public_key(&KeyPair::generate(0).unwrap().get_public_key()); + selector_boilerplate(&mut foreign_controllers.selector_controller); final_state_boilerplate( &mut foreign_controllers.final_state, @@ -2604,17 +2637,38 @@ fn send_and_receive_transaction() { .times(1) .with(predicate::eq(Slot::new(1, 0)), predicate::always()) .returning(move |_, changes| { - // 190 because 100 in the get_balance in the `final_state_boilerplate` and 90 from the transfer. + // 110 because 100 in the get_balance in the `final_state_boilerplate` and 10 from the transfer. assert_eq!( changes .ledger_changes - .get_balance_or_else(&recipient_address, || None), - Some(Amount::from_str("190").unwrap()) + .get_balance_or_else(&existing_user_recipient_address, || None), + Some(Amount::from_str("110").unwrap()) + ); + // 9.999 because -0.001 for address creation and 10 from the transfer. + assert_eq!( + changes + .ledger_changes + .get_balance_or_else(&non_existing_user_recipient_address, || None), + Some(Amount::from_str("9.999").unwrap()) + ); + // 110 because 100 in the get_balance in the `final_state_boilerplate` and 10 from the transfer. + assert_eq!( + changes + .ledger_changes + .get_balance_or_else(&existing_sc_recipient_address, || None), + Some(Amount::from_str("110").unwrap()) + ); + // Cannot transfer coins to a non-existing smart contract + assert_eq!( + changes + .ledger_changes + .get_balance_or_else(&non_existing_sc_recipient_address, || None), + None ); // block rewards computation let total_rewards = exec_cfg .block_reward - .saturating_add(Amount::from_str("10").unwrap()); // add 10 MAS for fees + .saturating_add(Amount::from_str("20").unwrap()); // add 20 MAS for fees let rewards_for_block_creator = total_rewards .checked_div_u64(BLOCK_CREDIT_PART_COUNT) .expect("critical: total_rewards checked_div factor is 0") @@ -2624,6 +2678,13 @@ fn send_and_receive_transaction() { .checked_rem_u64(BLOCK_CREDIT_PART_COUNT) .expect("critical: total_rewards checked_rem factor is 0"), ); + // 100 initial balance, + block rewards - transferred amount (3*10) - fees (4*5) + let sender_expected_balance = Amount::from_str("100") + .unwrap() + .saturating_add(rewards_for_block_creator) + .saturating_sub(Amount::from_str("30").unwrap()) + .saturating_sub(Amount::from_str("20").unwrap()); + assert_eq!( changes.ledger_changes.get_balance_or_else( &Address::from_public_key( @@ -2631,38 +2692,56 @@ fn send_and_receive_transaction() { ), || None ), - Some(rewards_for_block_creator) + Some(sender_expected_balance) ); finalized_waitpoint_trigger_handle.trigger(); }); let mut universe = ExecutionTestUniverse::new(foreign_controllers, exec_cfg.clone()); - // create the operation - let operation = Operation::new_verifiable( - Operation { - fee: Amount::from_str("10").unwrap(), - expire_period: 10, - op: OperationType::Transaction { - recipient_address, - amount: Amount::from_str("90").unwrap(), + // create the operations + let mut operation_vec = Vec::new(); + for recipient_address in recipient_addresses { + let operation = Operation::new_verifiable( + Operation { + fee: Amount::from_str("5").unwrap(), + expire_period: 10, + op: OperationType::Transaction { + recipient_address, + amount: Amount::from_str("10").unwrap(), + }, }, - }, - OperationSerializer::new(), - &KeyPair::from_str(TEST_SK_1).unwrap(), - *CHAINID, - ) - .unwrap(); + OperationSerializer::new(), + &KeyPair::from_str(TEST_SK_1).unwrap(), + *CHAINID, + ) + .unwrap(); + operation_vec.push(operation.clone()); + } + // create the block containing the transaction operation - universe.storage.store_operations(vec![operation.clone()]); + universe.storage.store_operations(operation_vec.clone()); let block = ExecutionTestUniverse::create_block( &KeyPair::from_str(TEST_SK_1).unwrap(), Slot::new(1, 0), - vec![operation], + operation_vec, vec![], vec![], ); // store the block in storage universe.send_and_finalize(&KeyPair::from_str(TEST_SK_1).unwrap(), block, None); finalized_waitpoint.wait(); + + let events = universe + .module_controller + .get_filtered_sc_output_event(EventFilter { + is_error: Some(true), + ..Default::default() + }); + // match the events + println!("{:?}", events); + assert!(events.len() == 1, "1 event was expected"); + assert!(events[0] + .data + .contains("cannot transfer coins to non-existing smart contract address")); } #[test] From 09cda310ce473c9953a63382b6c08f0d5b5908b6 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 16 Dec 2024 16:16:26 +0100 Subject: [PATCH 50/63] Add versioning to Allow transfers to SC addresses --- massa-execution-worker/src/context.rs | 83 +++++++++++++++---- .../src/speculative_ledger.rs | 46 ++++++++-- 2 files changed, 108 insertions(+), 21 deletions(-) diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 501a0e624e4..183ccbf663d 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -592,6 +592,7 @@ impl ExecutionContext { self.get_current_address()?, address, bytecode, + self.execution_component_version, )?; // add the address to owned addresses @@ -663,8 +664,13 @@ impl ExecutionContext { } // set data entry - self.speculative_ledger - .set_data_entry(&self.get_current_address()?, address, key, data) + self.speculative_ledger.set_data_entry( + &self.get_current_address()?, + address, + key, + data, + self.execution_component_version, + ) } /// Appends data to a datastore entry for an address in the speculative ledger. @@ -704,8 +710,13 @@ impl ExecutionContext { res_data.extend(data); // set data entry - self.speculative_ledger - .set_data_entry(&self.get_current_address()?, address, key, res_data) + self.speculative_ledger.set_data_entry( + &self.get_current_address()?, + address, + key, + res_data, + self.execution_component_version, + ) } /// Deletes a datastore entry for an address. @@ -728,8 +739,12 @@ impl ExecutionContext { } // delete entry - self.speculative_ledger - .delete_data_entry(&self.get_current_address()?, address, key) + self.speculative_ledger.delete_data_entry( + &self.get_current_address()?, + address, + key, + self.execution_component_version, + ) } /// Transfers coins from one address to another. @@ -748,20 +763,54 @@ impl ExecutionContext { amount: Amount, check_rights: bool, ) -> Result<(), ExecutionError> { + let execution_component_version = self.execution_component_version; + if let Some(from_addr) = &from_addr { // check access rights // ensure we can't spend from an address on which we have no write access - if check_rights && !self.has_write_rights_on(from_addr) { - return Err(ExecutionError::RuntimeError(format!( - "spending from address {} is not allowed in this context", - from_addr - ))); + // If execution component version is 0, we need to disallow sending to SC addresses + + match execution_component_version { + 0 => { + if check_rights { + if !self.has_write_rights_on(from_addr) { + return Err(ExecutionError::RuntimeError(format!( + "spending from address {} is not allowed in this context", + from_addr + ))); + } + + // ensure we can't transfer towards SC addresses on which we have no write access + if let Some(to_addr) = &to_addr { + if matches!(to_addr, Address::SC(..)) + && !self.has_write_rights_on(to_addr) + { + return Err(ExecutionError::RuntimeError(format!( + "crediting SC address {} is not allowed without write access to it", + to_addr + ))); + } + } + } + } + _ => { + if check_rights && !self.has_write_rights_on(from_addr) { + return Err(ExecutionError::RuntimeError(format!( + "spending from address {} is not allowed in this context", + from_addr + ))); + } + } } } // do the transfer - self.speculative_ledger - .transfer_coins(from_addr, to_addr, amount) + self.speculative_ledger.transfer_coins( + from_addr, + to_addr, + amount, + execution_component_version, + ) } /// Add a new asynchronous message to speculative pool @@ -1211,8 +1260,12 @@ impl ExecutionContext { } // set data entry - self.speculative_ledger - .set_bytecode(&self.get_current_address()?, address, bytecode) + self.speculative_ledger.set_bytecode( + &self.get_current_address()?, + address, + bytecode, + self.execution_component_version, + ) } /// Creates a new event but does not emit it. diff --git a/massa-execution-worker/src/speculative_ledger.rs b/massa-execution-worker/src/speculative_ledger.rs index 15f22c2a344..8df53071e29 100644 --- a/massa-execution-worker/src/speculative_ledger.rs +++ b/massa-execution-worker/src/speculative_ledger.rs @@ -144,6 +144,7 @@ impl SpeculativeLedger { from_addr: Option
, to_addr: Option
, amount: Amount, + execution_component_version: u32, ) -> Result<(), ExecutionError> { // init empty ledger changes let mut changes = LedgerChanges::default(); @@ -177,7 +178,7 @@ impl SpeculativeLedger { )) })?; changes.set_balance(to_addr, new_balance); - } else if matches!(to_addr, Address::SC(..)) { + } else if execution_component_version > 0 && matches!(to_addr, Address::SC(..)) { return Err(ExecutionError::RuntimeError(format!( "cannot transfer coins to non-existing smart contract address {}", to_addr @@ -236,6 +237,7 @@ impl SpeculativeLedger { creator_address: Address, addr: Address, bytecode: Bytecode, + execution_component_version: u32, ) -> Result<(), ExecutionError> { // check for address existence if !self.entry_exists(&creator_address) { @@ -283,7 +285,12 @@ impl SpeculativeLedger { ExecutionError::RuntimeError("overflow in ledger cost for bytecode".to_string()) })?; - self.transfer_coins(Some(creator_address), None, address_storage_cost)?; + self.transfer_coins( + Some(creator_address), + None, + address_storage_cost, + execution_component_version, + )?; self.added_changes.create_address(&addr); self.added_changes.set_bytecode(addr, bytecode); Ok(()) @@ -301,6 +308,7 @@ impl SpeculativeLedger { caller_addr: &Address, addr: &Address, bytecode: Bytecode, + execution_component_version: u32, ) -> Result<(), ExecutionError> { // check for address existence if !self.entry_exists(addr) { @@ -330,8 +338,18 @@ impl SpeculativeLedger { })?; match diff_size_storage.signum() { - 1 => self.transfer_coins(Some(*caller_addr), None, storage_cost_bytecode)?, - -1 => self.transfer_coins(None, Some(*caller_addr), storage_cost_bytecode)?, + 1 => self.transfer_coins( + Some(*caller_addr), + None, + storage_cost_bytecode, + execution_component_version, + )?, + -1 => self.transfer_coins( + None, + Some(*caller_addr), + storage_cost_bytecode, + execution_component_version, + )?, _ => {} }; } else { @@ -344,7 +362,12 @@ impl SpeculativeLedger { "overflow when calculating storage cost of bytecode".to_string(), ) })?; - self.transfer_coins(Some(*caller_addr), None, bytecode_storage_cost)?; + self.transfer_coins( + Some(*caller_addr), + None, + bytecode_storage_cost, + execution_component_version, + )?; } // set the bytecode of that address self.added_changes.set_bytecode(*addr, bytecode); @@ -528,6 +551,7 @@ impl SpeculativeLedger { caller_addr: &Address, old_key_value: Option<(&[u8], &[u8])>, new_key_value: Option<(&[u8], &[u8])>, + execution_component_version: u32, ) -> Result<(), ExecutionError> { // compute the old storage cost of the entry let old_storage_cost = old_key_value.map_or_else( @@ -549,6 +573,7 @@ impl SpeculativeLedger { Some(*caller_addr), None, new_storage_cost.saturating_sub(old_storage_cost), + execution_component_version, ) } Ordering::Less => { @@ -557,6 +582,7 @@ impl SpeculativeLedger { None, Some(*caller_addr), old_storage_cost.saturating_sub(new_storage_cost), + execution_component_version, ) } Ordering::Equal => { @@ -587,6 +613,7 @@ impl SpeculativeLedger { addr: &Address, key: Vec, value: Vec, + execution_component_version: u32, ) -> Result<(), ExecutionError> { // check for address existence if !self.entry_exists(addr) { @@ -620,6 +647,7 @@ impl SpeculativeLedger { caller_addr, prev_value.as_ref().map(|v| (&key[..], &v[..])), Some((&key, &value)), + execution_component_version, )?; } @@ -641,11 +669,17 @@ impl SpeculativeLedger { caller_addr: &Address, addr: &Address, key: &[u8], + execution_component_version: u32, ) -> Result<(), ExecutionError> { // check if the entry exists if let Some(value) = self.get_data_entry(addr, key) { // reimburse the storage costs of the entry - self.charge_datastore_entry_change_storage(caller_addr, Some((key, &value)), None)?; + self.charge_datastore_entry_change_storage( + caller_addr, + Some((key, &value)), + None, + execution_component_version, + )?; } else { return Err(ExecutionError::RuntimeError(format!( "could not delete data entry {:?} for address {}: entry or address does not exist", From 5ede1f18bc44409c2c2a1c105f0b2f2571293e91 Mon Sep 17 00:00:00 2001 From: Sydhds Date: Mon, 16 Dec 2024 16:25:34 +0100 Subject: [PATCH 51/63] Use coherent PublicKeyFormat in parse_slice (#4783) --- massa-execution-worker/src/active_history.rs | 2 ++ massa-execution-worker/src/interface_impl.rs | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/massa-execution-worker/src/active_history.rs b/massa-execution-worker/src/active_history.rs index 4e647816ca9..f92e342c426 100644 --- a/massa-execution-worker/src/active_history.rs +++ b/massa-execution-worker/src/active_history.rs @@ -445,6 +445,7 @@ mod test { let state_changes_1 = StateChanges { ledger_changes: Default::default(), async_pool_changes: async_pool_changes_1, + deferred_call_changes: Default::default(), pos_changes: Default::default(), executed_ops_changes: Default::default(), executed_denunciations_changes: Default::default(), @@ -454,6 +455,7 @@ mod test { let state_changes_2 = StateChanges { ledger_changes: Default::default(), async_pool_changes: async_pool_changes_2, + deferred_call_changes: Default::default(), pos_changes: Default::default(), executed_ops_changes: Default::default(), executed_denunciations_changes: Default::default(), diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 97b9ef45780..e7bccdb2af6 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1001,10 +1001,7 @@ impl Interface for InterfaceImpl { /// Address is the last 20 bytes of the hash of the public key. fn evm_get_address_from_pubkey(&self, public_key_: &[u8]) -> Result> { // parse the public key - let public_key = libsecp256k1::PublicKey::parse_slice( - public_key_, - Some(libsecp256k1::PublicKeyFormat::Raw), - )?; + let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; // compute the hash of the public key let hash = sha3::Keccak256::digest(public_key.serialize()); From 8efc3fc5fda8ecee828241714b6f04cd712d8446 Mon Sep 17 00:00:00 2001 From: Sydhds Date: Mon, 16 Dec 2024 16:26:51 +0100 Subject: [PATCH 52/63] Fix incorrect address hashing + add TU for evm_get_address_from_pubkey (#4785) --- massa-execution-worker/src/interface_impl.rs | 2 +- massa-execution-worker/src/tests/interface.rs | 37 ++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index e7bccdb2af6..51b6be8845b 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1004,7 +1004,7 @@ impl Interface for InterfaceImpl { let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; // compute the hash of the public key - let hash = sha3::Keccak256::digest(public_key.serialize()); + let hash = sha3::Keccak256::digest(&public_key.serialize()[1..]); // ignore the first 12 bytes of the hash let address = hash[12..].to_vec(); diff --git a/massa-execution-worker/src/tests/interface.rs b/massa-execution-worker/src/tests/interface.rs index 0fe73f62f05..c01cfa1812f 100644 --- a/massa-execution-worker/src/tests/interface.rs +++ b/massa-execution-worker/src/tests/interface.rs @@ -12,6 +12,8 @@ use massa_sc_runtime::Interface; #[test] fn test_hash_sha256() { + // test hashing using sha256 algo + let interface = InterfaceImpl::new_default( Address::from_str("AU12cMW9zRKFDS43Z2W88VCmdQFxmHjAo54XvuVV34UzJeXRLXW9M").unwrap(), None, @@ -25,6 +27,8 @@ fn test_hash_sha256() { #[test] fn test_evm_signature_verify() { + // Test that we can verify an Ethereum signature + let interface = InterfaceImpl::new_default( Address::from_str("AU12cMW9zRKFDS43Z2W88VCmdQFxmHjAo54XvuVV34UzJeXRLXW9M").unwrap(), None, @@ -55,6 +59,8 @@ fn test_evm_signature_verify() { #[test] fn test_evm_get_pubkey_from_signature() { + // Test that we can retrieve the public key from an Ethereum signature + let interface = InterfaceImpl::new_default( Address::from_str("AU12cMW9zRKFDS43Z2W88VCmdQFxmHjAo54XvuVV34UzJeXRLXW9M").unwrap(), None, @@ -79,7 +85,7 @@ fn test_evm_get_pubkey_from_signature() { assert!(result.is_ok()); assert_eq!(public_key.serialize(), result.unwrap().as_ref()); - // Invalid s + // Invalid s - expect failure { let mut signature_2 = libsecp256k1::Signature::parse_standard_slice(&signature_[..64]).unwrap(); @@ -89,7 +95,7 @@ fn test_evm_get_pubkey_from_signature() { assert!(result.is_err()); } - // Invalid v + // Invalid v - expect failure { let mut signature_2_ = signature_; signature_2_[64] ^= 1; @@ -98,6 +104,33 @@ fn test_evm_get_pubkey_from_signature() { } } +#[test] +fn test_evm_address() { + // Test that we can retrieve an Ethereum address from an Ethereum public key (from a private key) + + let interface = InterfaceImpl::new_default( + Address::from_str("AU12cMW9zRKFDS43Z2W88VCmdQFxmHjAo54XvuVV34UzJeXRLXW9M").unwrap(), + None, + None, + ); + + let _address = hex!("807a7bb5193edf9898b9092c1597bb966fe52514"); + // let message_ = b"test"; + // let signature_ = hex!("d0d05c35080635b5e865006c6c4f5b5d457ec342564d8fc67ce40edc264ccdab3f2f366b5bd1e38582538fed7fa6282148e86af97970a10cb3302896f5d68ef51b"); + let private_key_ = hex!("ed6602758bdd68dc9df67a6936ed69807a74b8cc89bdc18f3939149d02db17f3"); + + // build original public key + let private_key = libsecp256k1::SecretKey::parse_slice(&private_key_).unwrap(); + + let public_key = libsecp256k1::PublicKey::from_secret_key(&private_key); + let res = interface + .evm_get_address_from_pubkey(&public_key.serialize()) + .unwrap(); + // println!("***result: {:?}", res.to_vec()); + // println!("***add: {:?}", _address); + assert_eq!(res, _address); +} + #[test] fn test_emit_event() { // emit 2 events and check that the 2nd event is rejected (because the limit is reached) From 95eefb404bf26e04932a4782f37a4951fb0d87dd Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 16 Dec 2024 16:37:45 +0100 Subject: [PATCH 53/63] Add versioning to Use coherent PublicKeyFormat in parse_slice --- massa-execution-worker/src/interface_impl.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 51b6be8845b..2ada007e631 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1000,8 +1000,21 @@ impl Interface for InterfaceImpl { /// Get an EVM address from a raw secp256k1 public key (64 bytes). /// Address is the last 20 bytes of the hash of the public key. fn evm_get_address_from_pubkey(&self, public_key_: &[u8]) -> Result> { + + let execution_component_version = self.get_interface_version()?; + // parse the public key - let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; + let public_key = match execution { + 0 => { + libsecp256k1::PublicKey::parse_slice( + public_key_, + Some(libsecp256k1::PublicKeyFormat::Raw), + )?; + }, + _ => { + libsecp256k1::PublicKey::parse_slice(public_key_, None)?; + } + }; // compute the hash of the public key let hash = sha3::Keccak256::digest(&public_key.serialize()[1..]); From 5060ba4d8d1a809bfc1cec2ba0c55ee67ab66337 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 16 Dec 2024 16:40:52 +0100 Subject: [PATCH 54/63] Add versioning for Fix incorrect address hashing --- massa-execution-worker/src/interface_impl.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index 2ada007e631..d0d5bee9846 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1003,22 +1003,24 @@ impl Interface for InterfaceImpl { let execution_component_version = self.get_interface_version()?; - // parse the public key - let public_key = match execution { + let hash = match execution { 0 => { - libsecp256k1::PublicKey::parse_slice( + // parse the public key + let public_key = libsecp256k1::PublicKey::parse_slice( public_key_, Some(libsecp256k1::PublicKeyFormat::Raw), )?; + // compute the hash of the public key + sha3::Keccak256::digest(public_key.serialize()) }, _ => { - libsecp256k1::PublicKey::parse_slice(public_key_, None)?; + // parse the public key + let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; + // compute the hash of the public key + sha3::Keccak256::digest(&public_key.serialize()[1..]) } }; - // compute the hash of the public key - let hash = sha3::Keccak256::digest(&public_key.serialize()[1..]); - // ignore the first 12 bytes of the hash let address = hash[12..].to_vec(); From 47bfefde8982d35101bfe6fda17f95759663e598 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 16 Dec 2024 16:45:29 +0100 Subject: [PATCH 55/63] cargo fmt + check --- massa-execution-worker/src/active_history.rs | 2 -- massa-execution-worker/src/context.rs | 1 - massa-execution-worker/src/interface_impl.rs | 7 +++---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/massa-execution-worker/src/active_history.rs b/massa-execution-worker/src/active_history.rs index f92e342c426..04fa522c285 100644 --- a/massa-execution-worker/src/active_history.rs +++ b/massa-execution-worker/src/active_history.rs @@ -450,7 +450,6 @@ mod test { executed_ops_changes: Default::default(), executed_denunciations_changes: Default::default(), execution_trail_hash_change: Default::default(), - deferred_call_changes: Default::default(), }; let state_changes_2 = StateChanges { ledger_changes: Default::default(), @@ -460,7 +459,6 @@ mod test { executed_ops_changes: Default::default(), executed_denunciations_changes: Default::default(), execution_trail_hash_change: Default::default(), - deferred_call_changes: Default::default(), }; let exec_output_1 = ExecutionOutput { diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index 95c66d084a0..183ccbf663d 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -801,7 +801,6 @@ impl ExecutionContext { ))); } } - } } diff --git a/massa-execution-worker/src/interface_impl.rs b/massa-execution-worker/src/interface_impl.rs index d0d5bee9846..db81d673457 100644 --- a/massa-execution-worker/src/interface_impl.rs +++ b/massa-execution-worker/src/interface_impl.rs @@ -1000,10 +1000,9 @@ impl Interface for InterfaceImpl { /// Get an EVM address from a raw secp256k1 public key (64 bytes). /// Address is the last 20 bytes of the hash of the public key. fn evm_get_address_from_pubkey(&self, public_key_: &[u8]) -> Result> { - let execution_component_version = self.get_interface_version()?; - let hash = match execution { + let hash = match execution_component_version { 0 => { // parse the public key let public_key = libsecp256k1::PublicKey::parse_slice( @@ -1012,14 +1011,14 @@ impl Interface for InterfaceImpl { )?; // compute the hash of the public key sha3::Keccak256::digest(public_key.serialize()) - }, + } _ => { // parse the public key let public_key = libsecp256k1::PublicKey::parse_slice(public_key_, None)?; // compute the hash of the public key sha3::Keccak256::digest(&public_key.serialize()[1..]) } - }; + }; // ignore the first 12 bytes of the hash let address = hash[12..].to_vec(); From 47aee0c3fdbd1a3ace47473b7ad8fbabc383bf97 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 19 Dec 2024 16:58:18 +0100 Subject: [PATCH 56/63] Add versioning to massa event cache --- massa-event-cache/src/event_cache.rs | 4 ++-- massa-execution-worker/src/context.rs | 16 ++++++++++++---- massa-execution-worker/src/execution.rs | 8 ++++++-- massa-node/src/main.rs | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/massa-event-cache/src/event_cache.rs b/massa-event-cache/src/event_cache.rs index 5db636c3d26..37bd1fecf13 100644 --- a/massa-event-cache/src/event_cache.rs +++ b/massa-event-cache/src/event_cache.rs @@ -942,7 +942,7 @@ mod tests { use tempfile::TempDir; // internal use massa_models::config::{ - MAX_EVENT_DATA_SIZE, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK, + MAX_EVENT_DATA_SIZE_V0, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK, MAX_RECURSIVE_CALLS_DEPTH, THREAD_COUNT, }; use massa_models::operation::OperationId; @@ -957,7 +957,7 @@ mod tests { 300, THREAD_COUNT, MAX_RECURSIVE_CALLS_DEPTH, - MAX_EVENT_DATA_SIZE as u64, + MAX_EVENT_DATA_SIZE_V0 as u64, MAX_EVENT_PER_OPERATION as u64, MAX_OPERATIONS_PER_BLOCK as u64, 5000, // MAX_EVENTS_PER_QUERY, diff --git a/massa-execution-worker/src/context.rs b/massa-execution-worker/src/context.rs index ebdfc761d4b..6ecaaeff508 100644 --- a/massa-execution-worker/src/context.rs +++ b/massa-execution-worker/src/context.rs @@ -326,8 +326,12 @@ impl ExecutionContext { serde_json::json!({ "massa_execution_error": format!("{}", error) }).to_string(), true, ); - if event.data.len() > self.config.max_event_size { - event.data.truncate(self.config.max_event_size); + let max_event_size = match self.execution_component_version { + 0 => self.config.max_event_size_v0, + _ => self.config.max_event_size_v1, + }; + if event.data.len() > max_event_size { + event.data.truncate(max_event_size); } self.event_emit(event); @@ -1460,8 +1464,12 @@ impl ExecutionContext { let mut event = self.event_create(format!("DeferredCall execution fail call_id:{}", id), true); - if event.data.len() > self.config.max_event_size { - event.data.truncate(self.config.max_event_size); + let max_event_size = match self.execution_component_version { + 0 => self.config.max_event_size_v0, + _ => self.config.max_event_size_v1, + }; + if event.data.len() > max_event_size { + event.data.truncate(max_event_size); } self.event_emit(event); diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index f43bd9acc12..220459eb7f5 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -477,8 +477,12 @@ impl ExecutionState { context.transfer_coins(Some(sender_addr), None, operation.content.fee, false) { let mut error = format!("could not spend fees: {}", err); - if error.len() > self.config.max_event_size { - error.truncate(self.config.max_event_size); + let max_event_size = match execution_component_version { + 0 => self.config.max_event_size_v0, + _ => self.config.max_event_size_v1, + }; + if error.len() > max_event_size { + error.truncate(max_event_size); } let event = context.event_create(error.clone(), true); context.event_emit(event); diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 7411c80e09e..d1b2b878a41 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -481,7 +481,7 @@ async fn launch( event_cache_path: SETTINGS.execution.event_cache_path.clone(), max_event_cache_length: SETTINGS.execution.event_cache_size, snip_amount: SETTINGS.execution.event_snip_amount, - max_event_data_length: MAX_EVENT_DATA_SIZE as u64, + max_event_data_length: MAX_EVENT_DATA_SIZE_V0 as u64, thread_count: THREAD_COUNT, // Note: SCOutputEvent call stack comes from the execution module, and we assume // this should return a limited call stack length From 27ca46c6131cf1b9446939e6065d79c664f67362 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Thu, 19 Dec 2024 17:22:41 +0100 Subject: [PATCH 57/63] Fix versioning of massa event cache --- massa-event-cache/src/event_cache.rs | 4 ++-- massa-execution-worker/src/active_history.rs | 2 +- massa-execution-worker/src/execution.rs | 24 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/massa-event-cache/src/event_cache.rs b/massa-event-cache/src/event_cache.rs index 37bd1fecf13..370a486de6a 100644 --- a/massa-event-cache/src/event_cache.rs +++ b/massa-event-cache/src/event_cache.rs @@ -942,7 +942,7 @@ mod tests { use tempfile::TempDir; // internal use massa_models::config::{ - MAX_EVENT_DATA_SIZE_V0, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK, + MAX_EVENT_DATA_SIZE_V1, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK, MAX_RECURSIVE_CALLS_DEPTH, THREAD_COUNT, }; use massa_models::operation::OperationId; @@ -957,7 +957,7 @@ mod tests { 300, THREAD_COUNT, MAX_RECURSIVE_CALLS_DEPTH, - MAX_EVENT_DATA_SIZE_V0 as u64, + MAX_EVENT_DATA_SIZE_V1 as u64, MAX_EVENT_PER_OPERATION as u64, MAX_OPERATIONS_PER_BLOCK as u64, 5000, // MAX_EVENTS_PER_QUERY, diff --git a/massa-execution-worker/src/active_history.rs b/massa-execution-worker/src/active_history.rs index 47c85a72722..1661adb05bc 100644 --- a/massa-execution-worker/src/active_history.rs +++ b/massa-execution-worker/src/active_history.rs @@ -567,7 +567,7 @@ mod test { // Expect updates to be empty (or default) here { let current_updates = AsyncMessageUpdate::default(); - let fetched = active_history.fetch_message(&message_id_3_2, current_updates); + let fetched = active_history.fetch_message(&message_id_3_2, current_updates, 1); if let HistorySearchResult::Present(SetUpdateOrDelete::Update(updates)) = fetched { assert_eq!(updates, AsyncMessageUpdate::default()); } else { diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 220459eb7f5..1bcea890cf9 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -301,6 +301,30 @@ impl ExecutionState { // append generated events to the final event store exec_out.events.finalize(); + + let ts = get_block_slot_timestamp( + self.config.thread_count, + self.config.t0, + self.config.genesis_timestamp, + exec_out.slot, + ) + .expect("Time overflow"); + + let cur_version = self + .final_state + .read() + .get_mip_store() + .get_network_version_active_at(ts); + + if cur_version == 0 { + // Truncate the events before saving them to the event store + // Note: this is only needed during the MIP transition period + // When it becomes active, we will refuse such events so no need to truncate them + for event in exec_out.events.0.iter_mut() { + event.data.truncate(self.config.max_event_size_v1); + } + } + self.final_events_cache.save_events(exec_out.events.0); // update the prometheus metrics From 39698eb8ca7218899d19bb50f68cb871e9b3d428 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 30 Dec 2024 11:03:35 +0100 Subject: [PATCH 58/63] Reset cache if version becomes active (#4791) * Reset cache if version becomes active * add &mut + cargo fmt * Update massa-execution-worker/src/execution.rs Co-authored-by: Modship * Update execution.rs * Update hd_cache.rs * Avoid taking a lock * Rename + add info!() on cache reset --------- Co-authored-by: Modship --- massa-execution-worker/src/execution.rs | 16 ++++++++-- massa-module-cache/src/controller.rs | 5 ++++ massa-module-cache/src/hd_cache.rs | 40 +++++++++++++++++++++---- massa-module-cache/src/lru_cache.rs | 4 +++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 1bcea890cf9..2f59093083f 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -140,6 +140,7 @@ pub(crate) struct ExecutionState { pub(crate) execution_info: Arc>, #[cfg(feature = "dump-block")] block_storage_backend: Arc>, + cur_execution_version: u32, } impl ExecutionState { @@ -188,14 +189,16 @@ impl ExecutionState { }))); // Create an empty placeholder execution context, with shared atomic access - let execution_context = Arc::new(Mutex::new(ExecutionContext::new( + let execution_context = ExecutionContext::new( config.clone(), final_state.clone(), active_history.clone(), module_cache.clone(), mip_store.clone(), execution_trail_hash, - ))); + ); + let cur_execution_version = execution_context.execution_component_version; + let execution_context = Arc::new(Mutex::new(execution_context)); // Instantiate the interface providing ABI access to the VM, share the execution context with it let execution_interface = Box::new(InterfaceImpl::new( @@ -238,6 +241,7 @@ impl ExecutionState { config, #[cfg(feature = "dump-block")] block_storage_backend, + cur_execution_version, } } @@ -1458,7 +1462,7 @@ impl ExecutionState { /// # Returns /// An `ExecutionOutput` structure summarizing the output of the executed slot pub fn execute_slot( - &self, + &mut self, slot: &Slot, exec_target: Option<&(BlockId, ExecutionBlockMetadata)>, selector: Box, @@ -1488,6 +1492,12 @@ impl ExecutionState { ); let execution_version = execution_context.execution_component_version; + if self.cur_execution_version != execution_version { + // Reset the cache because a new execution version has become active + info!("A new execution version has become active! Resetting the module-cache."); + self.module_cache.write().reset(); + self.cur_execution_version = execution_version; + } let mut deferred_calls_slot_gas = 0; diff --git a/massa-module-cache/src/controller.rs b/massa-module-cache/src/controller.rs index ca877615410..c6605366ec0 100644 --- a/massa-module-cache/src/controller.rs +++ b/massa-module-cache/src/controller.rs @@ -39,6 +39,11 @@ impl ModuleCache { } } + pub fn reset(&mut self) { + self.lru_cache.reset(); + self.hd_cache.reset(); + } + /// Internal function to compile and build `ModuleInfo` fn compile_cached( &mut self, diff --git a/massa-module-cache/src/hd_cache.rs b/massa-module-cache/src/hd_cache.rs index b5a8ed91ff0..299029db095 100644 --- a/massa-module-cache/src/hd_cache.rs +++ b/massa-module-cache/src/hd_cache.rs @@ -5,7 +5,7 @@ use massa_hash::Hash; use massa_sc_runtime::{CondomLimits, GasCosts, RuntimeModule}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; use rand::RngCore; -use rocksdb::{Direction, IteratorMode, WriteBatch, DB}; +use rocksdb::{Direction, IteratorMode, Options, WriteBatch, DB}; use std::path::PathBuf; use tracing::debug; @@ -36,7 +36,7 @@ macro_rules! metadata_key { pub(crate) struct HDCache { /// RocksDB database - db: DB, + db: Option, /// How many entries are in the db. Count is initialized at creation time by iterating /// over all the entries in the db then it is maintained in memory entry_count: usize, @@ -63,7 +63,7 @@ impl HDCache { let entry_count = db.iterator(IteratorMode::Start).count(); Self { - db, + db: Some(db), entry_count, max_entry_count, snip_amount, @@ -72,6 +72,20 @@ impl HDCache { } } + pub fn reset(&mut self) { + let path = self.db.as_ref().unwrap().path().to_path_buf(); + + // Close the existing database by dropping it + let _ = self.db.take(); + + // Destroy the database files + DB::destroy(&Options::default(), &path).expect("Failed to destroy the database"); + // Reopen the database + let db = DB::open_default(&path).expect(OPEN_ERROR); + self.db = Some(db); + self.entry_count = 0; + } + /// Insert a new module in the cache pub fn insert(&mut self, hash: Hash, module_info: ModuleInfo) { if self.entry_count >= self.max_entry_count { @@ -103,7 +117,11 @@ impl HDCache { let mut batch = WriteBatch::default(); batch.put(module_key!(hash), ser_module); batch.put(metadata_key!(hash), ser_metadata); - self.db.write(batch).expect(CRUD_ERROR); + self.db + .as_ref() + .expect(CRUD_ERROR) + .write(batch) + .expect(CRUD_ERROR); self.entry_count = self.entry_count.saturating_add(1); @@ -121,6 +139,8 @@ impl HDCache { .serialize(&ModuleMetadata::Delta(init_cost), &mut ser_metadata) .expect(DATA_SER_ERROR); self.db + .as_ref() + .expect(CRUD_ERROR) .put(metadata_key!(hash), ser_metadata) .expect(CRUD_ERROR); } @@ -132,6 +152,8 @@ impl HDCache { .serialize(&ModuleMetadata::Invalid(err_msg), &mut ser_metadata) .expect(DATA_SER_ERROR); self.db + .as_ref() + .expect(CRUD_ERROR) .put(metadata_key!(hash), ser_metadata) .expect(CRUD_ERROR); } @@ -145,6 +167,8 @@ impl HDCache { ) -> Option { let mut iterator = self .db + .as_ref() + .expect(CRUD_ERROR) .iterator(IteratorMode::From(&module_key!(hash), Direction::Forward)); if let (Some(Ok((key_1, ser_module))), Some(Ok((key_2, ser_metadata)))) = @@ -181,7 +205,7 @@ impl HDCache { /// Try to remove as much as `self.amount_to_snip` entries from the db fn snip(&mut self) { - let mut iter = self.db.raw_iterator(); + let mut iter = self.db.as_ref().expect(CRUD_ERROR).raw_iterator(); let mut batch = WriteBatch::default(); let mut snipped_count: usize = 0; @@ -218,7 +242,11 @@ impl HDCache { } // delete the key and reduce entry_count - self.db.write(batch).expect(CRUD_ERROR); + self.db + .as_ref() + .expect(CRUD_ERROR) + .write(batch) + .expect(CRUD_ERROR); self.entry_count -= snipped_count; } } diff --git a/massa-module-cache/src/lru_cache.rs b/massa-module-cache/src/lru_cache.rs index a6ea66eeea8..af810a8f71b 100644 --- a/massa-module-cache/src/lru_cache.rs +++ b/massa-module-cache/src/lru_cache.rs @@ -27,6 +27,10 @@ impl LRUCache { } } + pub fn reset(&mut self) { + self.cache.clear(); + } + /// If the module is contained in the cache: /// * retrieve a copy of it /// * move it up in the LRU cache From dee7e28e6c76ab71d15505c90197fa1a7472004f Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 30 Dec 2024 12:22:22 +0100 Subject: [PATCH 59/63] In HD DB reset(), Do not panic if fails to destroy HD cache --- massa-module-cache/src/hd_cache.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/massa-module-cache/src/hd_cache.rs b/massa-module-cache/src/hd_cache.rs index 0157b48018f..eece9790c3f 100644 --- a/massa-module-cache/src/hd_cache.rs +++ b/massa-module-cache/src/hd_cache.rs @@ -85,7 +85,11 @@ impl HDCache { let _ = self.db.take(); // Destroy the database files - DB::destroy(&Options::default(), &path).expect("Failed to destroy the database"); + if path.exists() { + if let Err(e) = DB::destroy(&Options::default(), path.clone()) { + warn!("Failed to destroy the db: {:?}", e); + } + } // Reopen the database let db = DB::open_default(&path).expect(OPEN_ERROR); self.db = Some(db); From b2018c49cbcc2a461d7add7cf3e1bc950859a8d5 Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Tue, 31 Dec 2024 14:26:48 +0100 Subject: [PATCH 60/63] Fix async message execution order in slot with no active version (#4799) * Update the order of execution of async msg in slot * Remove debug print --- massa-execution-worker/src/execution.rs | 129 ++++++++++++------------ 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 2f59093083f..076c4b46ffd 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -1505,8 +1505,38 @@ impl ExecutionState { match execution_version { 0 => { + // Get asynchronous messages to execute + let messages = execution_context.take_async_batch_v0( + self.config.max_async_gas, + self.config.async_msg_cst_gas_cost, + ); + // Apply the created execution context for slot execution *context_guard!(self) = execution_context; + + // Try executing asynchronous messages. + // Effects are cancelled on failure and the sender is reimbursed. + for (opt_bytecode, message) in messages { + match self.execute_async_message(message, opt_bytecode, execution_version) { + Ok(_message_return) => { + cfg_if::cfg_if! { + if #[cfg(feature = "execution-trace")] { + // Safe to unwrap + slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); + } else if #[cfg(feature = "execution-info")] { + slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); + exec_info.async_messages.push(Ok(_message_return)); + } + } + } + Err(err) => { + let msg = format!("failed executing async message: {}", err); + #[cfg(feature = "execution-info")] + exec_info.async_messages.push(Err(msg.clone())); + debug!(msg); + } + } + } } _ => { // Deferred calls @@ -1907,76 +1937,41 @@ impl ExecutionState { // Async msg execution - match execution_version { - 0 => { - // Get asynchronous messages to execute - let messages = context_guard!(self).take_async_batch_v0( - self.config.max_async_gas, - self.config.async_msg_cst_gas_cost, - ); - - // Try executing asynchronous messages. - // Effects are cancelled on failure and the sender is reimbursed. - for (opt_bytecode, message) in messages { - match self.execute_async_message(message, opt_bytecode, execution_version) { - Ok(_message_return) => { - cfg_if::cfg_if! { - if #[cfg(feature = "execution-trace")] { - // Safe to unwrap - slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); - } else if #[cfg(feature = "execution-info")] { - slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); - exec_info.async_messages.push(Ok(_message_return)); - } + if execution_version > 0 { + // Get asynchronous messages to execute + // The gas available for async messages is the remaining block gas + async remaining gas (max_async - gas used by deferred calls) + let async_msg_gas_available = self + .config + .max_async_gas + .saturating_sub(deferred_calls_slot_gas) + .saturating_add(remaining_block_gas); + + // Get asynchronous messages to execute + let messages = context_guard!(self) + .take_async_batch_v1(async_msg_gas_available, self.config.async_msg_cst_gas_cost); + + // Try executing asynchronous messages. + // Effects are cancelled on failure and the sender is reimbursed. + for (_message_id, message) in messages { + let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); + + match self.execute_async_message(message, opt_bytecode, execution_version) { + Ok(_message_return) => { + cfg_if::cfg_if! { + if #[cfg(feature = "execution-trace")] { + // Safe to unwrap + slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); + } else if #[cfg(feature = "execution-info")] { + slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); + exec_info.async_messages.push(Ok(_message_return)); } } - Err(err) => { - let msg = format!("failed executing async message: {}", err); - #[cfg(feature = "execution-info")] - exec_info.async_messages.push(Err(msg.clone())); - debug!(msg); - } } - } - } - _ => { - // Get asynchronous messages to execute - // The gas available for async messages is the remaining block gas + async remaining gas (max_async - gas used by deferred calls) - let async_msg_gas_available = self - .config - .max_async_gas - .saturating_sub(deferred_calls_slot_gas) - .saturating_add(remaining_block_gas); - - // Get asynchronous messages to execute - let messages = context_guard!(self).take_async_batch_v1( - async_msg_gas_available, - self.config.async_msg_cst_gas_cost, - ); - - // Try executing asynchronous messages. - // Effects are cancelled on failure and the sender is reimbursed. - for (_message_id, message) in messages { - let opt_bytecode = context_guard!(self).get_bytecode(&message.destination); - - match self.execute_async_message(message, opt_bytecode, execution_version) { - Ok(_message_return) => { - cfg_if::cfg_if! { - if #[cfg(feature = "execution-trace")] { - // Safe to unwrap - slot_trace.asc_call_stacks.push(_message_return.traces.unwrap().0); - } else if #[cfg(feature = "execution-info")] { - slot_trace.asc_call_stacks.push(_message_return.traces.clone().unwrap().0); - exec_info.async_messages.push(Ok(_message_return)); - } - } - } - Err(err) => { - let msg = format!("failed executing async message: {}", err); - #[cfg(feature = "execution-info")] - exec_info.async_messages.push(Err(msg.clone())); - debug!(msg); - } + Err(err) => { + let msg = format!("failed executing async message: {}", err); + #[cfg(feature = "execution-info")] + exec_info.async_messages.push(Err(msg.clone())); + debug!(msg); } } } From 582869d3572a7b285d2a121ff348749344507425 Mon Sep 17 00:00:00 2001 From: Modship Date: Fri, 3 Jan 2025 11:16:36 +0100 Subject: [PATCH 61/63] current_mip_version to get_status (#4803) --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- massa-api-exports/src/node.rs | 2 ++ massa-api/src/public.rs | 7 +++++++ massa-client/src/display.rs | 1 + massa-grpc/src/private.rs | 1 + massa-grpc/src/public.rs | 1 + massa-node/base_config/openrpc.json | 8 ++++++-- 8 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08c69832ce0..3090c0b4683 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2915,7 +2915,7 @@ dependencies = [ [[package]] name = "massa-proto-rs" version = "0.1.0" -source = "git+https://github.com/massalabs/massa-proto-rs?rev=b5267178eaf266ec724691d7de163e4c34343416#b5267178eaf266ec724691d7de163e4c34343416" +source = "git+https://github.com/massalabs/massa-proto-rs?rev=c9a7c55914f3d308996113df0936f15d96490313#c9a7c55914f3d308996113df0936f15d96490313" dependencies = [ "glob", "prost", @@ -2929,7 +2929,7 @@ dependencies = [ [[package]] name = "massa-sc-runtime" version = "0.10.0" -source = "git+https://github.com/massalabs/massa-sc-runtime?rev=f5a584b9f8050f332c9ed332bd0a40f8e0372807#f5a584b9f8050f332c9ed332bd0a40f8e0372807" +source = "git+https://github.com/massalabs/massa-sc-runtime?rev=cd14c775aff31b8c0f49064613118f428f9dd45f#cd14c775aff31b8c0f49064613118f428f9dd45f" dependencies = [ "anyhow", "as-ffi-bindings", diff --git a/Cargo.toml b/Cargo.toml index ab0a91e3c62..92009e4a3da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,8 +111,8 @@ massa_event_cache = { path = "./massa-event-cache" } # Massa projects dependencies # massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", branch = "deferred_calls" } # massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "branch" = "deferred_calls" } -massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "b5267178eaf266ec724691d7de163e4c34343416" } -massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "f5a584b9f8050f332c9ed332bd0a40f8e0372807" } +massa-proto-rs = { git = "https://github.com/massalabs/massa-proto-rs", "rev" = "c9a7c55914f3d308996113df0936f15d96490313" } +massa-sc-runtime = { git = "https://github.com/massalabs/massa-sc-runtime", "rev" = "cd14c775aff31b8c0f49064613118f428f9dd45f" } peernet = { git = "https://github.com/massalabs/PeerNet", "rev" = "04b05ddd320fbe76cc858115af7b5fc28bdb8310" } # Dev only - use local dependencies diff --git a/massa-api-exports/src/node.rs b/massa-api-exports/src/node.rs index 9676aab2dd7..117978a1a25 100644 --- a/massa-api-exports/src/node.rs +++ b/massa-api-exports/src/node.rs @@ -46,6 +46,8 @@ pub struct NodeStatus { pub chain_id: u64, /// minimal fees to include an operation in a block pub minimal_fees: Amount, + /// current mip version + pub current_mip_version: u32, } impl std::fmt::Display for NodeStatus { diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index e28d638fe0e..1b0ea13e0d1 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -458,6 +458,12 @@ impl MassaRpcServer for API { let config = CompactConfig::default(); let now = MassaTime::now(); + let current_mip_version = self + .0 + .keypair_factory + .mip_store + .get_network_version_current(); + let last_slot_result = get_latest_block_slot_at_timestamp( api_settings.thread_count, api_settings.t0, @@ -555,6 +561,7 @@ impl MassaRpcServer for API { current_cycle, chain_id: self.0.api_settings.chain_id, minimal_fees: self.0.api_settings.minimal_fees, + current_mip_version, }) } diff --git a/massa-client/src/display.rs b/massa-client/src/display.rs index 5973228ef5b..f7bdc433016 100644 --- a/massa-client/src/display.rs +++ b/massa-client/src/display.rs @@ -275,6 +275,7 @@ impl Output for NodeStatus { println!(); println!("Chain id: {}", self.chain_id); + println!("Current MIP version: {}", self.current_mip_version); } } diff --git a/massa-grpc/src/private.rs b/massa-grpc/src/private.rs index dc362bf2a60..ab2fd0d5407 100644 --- a/massa-grpc/src/private.rs +++ b/massa-grpc/src/private.rs @@ -305,6 +305,7 @@ pub(crate) fn get_node_status( execution_stats: Some(execution_stats.into()), config: Some(config.into()), chain_id: grpc.grpc_config.chain_id, + current_mip_version: grpc.mip_store.get_network_version_current(), }; Ok(grpc_api::GetNodeStatusResponse { diff --git a/massa-grpc/src/public.rs b/massa-grpc/src/public.rs index 6dd98bed3c1..7dd1ec2e0f9 100644 --- a/massa-grpc/src/public.rs +++ b/massa-grpc/src/public.rs @@ -960,6 +960,7 @@ pub(crate) fn get_status( config: Some(config.into()), chain_id: grpc.grpc_config.chain_id, minimal_fees: Some(grpc.grpc_config.minimal_fees.into()), + current_mip_version: grpc.keypair_factory.mip_store.get_network_version_current(), }; Ok(grpc_api::GetStatusResponse { diff --git a/massa-node/base_config/openrpc.json b/massa-node/base_config/openrpc.json index b6d2b090e6c..6c049080890 100644 --- a/massa-node/base_config/openrpc.json +++ b/massa-node/base_config/openrpc.json @@ -1999,7 +1999,6 @@ "candidate_value": { "description": "The candidate datastore entry value bytes", "$ref": "#/components/schemas/BytesOption" - }, "final_value": { "description": "The final datastore entry value bytes", @@ -2781,7 +2780,8 @@ "pool_stats", "version", "execution_stats", - "chain_id" + "chain_id", + "current_mip_version" ], "type": "object", "properties": { @@ -2859,6 +2859,10 @@ "minimal_fees": { "description": "Minimal fee", "$ref": "#/components/schemas/Amount" + }, + "current_mip_version": { + "description": "Current MIP version", + "type": "number" } }, "additionalProperties": false From fa9acda857f4b3ec45c13101c22d78a0a9c3c55a Mon Sep 17 00:00:00 2001 From: Leo-Besancon Date: Mon, 6 Jan 2025 16:42:40 +0100 Subject: [PATCH 62/63] Fix versioned TU --- massa-ledger-worker/src/ledger_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-ledger-worker/src/ledger_db.rs b/massa-ledger-worker/src/ledger_db.rs index 235eeefe8cc..800c88953d4 100644 --- a/massa-ledger-worker/src/ledger_db.rs +++ b/massa-ledger-worker/src/ledger_db.rs @@ -732,7 +732,7 @@ mod tests { let mut batch = DBBatch::new(); let guard = ledger_db.db.read(); - delete_datastore_entries(&addr, &guard, &mut batch); + delete_datastore_entries(&addr, &guard, &mut batch, 1); drop(guard); let mut guard = ledger_db.db.write(); From c8b0b372fed5a255ec3b47dc7a25a9af953394bb Mon Sep 17 00:00:00 2001 From: Modship Date: Wed, 22 Jan 2025 20:18:04 +0100 Subject: [PATCH 63/63] metrics block version (#4832) * add metric for current_active_version * Metrics : remove counter for version 0 when current ver > 0 --- massa-metrics/src/lib.rs | 26 ++++++++++++++++++++++++++ massa-node/src/survey.rs | 1 + 2 files changed, 27 insertions(+) diff --git a/massa-metrics/src/lib.rs b/massa-metrics/src/lib.rs index 98da5ace180..cc539b01324 100644 --- a/massa-metrics/src/lib.rs +++ b/massa-metrics/src/lib.rs @@ -201,6 +201,7 @@ pub struct MassaMetrics { // network versions votes network_versions_votes: Arc>>, + network_current_version: IntGauge, pub tick_delay: Duration, @@ -451,6 +452,9 @@ impl MassaMetrics { let deferred_calls_failed = IntCounter::new("deferred_calls_failed", "number of deferred calls failed").unwrap(); + let network_current_version = + IntGauge::new("network_current_version", "current version of network").unwrap(); + let mut stopper = MetricsStopper::default(); if enabled { @@ -505,6 +509,7 @@ impl MassaMetrics { let _ = prometheus::register(Box::new(block_slot_delay.clone())); let _ = prometheus::register(Box::new(deferred_calls_executed.clone())); let _ = prometheus::register(Box::new(deferred_calls_failed.clone())); + let _ = prometheus::register(Box::new(network_current_version.clone())); stopper = server::bind_metrics(addr); } @@ -561,6 +566,7 @@ impl MassaMetrics { final_cursor_period, peers_bandwidth: Arc::new(RwLock::new(HashMap::new())), network_versions_votes: Arc::new(RwLock::new(HashMap::new())), + network_current_version, tick_delay, deferred_calls_executed, deferred_calls_failed, @@ -760,11 +766,17 @@ impl MassaMetrics { self.deferred_calls_failed.inc(); } + pub fn set_network_current_version(&self, version: u32) { + self.network_current_version.set(version as i64); + } + // Update the network version vote metrics pub fn update_network_version_vote(&self, data: HashMap) { if self.enabled { let mut write = self.network_versions_votes.write().unwrap(); + let current_version: u32 = self.network_current_version.get() as u32; + { let missing_version = write .keys() @@ -779,9 +791,23 @@ impl MassaMetrics { } } } + + if current_version > 0 { + // remove metrics for version 0 if we have a current version > 0 + // in this case 0 means no vote + if let Some(counter) = write.remove(&0) { + if let Err(e) = prometheus::unregister(Box::new(counter)) { + warn!("Failed to unregister network_version_vote_0 : {}", e); + } + } + } } for (version, count) in data.into_iter() { + if version.eq(&0) && current_version > 0 { + // skip version 0 if we have a current version + continue; + } if let Some(actual_counter) = write.get_mut(&version) { actual_counter.set(count as i64); } else { diff --git a/massa-node/src/survey.rs b/massa-node/src/survey.rs index a4dc7c5accd..3ba5a2cc2d9 100644 --- a/massa-node/src/survey.rs +++ b/massa-node/src/survey.rs @@ -134,6 +134,7 @@ impl MassaSurvey { } { + massa_metrics.set_network_current_version(mip_store.get_network_version_current()); let network_stats= mip_store.0.read().get_network_versions_stats(); massa_metrics.update_network_version_vote(network_stats); }