Merge branch 'mainnet_2_3' into mainnet_2_3_versioning_mip

.cargo/config.toml

@@ -1,2 +1,2 @@
 [alias]
-xtask = "run --package massa_xtask --"
+xtask = "run --features gas_calibration --package massa_xtask --"

massa-db-exports/src/constants.rs

@@ -61,3 +61,6 @@ pub const KEY_LEN_SER_ERROR: &str = "critical: key length serialization failed";
 // deferred calls
 pub const DEFERRED_CALL_DESER_ERROR: &str = "critical: message deserialization failed";
 pub const DEFERRED_CALL_SER_ERROR: &str = "critical: message serialization failed";
+pub const DEFERRED_CALL_TOTAL_GAS: &str = "deferred_call_total_gas";
+
+pub const DEFERRED_CALL_TOTAL_REGISTERED: &str = "deferred_call_total_registered";

massa-deferred-calls/src/call.rs

@@ -156,12 +156,12 @@ impl Serializer<DeferredCall> for DeferredCallSerializer {
 #[derive(Clone)]
 pub struct DeferredCallDeserializer {
     slot_deserializer: SlotDeserializer,
-    address_deserializer: AddressDeserializer,
-    string_deserializer: StringDeserializer<U16VarIntDeserializer, u16>,
-    vec_u8_deserializer: VecU8Deserializer,
+    pub(crate) address_deserializer: AddressDeserializer,
+    pub(crate) string_deserializer: StringDeserializer<U16VarIntDeserializer, u16>,
+    pub(crate) vec_u8_deserializer: VecU8Deserializer,
     pub(crate) amount_deserializer: AmountDeserializer,
     pub(crate) u64_var_int_deserializer: U64VarIntDeserializer,
-    bool_deserializer: BoolDeserializer,
+    pub(crate) bool_deserializer: BoolDeserializer,
 }
 
 impl DeferredCallDeserializer {

massa-deferred-calls/src/lib.rs

@@ -1,11 +1,17 @@
 use call::{DeferredCallDeserializer, DeferredCallSerializer};
 use config::DeferredCallsConfig;
-use macros::{DEFERRED_CALL_TOTAL_GAS, DEFERRED_CALL_TOTAL_REGISTERED};
+use macros::{
+    CALL_FIELD_CANCELED, CALL_FIELD_COINS, CALL_FIELD_FEE, CALL_FIELD_MAX_GAS,
+    CALL_FIELD_PARAMETERS, CALL_FIELD_SENDER_ADDRESS, CALL_FIELD_TARGET_ADDRESS,
+    CALL_FIELD_TARGET_FUNCTION, CALL_FIELD_TARGET_SLOT,
+};
 use massa_db_exports::{
     DBBatch, ShareableMassaDBController, CRUD_ERROR, DEFERRED_CALLS_PREFIX,
-    DEFERRED_CALL_DESER_ERROR, DEFERRED_CALL_SER_ERROR, KEY_DESER_ERROR, STATE_CF,
+    DEFERRED_CALL_DESER_ERROR, DEFERRED_CALL_SER_ERROR, DEFERRED_CALL_TOTAL_GAS,
+    DEFERRED_CALL_TOTAL_REGISTERED, KEY_DESER_ERROR, STATE_CF,
 };
-use massa_serialization::{DeserializeError, Deserializer, Serializer};
+use massa_models::address::Address;
+use massa_serialization::{buf_to_array_ctr, DeserializeError, Deserializer, Serializer};
 use registry_changes::{
     DeferredCallRegistryChanges, DeferredRegistryChangesDeserializer,
     DeferredRegistryChangesSerializer,
@@ -434,6 +440,128 @@ impl DeferredCallRegistry {
             DeferredRegistryGasChange::Keep => {}
         }
     }
+
+    pub fn is_key_value_valid(&self, serialized_key: &[u8], serialized_value: &[u8]) -> bool {
+        if serialized_key.starts_with(DEFERRED_CALLS_PREFIX.as_bytes()) {
+            // check for  [DEFERRED_CALLS_PREFIX][slot]
+            if let Some((_rest, slot)) = buf_to_array_ctr(
+                &serialized_key[DEFERRED_CALLS_PREFIX.len()..],
+                Slot::from_bytes_key,
+            ) {
+                // check for  [DEFERRED_CALLS_PREFIX][slot][SLOT_TOTAL_GAS]
+                if serialized_key
+                    .starts_with(&deferred_call_slot_total_gas_key!(&slot.to_bytes_key()))
+                {
+                    return self
+                        .call_deserializer
+                        .u64_var_int_deserializer
+                        .deserialize::<DeserializeError>(serialized_value)
+                        .is_ok();
+                } else if serialized_key
+                    .starts_with(&deferred_call_slot_base_fee_key!(&slot.to_bytes_key()))
+                {
+                    // check for  [DEFERRED_CALLS_PREFIX][slot][SLOT_BASE_FEE]
+                    return self
+                        .call_deserializer
+                        .amount_deserializer
+                        .deserialize::<DeserializeError>(serialized_value)
+                        .is_ok();
+                } else {
+                    // check for  [DEFERRED_CALLS_PREFIX][slot][CALLS_TAG][id][CALL_FIELD_X_TAG]
+
+                    // [DEFERRED_CALLS_PREFIX][slot][CALLS_TAG]
+                    let k = deferred_slot_call_prefix_key!(&slot.to_bytes_key());
+                    let rest_key = &serialized_key[k.len()..];
+
+                    if let Ok((rest, _id)) = self
+                        .call_id_deserializer
+                        .deserialize::<DeserializeError>(rest_key)
+                    {
+                        match rest[0] {
+                            CALL_FIELD_SENDER_ADDRESS => {
+                                let res: Result<(&[u8], Address), nom::Err<DeserializeError<'_>>> =
+                                    self.call_deserializer
+                                        .address_deserializer
+                                        .deserialize::<DeserializeError>(serialized_value);
+
+                                return res.is_ok();
+                            }
+                            CALL_FIELD_TARGET_SLOT => {
+                                return self
+                                    .registry_changes_deserializer
+                                    .slot_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok()
+                            }
+                            CALL_FIELD_TARGET_ADDRESS => {
+                                let res: Result<(&[u8], Address), nom::Err<DeserializeError<'_>>> =
+                                    self.call_deserializer
+                                        .address_deserializer
+                                        .deserialize::<DeserializeError>(serialized_value);
+                                return res.is_ok();
+                            }
+                            CALL_FIELD_TARGET_FUNCTION => {
+                                return self
+                                    .call_deserializer
+                                    .string_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok()
+                            }
+                            CALL_FIELD_PARAMETERS => {
+                                return self
+                                    .call_deserializer
+                                    .vec_u8_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok()
+                            }
+                            CALL_FIELD_MAX_GAS => {
+                                return self
+                                    .call_deserializer
+                                    .u64_var_int_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok()
+                            }
+                            CALL_FIELD_FEE => {
+                                return self
+                                    .call_deserializer
+                                    .amount_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok()
+                            }
+                            CALL_FIELD_CANCELED => {
+                                return self
+                                    .call_deserializer
+                                    .bool_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok();
+                            }
+                            CALL_FIELD_COINS => {
+                                return self
+                                    .call_deserializer
+                                    .amount_deserializer
+                                    .deserialize::<DeserializeError>(serialized_value)
+                                    .is_ok();
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+            }
+        } else if serialized_key.eq(DEFERRED_CALL_TOTAL_GAS.as_bytes()) {
+            return self
+                .registry_changes_deserializer
+                .effective_total_gas_deserializer
+                .deserialize::<DeserializeError>(serialized_value)
+                .is_ok();
+        } else if serialized_key.eq(DEFERRED_CALL_TOTAL_REGISTERED.as_bytes()) {
+            return self
+                .registry_changes_deserializer
+                .total_calls_registered_deserializer
+                .deserialize::<DeserializeError>(serialized_value)
+                .is_ok();
+        }
+        false
+    }
 }
 
 pub type DeferredRegistryCallChange = SetOrDelete<DeferredCall>;

massa-deferred-calls/src/macros.rs

@@ -1,7 +1,3 @@
-pub(crate) const DEFERRED_CALL_TOTAL_GAS: &str = "deferred_call_total_gas";
-
-pub(crate) const DEFERRED_CALL_TOTAL_REGISTERED: &str = "deferred_call_total_registered";
-
 pub(crate) const CALLS_TAG: u8 = 0u8;
 // slot fields
 pub(crate) const SLOT_TOTAL_GAS: u8 = 1u8;

massa-deferred-calls/src/registry_changes.rs

@@ -176,7 +176,7 @@ impl Serializer<DeferredCallRegistryChanges> for DeferredRegistryChangesSerializ
 pub struct DeferredRegistryChangesDeserializer {
     pub(crate) u64_deserializer: U64VarIntDeserializer,
     slot_changes_deserializer: DeferredRegistrySlotChangesDeserializer,
-    slot_deserializer: SlotDeserializer,
+    pub(crate) slot_deserializer: SlotDeserializer,
     pub(crate) effective_total_gas_deserializer:
         SetOrKeepDeserializer<u128, U128VarIntDeserializer>,
     pub(crate) total_calls_registered_deserializer:

massa-event-cache/src/event_cache.rs

@@ -677,21 +677,28 @@ impl EventCache {
     ) -> Result<u64, ModelsError> {
         match filter_item {
             FilterItem::SlotStart(start) => {
-                let diff = self.last_slot.slots_since(start, self.thread_count)?;
-                // Note: Pessimistic estimation - should we keep an average count of events per slot
-                //       and use that instead?
+                let diff = self
+                    .last_slot
+                    .slots_since(start, self.thread_count)
+                    .unwrap_or(0); // If start is after self.last_slot, we have no events in the filter
+                                   // Note: Pessimistic estimation - should we keep an average count of events per slot
+                                   //       and use that instead?
                 Ok(diff
                     .saturating_mul(self.max_events_per_operation)
                     .saturating_mul(self.max_operations_per_block))
             }
             FilterItem::SlotStartEnd(start, end) => {
-                let diff = end.slots_since(start, self.thread_count)?;
+                // If start is after end, we have no events in the filter as it is inconsistent
+                let diff = end.slots_since(start, self.thread_count).unwrap_or(0);
                 Ok(diff
                     .saturating_mul(self.max_events_per_operation)
                     .saturating_mul(self.max_operations_per_block))
             }
             FilterItem::SlotEnd(end) => {
-                let diff = end.slots_since(&self.first_slot, self.thread_count)?;
+                // If end is after self.first_slot, we have no events in the filter
+                let diff = end
+                    .slots_since(&self.first_slot, self.thread_count)
+                    .unwrap_or(0);
                 Ok(diff
                     .saturating_mul(self.max_events_per_operation)
                     .saturating_mul(self.max_operations_per_block))
@@ -1799,6 +1806,54 @@ mod tests {
         assert_eq!(emit_addr_1_count, (threshold) as u64);
         assert_eq!(emit_addr_2_count, (threshold + 1 + 2) as u64);
 
+        // Check if we correctly count the number of events in the DB with slot related filters
+        // First with filters that should give a count of 0 (too early, too late, or inconsistent start/end)
+        let slot_start_1 = Slot::new(3, 0); // start is after the last slot
+        let slot_end_1 = Slot::new(0, 0); // end is before the first slot
+        let slot_start_end_1 = Slot::new(2, 0)..Slot::new(1, 0); // start is after end
+
+        // Then with filters that should give a positive count
+        // num_thread * max_events_per_operation * max_operations_per_block
+        let slot_start_2 = Slot::new(1, 0);
+        let slot_end_2 = Slot::new(2, 0);
+        let slot_start_end_2 = Slot::new(1, 0)..Slot::new(2, 0);
+
+        let slot_start_count_1 = cache
+            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_1))
+            .unwrap();
+        let slot_end_count_1 = cache
+            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_1))
+            .unwrap();
+        let slot_start_end_count_1 = cache
+            .filter_item_estimate_count(
+                &KeyIndent::Event,
+                &FilterItem::SlotStartEnd(slot_start_end_1.start, slot_start_end_1.end),
+            )
+            .unwrap();
+
+        let slot_start_count_2 = cache
+            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_2))
+            .unwrap();
+        let slot_end_count_2 = cache
+            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_2))
+            .unwrap();
+        let slot_start_end_count_2 = cache
+            .filter_item_estimate_count(
+                &KeyIndent::Event,
+                &FilterItem::SlotStartEnd(slot_start_end_2.start, slot_start_end_2.end),
+            )
+            .unwrap();
+
+        assert_eq!(slot_start_count_1, 0);
+        assert_eq!(slot_end_count_1, 0);
+        assert_eq!(slot_start_end_count_1, 0);
+
+        let expected_count_per_period =
+            THREAD_COUNT as u64 * MAX_EVENT_PER_OPERATION as u64 * MAX_OPERATIONS_PER_BLOCK as u64;
+        assert_eq!(slot_start_count_2, expected_count_per_period);
+        assert_eq!(slot_end_count_2, 2 * expected_count_per_period); // 2 periods as first_slot is (0,0)
+        assert_eq!(slot_start_end_count_2, expected_count_per_period);
+
         // Check if we query first by emitter address then is_error
 
         let filter_1 = EventFilter {

massa-execution-worker/src/execution.rs

@@ -1950,6 +1950,9 @@ impl ExecutionState {
             let messages = context_guard!(self)
                 .take_async_batch_v1(async_msg_gas_available, self.config.async_msg_cst_gas_cost);
 
+            // clear operation id (otherwise events will be generated using this operation id)
+            self.execution_context.lock().origin_operation_id = None;
+
             // Try executing asynchronous messages.
             // Effects are cancelled on failure and the sender is reimbursed.
             for (_message_id, message) in messages {

massa-final-state/src/final_state.rs

@@ -12,7 +12,8 @@ use anyhow::{anyhow, Result as AnyResult};
 use massa_async_pool::AsyncPool;
 use massa_db_exports::{
     DBBatch, MassaIteratorMode, ShareableMassaDBController, ASYNC_POOL_PREFIX,
-    CYCLE_HISTORY_PREFIX, DEFERRED_CREDITS_PREFIX, EXECUTED_DENUNCIATIONS_PREFIX,
+    CYCLE_HISTORY_PREFIX, DEFERRED_CALLS_PREFIX, DEFERRED_CALL_TOTAL_GAS,
+    DEFERRED_CALL_TOTAL_REGISTERED, DEFERRED_CREDITS_PREFIX, EXECUTED_DENUNCIATIONS_PREFIX,
     EXECUTED_OPS_PREFIX, LEDGER_PREFIX, MIP_STORE_PREFIX, STATE_CF,
 };
 use massa_db_exports::{EXECUTION_TRAIL_HASH_PREFIX, MIP_STORE_STATS_PREFIX, VERSIONING_CF};
@@ -670,6 +671,20 @@ impl FinalState {
                 }
             } else if serialized_key.starts_with(EXECUTION_TRAIL_HASH_PREFIX.as_bytes()) {
                 // no checks here as they are performed above by direct reading
+            } else if serialized_key.starts_with(DEFERRED_CALLS_PREFIX.as_bytes())
+                || serialized_key.eq(DEFERRED_CALL_TOTAL_GAS.as_bytes())
+                || serialized_key.eq(DEFERRED_CALL_TOTAL_REGISTERED.as_bytes())
+            {
+                if !self
+                    .deferred_call_registry
+                    .is_key_value_valid(&serialized_key, &serialized_value)
+                {
+                    warn!("Wrong key/value for DEFERRED_CALLS_PREFIX serialized_key: {:?}, serialized_value: {:?}", serialized_key, serialized_value);
+                    return Err(anyhow!(
+                        "Wrong key/value for DEFERRED_CALLS_PREFIX serialized_key: {:?}, serialized_value: {:?}",
+                        serialized_key, serialized_value
+                    ));
+                }
             } else {
                 warn!(
                     "Key/value does not correspond to any prefix: serialized_key: {:?}, serialized_value: {:?}",