From 094f99d173b860132c711bebdf4833f951e772d4 Mon Sep 17 00:00:00 2001 From: Paul Liu Date: Mon, 14 Oct 2024 08:20:32 +0800 Subject: [PATCH] feat(ckbtc): rotate KYT RPC providers in case of transient error (#1864) XC-159: Support multiple RPC providers Support multiple bitcoin mainnet RPC providers in the KYT canister. Each RPC call will go to one provider, and the next call will go to the next provider and so on, in a round robin manner. When an RPC call returns http error code, it is treated as a transient error so that the same call can be tried again with the next provider. Because RPC errors can occur for all providers indefinitely, it is up to the caller to decide whether or not to call `check_transaction` again. --------- Co-authored-by: gregorydemay <112856886+gregorydemay@users.noreply.github.com> --- rs/bitcoin/kyt/src/dashboard.rs | 2 +- rs/bitcoin/kyt/src/fetch.rs | 55 ++++++++++++---- rs/bitcoin/kyt/src/fetch/tests.rs | 69 ++++++++++++++++---- rs/bitcoin/kyt/src/lib.rs | 17 +++-- rs/bitcoin/kyt/src/providers.rs | 101 +++++++++++++++++++++++++++--- rs/bitcoin/kyt/src/state.rs | 11 +++- rs/bitcoin/kyt/tests/tests.rs | 87 ++++++++++++++++++++++++- 7 files changed, 294 insertions(+), 48 deletions(-) diff --git a/rs/bitcoin/kyt/src/dashboard.rs b/rs/bitcoin/kyt/src/dashboard.rs index 2bd6b698a9b..cbeca4c2d9a 100644 --- a/rs/bitcoin/kyt/src/dashboard.rs +++ b/rs/bitcoin/kyt/src/dashboard.rs @@ -113,7 +113,7 @@ pub fn dashboard(page_index: usize) -> DashboardTemplate { match status { FetchTxStatus::PendingOutcall => Status::PendingOutcall, FetchTxStatus::PendingRetry { .. } => Status::PendingRetry, - FetchTxStatus::Error(err) => Status::Error(format!("{:?}", err)), + FetchTxStatus::Error(err) => Status::Error(format!("{:?}", err.error)), FetchTxStatus::Fetched(fetched) => { // Return an empty list if no input address is available yet. let input_addresses = diff --git a/rs/bitcoin/kyt/src/fetch.rs b/rs/bitcoin/kyt/src/fetch.rs index 9abdddd7bb4..a9bd31609dd 100644 --- a/rs/bitcoin/kyt/src/fetch.rs +++ b/rs/bitcoin/kyt/src/fetch.rs @@ -1,6 +1,9 @@ -use crate::state::{FetchGuardError, FetchTxStatus, FetchedTx, HttpGetTxError, TransactionKytData}; +use crate::state::{ + FetchGuardError, FetchTxStatus, FetchTxStatusError, FetchedTx, HttpGetTxError, + TransactionKytData, +}; use crate::types::{CheckTransactionResponse, CheckTransactionRetriable, CheckTransactionStatus}; -use crate::{blocklist_contains, state}; +use crate::{blocklist_contains, providers, state, BtcNetwork}; use bitcoin::Transaction; use futures::future::try_join_all; use ic_btc_interface::Txid; @@ -44,7 +47,6 @@ pub enum FetchResult { pub enum TryFetchResult { Pending, HighLoad, - Error(HttpGetTxError), NotEnoughCycles, Fetched(FetchedTx), ToFetch(F), @@ -54,8 +56,11 @@ pub enum TryFetchResult { pub trait FetchEnv { type FetchGuard; fn new_fetch_guard(&self, txid: Txid) -> Result; + fn btc_network(&self) -> BtcNetwork; + async fn http_get_tx( &self, + provider: providers::Provider, txid: Txid, max_response_bytes: u32, ) -> Result; @@ -71,13 +76,24 @@ pub trait FetchEnv { &self, txid: Txid, ) -> TryFetchResult>> { - let max_response_bytes = match state::get_fetch_status(txid) { - None => INITIAL_MAX_RESPONSE_BYTES, + let (provider, max_response_bytes) = match state::get_fetch_status(txid) { + None => ( + providers::next_provider(self.btc_network()), + INITIAL_MAX_RESPONSE_BYTES, + ), Some(FetchTxStatus::PendingRetry { max_response_bytes, .. - }) => max_response_bytes, + }) => ( + providers::next_provider(self.btc_network()), + max_response_bytes, + ), Some(FetchTxStatus::PendingOutcall { .. }) => return TryFetchResult::Pending, - Some(FetchTxStatus::Error(msg)) => return TryFetchResult::Error(msg), + Some(FetchTxStatus::Error(err)) => ( + // An FetchTxStatus error can be retried with another provider + err.provider.next(), + // The next provider can use the same max_response_bytes + err.max_response_bytes, + ), Some(FetchTxStatus::Fetched(fetched)) => return TryFetchResult::Fetched(fetched), }; let guard = match self.new_fetch_guard(txid) { @@ -88,7 +104,7 @@ pub trait FetchEnv { if self.cycles_accept(cycle_cost) < cycle_cost { TryFetchResult::NotEnoughCycles } else { - TryFetchResult::ToFetch(self.fetch_tx(guard, txid, max_response_bytes)) + TryFetchResult::ToFetch(self.fetch_tx(guard, provider, txid, max_response_bytes)) } } @@ -104,10 +120,11 @@ pub trait FetchEnv { async fn fetch_tx( &self, _guard: Self::FetchGuard, + provider: providers::Provider, txid: Txid, max_response_bytes: u32, ) -> Result { - match self.http_get_tx(txid, max_response_bytes).await { + match self.http_get_tx(provider, txid, max_response_bytes).await { Ok(tx) => { let input_addresses = tx.input.iter().map(|_| None).collect(); match TransactionKytData::try_from(tx) { @@ -121,7 +138,14 @@ pub trait FetchEnv { } Err(err) => { let err = HttpGetTxError::TxEncoding(err.to_string()); - state::set_fetch_status(txid, FetchTxStatus::Error(err.clone())); + state::set_fetch_status( + txid, + FetchTxStatus::Error(FetchTxStatusError { + provider, + max_response_bytes, + error: err.clone(), + }), + ); Ok(FetchResult::Error(err)) } } @@ -138,7 +162,14 @@ pub trait FetchEnv { Ok(FetchResult::RetryWithBiggerBuffer) } Err(err) => { - state::set_fetch_status(txid, FetchTxStatus::Error(err.clone())); + state::set_fetch_status( + txid, + FetchTxStatus::Error(FetchTxStatusError { + provider, + max_response_bytes, + error: err.clone(), + }), + ); Ok(FetchResult::Error(err)) } } @@ -191,7 +222,7 @@ pub trait FetchEnv { state::set_fetched_address(txid, index, address.clone()); } Pending => continue, - HighLoad | NotEnoughCycles | Error(_) => break, + HighLoad | NotEnoughCycles => break, } } } diff --git a/rs/bitcoin/kyt/src/fetch/tests.rs b/rs/bitcoin/kyt/src/fetch/tests.rs index 3024d8cd5c8..6bfb4513d7b 100644 --- a/rs/bitcoin/kyt/src/fetch/tests.rs +++ b/rs/bitcoin/kyt/src/fetch/tests.rs @@ -1,6 +1,7 @@ use super::*; -use crate::types::BtcNetwork; -use crate::{blocklist, CheckTransactionIrrecoverableError}; +use crate::{ + blocklist, providers::Provider, types::BtcNetwork, CheckTransactionIrrecoverableError, +}; use bitcoin::{ absolute::LockTime, address::Address, hashes::Hash, transaction::Version, Amount, OutPoint, PubkeyHash, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Witness, @@ -18,6 +19,7 @@ struct MockEnv { replies: RefCell>>, available_cycles: RefCell, accepted_cycles: RefCell, + called_provider: RefCell>, } impl FetchEnv for MockEnv { @@ -30,19 +32,26 @@ impl FetchEnv for MockEnv { Ok(()) } } + + fn btc_network(&self) -> BtcNetwork { + BtcNetwork::Mainnet + } + async fn http_get_tx( &self, + provider: Provider, txid: Txid, max_response_bytes: u32, ) -> Result { self.calls .borrow_mut() .push_back((txid, max_response_bytes)); + *self.called_provider.borrow_mut() = Some(provider); self.replies .borrow_mut() .pop_front() .unwrap_or(Err(HttpGetTxError::Rejected { - code: RejectionCode::from(0), + code: RejectionCode::SysTransient, message: "no more reply".to_string(), })) } @@ -67,6 +76,7 @@ impl MockEnv { replies: RefCell::new(VecDeque::new()), available_cycles: RefCell::new(available_cycles), accepted_cycles: RefCell::new(0), + called_provider: RefCell::new(None), } } fn assert_get_tx_call(&self, txid: Txid, max_response_bytes: u32) { @@ -137,6 +147,7 @@ fn mock_transaction_with_inputs(input_txids: Vec<(Txid, u32)>) -> Transaction { async fn test_mock_env() { // Test cycle mock functions let env = MockEnv::new(CHECK_TRANSACTION_CYCLES_REQUIRED); + let provider = providers::next_provider(env.btc_network()); assert_eq!( env.cycles_accept(CHECK_TRANSACTION_CYCLES_SERVICE_FEE), CHECK_TRANSACTION_CYCLES_SERVICE_FEE @@ -155,7 +166,9 @@ async fn test_mock_env() { let env = MockEnv::new(0); let txid = mock_txid(0); env.expect_get_tx_with_reply(Ok(mock_transaction())); - let result = env.http_get_tx(txid, INITIAL_MAX_RESPONSE_BYTES).await; + let result = env + .http_get_tx(provider, txid, INITIAL_MAX_RESPONSE_BYTES) + .await; assert!(result.is_ok()); env.assert_get_tx_call(txid, INITIAL_MAX_RESPONSE_BYTES); env.assert_no_more_get_tx_call(); @@ -210,6 +223,7 @@ fn test_try_fetch_tx() { #[tokio::test] async fn test_fetch_tx() { let env = MockEnv::new(CHECK_TRANSACTION_CYCLES_REQUIRED); + let provider = providers::next_provider(env.btc_network()); let txid_0 = mock_txid(0); let txid_1 = mock_txid(1); let txid_2 = mock_txid(2); @@ -218,7 +232,9 @@ async fn test_fetch_tx() { let tx_0 = mock_transaction_with_inputs(vec![(txid_1, 0), (txid_2, 1)]); env.expect_get_tx_with_reply(Ok(tx_0.clone())); - let result = env.fetch_tx((), txid_0, INITIAL_MAX_RESPONSE_BYTES).await; + let result = env + .fetch_tx((), provider, txid_0, INITIAL_MAX_RESPONSE_BYTES) + .await; assert!(matches!(result, Ok(FetchResult::Fetched(_)))); assert!(matches!( state::get_fetch_status(txid_0), @@ -233,7 +249,9 @@ async fn test_fetch_tx() { // case RetryWithBiggerBuffer env.expect_get_tx_with_reply(Err(HttpGetTxError::ResponseTooLarge)); - let result = env.fetch_tx((), txid_1, INITIAL_MAX_RESPONSE_BYTES).await; + let result = env + .fetch_tx((), provider, txid_1, INITIAL_MAX_RESPONSE_BYTES) + .await; assert!(matches!(result, Ok(FetchResult::RetryWithBiggerBuffer))); assert!(matches!( state::get_fetch_status(txid_1), @@ -243,23 +261,25 @@ async fn test_fetch_tx() { env.expect_get_tx_with_reply(Err(HttpGetTxError::TxEncoding( "failed to decode tx".to_string(), ))); - let result = env.fetch_tx((), txid_2, INITIAL_MAX_RESPONSE_BYTES).await; + let result = env + .fetch_tx((), provider, txid_2, INITIAL_MAX_RESPONSE_BYTES) + .await; assert!(matches!( result, Ok(FetchResult::Error(HttpGetTxError::TxEncoding(_))) )); assert!(matches!( state::get_fetch_status(txid_2), - Some(FetchTxStatus::Error(HttpGetTxError::TxEncoding(_))) + Some(FetchTxStatus::Error(FetchTxStatusError { + error: HttpGetTxError::TxEncoding(_), + .. + })) )); } #[tokio::test] async fn test_check_fetched() { let mut env = MockEnv::new(CHECK_TRANSACTION_CYCLES_REQUIRED); - state::set_config(state::Config { - btc_network: BtcNetwork::Mainnet, - }); let good_address = Address::from_str("12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S") .unwrap() .assume_checked(); @@ -461,4 +481,31 @@ async fn test_check_fetched() { - get_tx_cycle_cost(INITIAL_MAX_RESPONSE_BYTES) * 2 - get_tx_cycle_cost(RETRY_MAX_RESPONSE_BYTES) ); + + // case HttpGetTxError can be retried. + let remaining_cycles = env.cycles_available(); + let provider = providers::next_provider(env.btc_network()); + state::set_fetch_status( + txid_2, + FetchTxStatus::Error(FetchTxStatusError { + provider, + max_response_bytes: RETRY_MAX_RESPONSE_BYTES, + error: HttpGetTxError::Rejected { + code: RejectionCode::SysTransient, + message: "no more reply".to_string(), + }, + }), + ); + env.expect_get_tx_with_reply(Ok(tx_2.clone())); + assert!(matches!( + env.check_fetched(txid_0, &fetched).await, + CheckTransactionResponse::Passed + )); + // check if provider has been rotated + assert!(*env.called_provider.borrow() == Some(provider.next())); + // Check remaining cycle. The cost should match RETRY_MAX_RESPONSE_BYTES + assert_eq!( + env.cycles_available(), + remaining_cycles - get_tx_cycle_cost(RETRY_MAX_RESPONSE_BYTES) + ); } diff --git a/rs/bitcoin/kyt/src/lib.rs b/rs/bitcoin/kyt/src/lib.rs index 4dfd405984e..dff59c88f85 100644 --- a/rs/bitcoin/kyt/src/lib.rs +++ b/rs/bitcoin/kyt/src/lib.rs @@ -54,26 +54,26 @@ impl FetchEnv for KytCanisterEnv { state::FetchGuard::new(txid) } + fn btc_network(&self) -> BtcNetwork { + get_config().btc_network + } + async fn http_get_tx( &self, + provider: providers::Provider, txid: Txid, max_response_bytes: u32, ) -> Result { - // TODO(XC-159): Support multiple providers - let request = providers::create_request(get_config().btc_network, txid, max_response_bytes); + let request = provider.create_request(txid, max_response_bytes); let url = request.url.clone(); let cycles = get_tx_cycle_cost(max_response_bytes); match http_request(request, cycles).await { Ok((response,)) => { // Ensure response is 200 before decoding if response.status != 200u32 { - let code = if response.status == 429u32 { - RejectionCode::SysTransient - } else { - RejectionCode::SysFatal - }; + // All non-200 status are treated as transient errors return Err(HttpGetTxError::Rejected { - code, + code: RejectionCode::SysTransient, message: format!("HTTP GET {} received code {}", url, response.status), }); } @@ -129,7 +129,6 @@ pub async fn check_transaction_inputs(txid: Txid) -> CheckTransactionResponse { match env.try_fetch_tx(txid) { TryFetchResult::Pending => CheckTransactionRetriable::Pending.into(), TryFetchResult::HighLoad => CheckTransactionRetriable::HighLoad.into(), - TryFetchResult::Error(err) => (txid, err).into(), TryFetchResult::NotEnoughCycles => CheckTransactionStatus::NotEnoughCycles.into(), TryFetchResult::Fetched(fetched) => env.check_fetched(txid, &fetched).await, TryFetchResult::ToFetch(do_fetch) => { diff --git a/rs/bitcoin/kyt/src/providers.rs b/rs/bitcoin/kyt/src/providers.rs index 1a17b53f147..4c2c3afa48b 100644 --- a/rs/bitcoin/kyt/src/providers.rs +++ b/rs/bitcoin/kyt/src/providers.rs @@ -3,15 +3,92 @@ use ic_btc_interface::Txid; use ic_cdk::api::management_canister::http_request::{ CanisterHttpRequestArgument, HttpHeader, HttpMethod, TransformContext, TransformFunc, }; +use std::cell::RefCell; +use std::fmt; -pub fn create_request( +/// Return the next bitcoin API provider for the given `btc_network`. +/// +/// Internally it remembers the previously used provider in a thread local +/// state and would iterate through all providers in a round-robin manner. +pub fn next_provider(btc_network: BtcNetwork) -> Provider { + PREVIOUS_PROVIDER_ID.with(|previous| { + let provider = (Provider { + btc_network, + provider_id: *previous.borrow(), + }) + .next(); + *previous.borrow_mut() = provider.provider_id; + provider + }) +} + +thread_local! { + static PREVIOUS_PROVIDER_ID: RefCell = const { RefCell::new(ProviderId::Btcscan) }; +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum ProviderId { + Btcscan, + Blockstream, + MempoolSpace, +} + +impl fmt::Display for ProviderId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Btcscan => write!(f, "btcscan.org"), + Self::Blockstream => write!(f, "blockstream.info"), + Self::MempoolSpace => write!(f, "mempool.space"), + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct Provider { btc_network: BtcNetwork, - txid: Txid, - max_response_bytes: u32, -) -> CanisterHttpRequestArgument { - match btc_network { - BtcNetwork::Mainnet => btcscan_request(txid, max_response_bytes), - BtcNetwork::Testnet => mempool_space_testnet_request(txid, max_response_bytes), + provider_id: ProviderId, +} + +impl Provider { + // Return the next provider by cycling through all available providers. + pub fn next(&self) -> Self { + let btc_network = self.btc_network; + let provider_id = match (self.btc_network, self.provider_id) { + (BtcNetwork::Mainnet, ProviderId::Btcscan) => ProviderId::Blockstream, + (BtcNetwork::Mainnet, ProviderId::Blockstream) => ProviderId::MempoolSpace, + (BtcNetwork::Mainnet, ProviderId::MempoolSpace) => ProviderId::Btcscan, + (BtcNetwork::Testnet, ProviderId::Blockstream) => ProviderId::MempoolSpace, + (BtcNetwork::Testnet, _) => ProviderId::Blockstream, + }; + Self { + btc_network, + provider_id, + } + } + + pub fn create_request( + &self, + txid: Txid, + max_response_bytes: u32, + ) -> CanisterHttpRequestArgument { + match (self.provider_id, self.btc_network) { + (ProviderId::Blockstream, _) => make_request( + "blockstream.info", + self.btc_network, + txid, + max_response_bytes, + ), + (ProviderId::MempoolSpace, _) => { + make_request("mempool.space", self.btc_network, txid, max_response_bytes) + } + (ProviderId::Btcscan, BtcNetwork::Mainnet) => btcscan_request(txid, max_response_bytes), + (provider, btc_network) => { + panic!( + "Provider {} does not support bitcoin {}", + provider, btc_network + ) + } + } } } @@ -38,12 +115,16 @@ fn btcscan_request(txid: Txid, max_response_bytes: u32) -> CanisterHttpRequestAr } } -fn mempool_space_testnet_request( +fn make_request( + host: &str, + network: BtcNetwork, txid: Txid, max_response_bytes: u32, ) -> CanisterHttpRequestArgument { - let host = "mempool.space"; - let url = format!("https://{}/testnet/api/tx/{}/raw", host, txid); + let url = match network { + BtcNetwork::Mainnet => format!("https://{}/api/tx/{}/raw", host, txid), + BtcNetwork::Testnet => format!("https://{}/testnet/api/tx/{}/raw", host, txid), + }; let request_headers = vec![HttpHeader { name: "Host".to_string(), value: format!("{host}:443"), diff --git a/rs/bitcoin/kyt/src/state.rs b/rs/bitcoin/kyt/src/state.rs index 2494083fb09..8d0b3b1fd75 100644 --- a/rs/bitcoin/kyt/src/state.rs +++ b/rs/bitcoin/kyt/src/state.rs @@ -1,4 +1,4 @@ -use crate::types::BtcNetwork; +use crate::{providers::Provider, types::BtcNetwork}; use bitcoin::{Address, Transaction}; use ic_btc_interface::Txid; use ic_cdk::api::call::RejectionCode; @@ -35,10 +35,17 @@ pub enum HttpGetTxError { pub enum FetchTxStatus { PendingOutcall, PendingRetry { max_response_bytes: u32 }, - Error(HttpGetTxError), + Error(FetchTxStatusError), Fetched(FetchedTx), } +#[derive(Debug, Clone)] +pub struct FetchTxStatusError { + pub provider: Provider, + pub max_response_bytes: u32, + pub error: HttpGetTxError, +} + /// Once the transaction data is successfully fetched, we create /// a list of `input_addresses` (matching the number of inputs) /// that is initialized as `None`. Once a corresponding input diff --git a/rs/bitcoin/kyt/tests/tests.rs b/rs/bitcoin/kyt/tests/tests.rs index 4b837c203bb..ddda38cda85 100644 --- a/rs/bitcoin/kyt/tests/tests.rs +++ b/rs/bitcoin/kyt/tests/tests.rs @@ -348,11 +348,92 @@ fn test_check_transaction_error() { .env .await_call(call_id) .expect("the fetch request didn't finish"); + // 500 error is retriable assert!(matches!( - dbg!(decode::(&result)), + decode::(&result), CheckTransactionResponse::Unknown(CheckTransactionStatus::Retriable( - CheckTransactionRetriable::TransientInternalError(_) - )) + CheckTransactionRetriable::TransientInternalError(msg) + )) if msg.contains("received code 500") + )); + let cycles_after = setup.env.cycle_balance(setup.caller); + let expected_cost = + CHECK_TRANSACTION_CYCLES_SERVICE_FEE + get_tx_cycle_cost(INITIAL_MAX_RESPONSE_BYTES); + let actual_cost = cycles_before - cycles_after; + assert!(actual_cost > expected_cost); + assert!(actual_cost - expected_cost < UNIVERSAL_CANISTER_CYCLE_MARGIN); + + // Test for 404 error + let cycles_before = setup.env.cycle_balance(setup.caller); + let call_id = setup + .submit_kyt_call( + "check_transaction", + Encode!(&CheckTransactionArgs { txid: txid.clone() }).unwrap(), + CHECK_TRANSACTION_CYCLES_REQUIRED, + ) + .expect("submit_call failed to return call id"); + let canister_http_requests = tick_until_next_request(&setup.env); + setup + .env + .mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: canister_http_requests[0].subnet_id, + request_id: canister_http_requests[0].request_id, + response: CanisterHttpResponse::CanisterHttpReply(CanisterHttpReply { + status: 404, + headers: vec![], + body: vec![], + }), + additional_responses: vec![], + }); + let result = setup + .env + .await_call(call_id) + .expect("the fetch request didn't finish"); + // 404 error is retriable too + assert!(matches!( + decode::(&result), + CheckTransactionResponse::Unknown(CheckTransactionStatus::Retriable( + CheckTransactionRetriable::TransientInternalError(msg) + )) if msg.contains("received code 404") + )); + let cycles_after = setup.env.cycle_balance(setup.caller); + let expected_cost = + CHECK_TRANSACTION_CYCLES_SERVICE_FEE + get_tx_cycle_cost(INITIAL_MAX_RESPONSE_BYTES); + let actual_cost = cycles_before - cycles_after; + assert!(actual_cost > expected_cost); + assert!(actual_cost - expected_cost < UNIVERSAL_CANISTER_CYCLE_MARGIN); + + // Test for malformatted transaction data + let cycles_before = setup.env.cycle_balance(setup.caller); + let call_id = setup + .submit_kyt_call( + "check_transaction", + Encode!(&CheckTransactionArgs { txid: txid.clone() }).unwrap(), + CHECK_TRANSACTION_CYCLES_REQUIRED, + ) + .expect("submit_call failed to return call id"); + let canister_http_requests = tick_until_next_request(&setup.env); + setup + .env + .mock_canister_http_response(MockCanisterHttpResponse { + subnet_id: canister_http_requests[0].subnet_id, + request_id: canister_http_requests[0].request_id, + response: CanisterHttpResponse::CanisterHttpReply(CanisterHttpReply { + status: 200, + headers: vec![], + body: vec![2, 0, 0, 0], + }), + additional_responses: vec![], + }); + let result = setup + .env + .await_call(call_id) + .expect("the fetch request didn't finish"); + // malformated tx error is retriable + assert!(matches!( + decode::(&result), + CheckTransactionResponse::Unknown(CheckTransactionStatus::Retriable( + CheckTransactionRetriable::TransientInternalError(msg) + )) if msg.contains("TxEncoding") )); let cycles_after = setup.env.cycle_balance(setup.caller); let expected_cost =