From 96ee2ed467ab8dd582e4cc8f4ffcd0de4880cfab Mon Sep 17 00:00:00 2001
From: Ethan Oroshiba <ethan@astria.org>
Date: Thu, 19 Dec 2024 09:00:28 -0600
Subject: [PATCH] chore(sequencer): add instrumentation (#1761)

## Summary
Added instrumentation to most async functions which did not have it.

## Background
Adding instrumentation to all async calls will aid in tracing since
spans will be emitted even if no events happen under them.

## Changes
- Added instrumentation to most async functions which did not have it.
- Added `err` argument to the `instrument` macro where applicable.
- Added fields to the `instrument` macro where applicable.

## Testing
Passing all tests.

## Changelogs
No updates required

## Related Issues
closes #1321
---
 .../src/accounts/component.rs                 |  2 +-
 crates/astria-sequencer/src/accounts/query.rs | 12 +++++-
 .../src/accounts/state_ext.rs                 | 17 ++++----
 .../src/action_handler/impls/bridge_lock.rs   |  5 +++
 .../impls/bridge_sudo_change.rs               |  5 +++
 .../src/action_handler/impls/bridge_unlock.rs |  6 +++
 .../action_handler/impls/fee_asset_change.rs  |  5 +++
 .../src/action_handler/impls/fee_change.rs    |  5 +++
 .../impls/ibc_relayer_change.rs               |  5 +++
 .../action_handler/impls/ibc_sudo_change.rs   |  5 +++
 .../action_handler/impls/ics20_withdrawal.rs  |  6 +++
 .../impls/init_bridge_account.rs              |  5 +++
 .../impls/rollup_data_submission.rs           |  5 +++
 .../impls/sudo_address_change.rs              |  5 +++
 .../src/action_handler/impls/transaction.rs   |  7 +++-
 .../src/action_handler/impls/transfer.rs      |  5 +++
 .../action_handler/impls/validator_update.rs  |  5 +++
 .../astria-sequencer/src/address/state_ext.rs | 16 ++++++--
 crates/astria-sequencer/src/app/mod.rs        | 26 ++++++------
 crates/astria-sequencer/src/app/state_ext.rs  | 15 ++++---
 crates/astria-sequencer/src/assets/query.rs   |  2 +
 .../astria-sequencer/src/assets/state_ext.rs  | 11 +++--
 .../src/authority/component.rs                | 11 +++--
 .../src/authority/state_ext.rs                | 11 +++--
 crates/astria-sequencer/src/bridge/query.rs   |  6 ++-
 .../astria-sequencer/src/bridge/state_ext.rs  | 24 +++++++----
 crates/astria-sequencer/src/fees/component.rs |  2 +-
 crates/astria-sequencer/src/fees/mod.rs       |  2 +-
 crates/astria-sequencer/src/fees/query.rs     |  5 ++-
 crates/astria-sequencer/src/fees/state_ext.rs | 14 ++++---
 crates/astria-sequencer/src/grpc/state_ext.rs | 21 ++++++----
 crates/astria-sequencer/src/ibc/component.rs  | 11 +++--
 .../src/ibc/host_interface.rs                 |  5 +++
 .../src/ibc/ics20_transfer.rs                 | 31 ++++++++++----
 crates/astria-sequencer/src/ibc/state_ext.rs  | 11 ++---
 .../src/mempool/mempool_state.rs              |  7 +++-
 crates/astria-sequencer/src/mempool/mod.rs    | 11 +++--
 .../src/mempool/transactions_container.rs     |  8 +++-
 crates/astria-sequencer/src/sequencer.rs      | 41 +++++++++++--------
 .../astria-sequencer/src/service/consensus.rs | 10 +++--
 .../src/service/mempool/mod.rs                |  4 ++
 .../src/transaction/checks.rs                 | 11 +++--
 42 files changed, 299 insertions(+), 122 deletions(-)

diff --git a/crates/astria-sequencer/src/accounts/component.rs b/crates/astria-sequencer/src/accounts/component.rs
index c9d515ec45..b8869d4a7c 100644
--- a/crates/astria-sequencer/src/accounts/component.rs
+++ b/crates/astria-sequencer/src/accounts/component.rs
@@ -25,7 +25,7 @@ pub(crate) struct AccountsComponent;
 impl Component for AccountsComponent {
     type AppState = GenesisAppState;
 
-    #[instrument(name = "AccountsComponent::init_chain", skip_all)]
+    #[instrument(name = "AccountsComponent::init_chain", skip_all, err)]
     async fn init_chain<S>(mut state: S, app_state: &Self::AppState) -> Result<()>
     where
         S: accounts::StateWriteExt + assets::StateReadExt,
diff --git a/crates/astria-sequencer/src/accounts/query.rs b/crates/astria-sequencer/src/accounts/query.rs
index 00c23a4095..af7d320228 100644
--- a/crates/astria-sequencer/src/accounts/query.rs
+++ b/crates/astria-sequencer/src/accounts/query.rs
@@ -28,7 +28,10 @@ use tendermint::{
     },
     block::Height,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     accounts::StateReadExt as _,
@@ -36,6 +39,7 @@ use crate::{
     assets::StateReadExt as _,
 };
 
+#[instrument(skip_all, fields(%asset), err(level = Level::DEBUG))]
 async fn ibc_to_trace<S: StateRead>(
     state: S,
     asset: &asset::IbcPrefixed,
@@ -47,7 +51,7 @@ async fn ibc_to_trace<S: StateRead>(
         .ok_or_eyre("asset not found when user has balance of it; this is a bug")
 }
 
-#[instrument(skip_all, fields(%address))]
+#[instrument(skip_all, fields(%address), err(level = Level::DEBUG))]
 async fn get_trace_prefixed_account_balances<S: StateRead>(
     state: &S,
     address: &Address,
@@ -70,6 +74,7 @@ async fn get_trace_prefixed_account_balances<S: StateRead>(
 
 /// Returns a list of [`AssetBalance`]s for the provided address. `AssetBalance`s are sorted
 /// alphabetically by [`asset::Denom`].
+#[instrument(skip_all)]
 pub(crate) async fn balance_request(
     storage: Storage,
     request: request::Query,
@@ -112,6 +117,7 @@ pub(crate) async fn balance_request(
     }
 }
 
+#[instrument(skip_all)]
 pub(crate) async fn nonce_request(
     storage: Storage,
     request: request::Query,
@@ -150,6 +156,7 @@ pub(crate) async fn nonce_request(
     }
 }
 
+#[instrument(skip_all, fields(%height), err(level = Level::DEBUG))]
 async fn get_snapshot_and_height(storage: &Storage, height: Height) -> Result<(Snapshot, Height)> {
     let snapshot = match height.value() {
         0 => storage.latest_snapshot(),
@@ -173,6 +180,7 @@ async fn get_snapshot_and_height(storage: &Storage, height: Height) -> Result<(S
     Ok((snapshot, height))
 }
 
+#[instrument(skip_all)]
 async fn preprocess_request(
     storage: &Storage,
     request: &request::Query,
diff --git a/crates/astria-sequencer/src/accounts/state_ext.rs b/crates/astria-sequencer/src/accounts/state_ext.rs
index ff466469db..2cae372809 100644
--- a/crates/astria-sequencer/src/accounts/state_ext.rs
+++ b/crates/astria-sequencer/src/accounts/state_ext.rs
@@ -25,7 +25,10 @@ use cnidarium::{
 };
 use futures::Stream;
 use pin_project_lite::pin_project;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::storage::{
     self,
@@ -141,7 +144,7 @@ pub(crate) trait StateReadExt: StateRead + crate::assets::StateReadExt {
         }
     }
 
-    #[instrument(skip_all, fields(address = %address.display_address(), %asset), err)]
+    #[instrument(skip_all, fields(address = %address.display_address(), %asset), err(level = Level::WARN))]
     async fn get_account_balance<'a, TAddress, TAsset>(
         &self,
         address: &TAddress,
@@ -165,7 +168,7 @@ pub(crate) trait StateReadExt: StateRead + crate::assets::StateReadExt {
             .wrap_err("invalid balance bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err)]
     async fn get_account_nonce<T: AddressBytes>(&self, address: &T) -> Result<u32> {
         let bytes = self
             .get_raw(&keys::nonce(address))
@@ -186,7 +189,7 @@ impl<T: StateRead + ?Sized> StateReadExt for T {}
 
 #[async_trait]
 pub(crate) trait StateWriteExt: StateWrite {
-    #[instrument(skip_all, fields(address = %address.display_address(), %asset, balance), err)]
+    #[instrument(skip_all, fields(address = %address.display_address(), %asset, balance), err(level = Level::WARN))]
     fn put_account_balance<'a, TAddress, TAsset>(
         &mut self,
         address: &TAddress,
@@ -205,7 +208,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         Ok(())
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(address = %address.display_address(), nonce), err(level = Level::WARN))]
     fn put_account_nonce<T: AddressBytes>(&mut self, address: &T, nonce: u32) -> Result<()> {
         let bytes = StoredValue::from(storage::Nonce::from(nonce))
             .serialize()
@@ -214,7 +217,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         Ok(())
     }
 
-    #[instrument(skip_all, fields(address = %address.display_address(), %asset, amount), err)]
+    #[instrument(skip_all, fields(address = %address.display_address(), %asset, amount), err(level = Level::WARN))]
     async fn increase_balance<'a, TAddress, TAsset>(
         &mut self,
         address: &TAddress,
@@ -241,7 +244,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         Ok(())
     }
 
-    #[instrument(skip_all, fields(address = %address.display_address(), %asset, amount))]
+    #[instrument(skip_all, fields(address = %address.display_address(), %asset, amount), err(level = Level::DEBUG))]
     async fn decrease_balance<'a, TAddress, TAsset>(
         &mut self,
         address: &TAddress,
diff --git a/crates/astria-sequencer/src/action_handler/impls/bridge_lock.rs b/crates/astria-sequencer/src/action_handler/impls/bridge_lock.rs
index 3be717fc61..38c61520c6 100644
--- a/crates/astria-sequencer/src/action_handler/impls/bridge_lock.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/bridge_lock.rs
@@ -13,6 +13,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::{
@@ -36,6 +40,7 @@ impl ActionHandler for BridgeLock {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/bridge_sudo_change.rs b/crates/astria-sequencer/src/action_handler/impls/bridge_sudo_change.rs
index a69775cf5f..97f7dbaa57 100644
--- a/crates/astria-sequencer/src/action_handler/impls/bridge_sudo_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/bridge_sudo_change.rs
@@ -7,6 +7,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -24,6 +28,7 @@ impl ActionHandler for BridgeSudoChange {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/bridge_unlock.rs b/crates/astria-sequencer/src/action_handler/impls/bridge_unlock.rs
index d2362a5742..3789fa5bde 100644
--- a/crates/astria-sequencer/src/action_handler/impls/bridge_unlock.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/bridge_unlock.rs
@@ -10,6 +10,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::{
@@ -28,6 +32,7 @@ use crate::{
 #[async_trait]
 impl ActionHandler for BridgeUnlock {
     // TODO(https://github.com/astriaorg/astria/issues/1430): move checks to the `BridgeUnlock` parsing.
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_stateless(&self) -> Result<()> {
         ensure!(self.amount > 0, "amount must be greater than zero",);
         ensure!(self.memo.len() <= 64, "memo must not be more than 64 bytes");
@@ -46,6 +51,7 @@ impl ActionHandler for BridgeUnlock {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/fee_asset_change.rs b/crates/astria-sequencer/src/action_handler/impls/fee_asset_change.rs
index c92161e751..fabbe3247c 100644
--- a/crates/astria-sequencer/src/action_handler/impls/fee_asset_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/fee_asset_change.rs
@@ -8,6 +8,10 @@ use async_trait::async_trait;
 use cnidarium::StateWrite;
 use futures::StreamExt as _;
 use tokio::pin;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -25,6 +29,7 @@ impl ActionHandler for FeeAssetChange {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> eyre::Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/fee_change.rs b/crates/astria-sequencer/src/action_handler/impls/fee_change.rs
index f066ded9ea..3b399ddc40 100644
--- a/crates/astria-sequencer/src/action_handler/impls/fee_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/fee_change.rs
@@ -6,6 +6,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -22,6 +26,7 @@ impl ActionHandler for FeeChange {
 
     /// check that the signer of the transaction is the current sudo address,
     /// as only that address can change the fee
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> eyre::Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/ibc_relayer_change.rs b/crates/astria-sequencer/src/action_handler/impls/ibc_relayer_change.rs
index e5de94afbe..9ddce3ea30 100644
--- a/crates/astria-sequencer/src/action_handler/impls/ibc_relayer_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/ibc_relayer_change.rs
@@ -6,6 +6,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -23,6 +27,7 @@ impl ActionHandler for IbcRelayerChange {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/ibc_sudo_change.rs b/crates/astria-sequencer/src/action_handler/impls/ibc_sudo_change.rs
index f82af648fc..487f71b4e6 100644
--- a/crates/astria-sequencer/src/action_handler/impls/ibc_sudo_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/ibc_sudo_change.rs
@@ -6,6 +6,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -21,6 +25,7 @@ impl ActionHandler for IbcSudoChange {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/ics20_withdrawal.rs b/crates/astria-sequencer/src/action_handler/impls/ics20_withdrawal.rs
index 34fecb3cf1..cf29705315 100644
--- a/crates/astria-sequencer/src/action_handler/impls/ics20_withdrawal.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/ics20_withdrawal.rs
@@ -37,6 +37,10 @@ use penumbra_ibc::component::packet::{
     Unchecked,
 };
 use penumbra_proto::core::component::ibc::v1::FungibleTokenPacketData;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     accounts::{
@@ -60,6 +64,7 @@ use crate::{
 #[async_trait]
 impl ActionHandler for action::Ics20Withdrawal {
     // TODO(https://github.com/astriaorg/astria/issues/1430): move checks to the `Ics20Withdrawal` parsing.
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_stateless(&self) -> Result<()> {
         ensure!(self.timeout_time() != 0, "timeout time must be non-zero",);
         ensure!(self.amount() > 0, "amount must be greater than zero",);
@@ -95,6 +100,7 @@ impl ActionHandler for action::Ics20Withdrawal {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/init_bridge_account.rs b/crates/astria-sequencer/src/action_handler/impls/init_bridge_account.rs
index 85e05de042..fdea8cab4f 100644
--- a/crates/astria-sequencer/src/action_handler/impls/init_bridge_account.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/init_bridge_account.rs
@@ -9,6 +9,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -26,6 +30,7 @@ impl ActionHandler for InitBridgeAccount {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/rollup_data_submission.rs b/crates/astria-sequencer/src/action_handler/impls/rollup_data_submission.rs
index 3881e221bf..23be358129 100644
--- a/crates/astria-sequencer/src/action_handler/impls/rollup_data_submission.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/rollup_data_submission.rs
@@ -5,11 +5,16 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::action_handler::ActionHandler;
 
 #[async_trait]
 impl ActionHandler for RollupDataSubmission {
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_stateless(&self) -> Result<()> {
         // TODO: do we want to place a maximum on the size of the data?
         // https://github.com/astriaorg/astria/issues/222
diff --git a/crates/astria-sequencer/src/action_handler/impls/sudo_address_change.rs b/crates/astria-sequencer/src/action_handler/impls/sudo_address_change.rs
index 7c7cb2e5e2..eb3f4f771a 100644
--- a/crates/astria-sequencer/src/action_handler/impls/sudo_address_change.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/sudo_address_change.rs
@@ -6,6 +6,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -25,6 +29,7 @@ impl ActionHandler for SudoAddressChange {
 
     /// check that the signer of the transaction is the current sudo address,
     /// as only that address can change the sudo address
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/transaction.rs b/crates/astria-sequencer/src/action_handler/impls/transaction.rs
index a5b66bf4fd..8605906ad4 100644
--- a/crates/astria-sequencer/src/action_handler/impls/transaction.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/transaction.rs
@@ -18,6 +18,10 @@ use astria_eyre::{
     },
 };
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     accounts::{
@@ -74,6 +78,7 @@ impl std::error::Error for InvalidNonce {}
 
 #[async_trait::async_trait]
 impl ActionHandler for Transaction {
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_stateless(&self) -> Result<()> {
         ensure!(!self.actions().is_empty(), "must have at least one action");
 
@@ -149,7 +154,7 @@ impl ActionHandler for Transaction {
     // FIXME (https://github.com/astriaorg/astria/issues/1584): because most lines come from delegating (and error wrapping) to the
     // individual actions. This could be tidied up by implementing `ActionHandler for Action`
     // and letting it delegate.
-    #[expect(clippy::too_many_lines, reason = "should be refactored")]
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         // Add the current signed transaction into the ephemeral state in case
         // downstream actions require access to it.
diff --git a/crates/astria-sequencer/src/action_handler/impls/transfer.rs b/crates/astria-sequencer/src/action_handler/impls/transfer.rs
index 27d955bca3..75735e8df4 100644
--- a/crates/astria-sequencer/src/action_handler/impls/transfer.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/transfer.rs
@@ -6,6 +6,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::{
@@ -23,6 +27,7 @@ impl ActionHandler for Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/action_handler/impls/validator_update.rs b/crates/astria-sequencer/src/action_handler/impls/validator_update.rs
index 4bd0d72c3c..5ad1ceab06 100644
--- a/crates/astria-sequencer/src/action_handler/impls/validator_update.rs
+++ b/crates/astria-sequencer/src/action_handler/impls/validator_update.rs
@@ -7,6 +7,10 @@ use astria_eyre::eyre::{
 };
 use async_trait::async_trait;
 use cnidarium::StateWrite;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     action_handler::ActionHandler,
@@ -23,6 +27,7 @@ impl ActionHandler for ValidatorUpdate {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_execute<S: StateWrite>(&self, mut state: S) -> Result<()> {
         let from = state
             .get_transaction_context()
diff --git a/crates/astria-sequencer/src/address/state_ext.rs b/crates/astria-sequencer/src/address/state_ext.rs
index 0f5cd5bf30..b217e80a56 100644
--- a/crates/astria-sequencer/src/address/state_ext.rs
+++ b/crates/astria-sequencer/src/address/state_ext.rs
@@ -16,16 +16,23 @@ use cnidarium::{
     StateRead,
     StateWrite,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::storage::{
     self,
     keys,
 };
-use crate::storage::StoredValue;
+use crate::{
+    accounts::AddressBytes as _,
+    storage::StoredValue,
+};
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
+    #[instrument(skip_all, fields(address = %address.display_address()), err(level = Level::DEBUG))]
     async fn ensure_base_prefix(&self, address: &Address<Bech32m>) -> Result<()> {
         let prefix = self
             .get_base_prefix()
@@ -39,6 +46,7 @@ pub(crate) trait StateReadExt: StateRead {
         Ok(())
     }
 
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn try_base_prefixed(&self, slice: &[u8]) -> Result<Address> {
         let prefix = self
             .get_base_prefix()
@@ -51,7 +59,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("failed to construct address from byte slice and state-provided base prefix")
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_base_prefix(&self) -> Result<String> {
         let Some(bytes) = self
             .get_raw(keys::BASE_PREFIX)
@@ -66,7 +74,7 @@ pub(crate) trait StateReadExt: StateRead {
             .context("invalid base prefix bytes")
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_ibc_compat_prefix(&self) -> Result<String> {
         let Some(bytes) = self
             .get_raw(keys::IBC_COMPAT_PREFIX)
diff --git a/crates/astria-sequencer/src/app/mod.rs b/crates/astria-sequencer/src/app/mod.rs
index 78386c6f9f..794f5720c2 100644
--- a/crates/astria-sequencer/src/app/mod.rs
+++ b/crates/astria-sequencer/src/app/mod.rs
@@ -79,6 +79,7 @@ use tracing::{
     debug,
     info,
     instrument,
+    Level,
 };
 
 pub(crate) use self::state_ext::{
@@ -235,6 +236,7 @@ pub(crate) struct App {
 }
 
 impl App {
+    #[instrument(name = "App::new", skip_all, err)]
     pub(crate) async fn new(
         snapshot: Snapshot,
         mempool: Mempool,
@@ -268,7 +270,7 @@ impl App {
         })
     }
 
-    #[instrument(name = "App:init_chain", skip_all)]
+    #[instrument(name = "App:init_chain", skip_all, err)]
     pub(crate) async fn init_chain(
         &mut self,
         storage: Storage,
@@ -355,7 +357,7 @@ impl App {
     /// It puts this special "commitment" as the first transaction in a block.
     /// When other validators receive the block, they know the first transaction is
     /// supposed to be the commitment, and verifies that is it correct.
-    #[instrument(name = "App::prepare_proposal", skip_all)]
+    #[instrument(name = "App::prepare_proposal", skip_all, err(level = Level::WARN))]
     pub(crate) async fn prepare_proposal(
         &mut self,
         prepare_proposal: abci::request::PrepareProposal,
@@ -405,7 +407,7 @@ impl App {
     /// Generates a commitment to the `sequence::Actions` in the block's transactions
     /// and ensures it matches the commitment created by the proposer, which
     /// should be the first transaction in the block.
-    #[instrument(name = "App::process_proposal", skip_all)]
+    #[instrument(name = "App::process_proposal", skip_all, err(level = Level::WARN))]
     pub(crate) async fn process_proposal(
         &mut self,
         process_proposal: abci::request::ProcessProposal,
@@ -565,7 +567,7 @@ impl App {
     ///
     /// As a result, all transactions in a sequencer block are guaranteed to execute
     /// successfully.
-    #[instrument(name = "App::execute_transactions_prepare_proposal", skip_all)]
+    #[instrument(name = "App::execute_transactions_prepare_proposal", skip_all, err(level = Level::DEBUG))]
     async fn execute_transactions_prepare_proposal(
         &mut self,
         block_size_constraints: &mut BlockSizeConstraints,
@@ -745,7 +747,7 @@ impl App {
     ///
     /// As a result, all transactions in a sequencer block are guaranteed to execute
     /// successfully.
-    #[instrument(name = "App::execute_transactions_process_proposal", skip_all)]
+    #[instrument(name = "App::execute_transactions_process_proposal", skip_all, err(level = Level::DEBUG))]
     async fn execute_transactions_process_proposal(
         &mut self,
         txs: Vec<Transaction>,
@@ -823,7 +825,7 @@ impl App {
     ///
     /// this *must* be called anytime before a block's txs are executed, whether it's
     /// during the proposal phase, or finalize_block phase.
-    #[instrument(name = "App::pre_execute_transactions", skip_all, err)]
+    #[instrument(name = "App::pre_execute_transactions", skip_all, err(level = Level::WARN))]
     async fn pre_execute_transactions(&mut self, block_data: BlockData) -> Result<()> {
         let chain_id = self
             .state
@@ -878,7 +880,7 @@ impl App {
     /// `SequencerBlock`.
     ///
     /// this must be called after a block's transactions are executed.
-    #[instrument(name = "App::post_execute_transactions", skip_all)]
+    #[instrument(name = "App::post_execute_transactions", skip_all, err(level = Level::WARN))]
     async fn post_execute_transactions(
         &mut self,
         block_hash: Hash,
@@ -962,7 +964,7 @@ impl App {
     ///
     /// This is called by cometbft after the block has already been
     /// committed by the network's consensus.
-    #[instrument(name = "App::finalize_block", skip_all)]
+    #[instrument(name = "App::finalize_block", skip_all, err)]
     pub(crate) async fn finalize_block(
         &mut self,
         finalize_block: abci::request::FinalizeBlock,
@@ -1076,7 +1078,7 @@ impl App {
         Ok(finalize_block)
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn prepare_commit(&mut self, storage: Storage) -> Result<AppHash> {
         // extract the state we've built up to so we can prepare it as a `StagedWriteBatch`.
         let dummy_state = StateDelta::new(storage.latest_snapshot());
@@ -1113,7 +1115,7 @@ impl App {
         Ok(app_hash)
     }
 
-    #[instrument(name = "App::begin_block", skip_all)]
+    #[instrument(name = "App::begin_block", skip_all, err(level = Level::WARN))]
     async fn begin_block(
         &mut self,
         begin_block: &abci::request::BeginBlock,
@@ -1149,7 +1151,7 @@ impl App {
     }
 
     /// Executes a signed transaction.
-    #[instrument(name = "App::execute_transaction", skip_all)]
+    #[instrument(name = "App::execute_transaction", skip_all, err(level = Level::DEBUG))]
     async fn execute_transaction(&mut self, signed_tx: Arc<Transaction>) -> Result<Vec<Event>> {
         signed_tx
             .check_stateless()
@@ -1186,7 +1188,7 @@ impl App {
         Ok(events)
     }
 
-    #[instrument(name = "App::end_block", skip_all)]
+    #[instrument(name = "App::end_block", skip_all, err(level = Level::WARN))]
     async fn end_block(
         &mut self,
         height: u64,
diff --git a/crates/astria-sequencer/src/app/state_ext.rs b/crates/astria-sequencer/src/app/state_ext.rs
index b9a2e3b78e..d1dfb5e07a 100644
--- a/crates/astria-sequencer/src/app/state_ext.rs
+++ b/crates/astria-sequencer/src/app/state_ext.rs
@@ -12,7 +12,10 @@ use cnidarium::{
     StateWrite,
 };
 use tendermint::Time;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::storage::{
     self,
@@ -22,7 +25,7 @@ use crate::storage::StoredValue;
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_chain_id(&self) -> Result<tendermint::chain::Id> {
         let Some(bytes) = self
             .get_raw(keys::CHAIN_ID)
@@ -37,7 +40,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid chain id bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_revision_number(&self) -> Result<u64> {
         let Some(bytes) = self
             .get_raw(keys::REVISION_NUMBER)
@@ -52,7 +55,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid revision number bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_block_height(&self) -> Result<u64> {
         let Some(bytes) = self
             .get_raw(keys::BLOCK_HEIGHT)
@@ -67,7 +70,7 @@ pub(crate) trait StateReadExt: StateRead {
             .context("invalid block height bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_block_timestamp(&self) -> Result<Time> {
         let Some(bytes) = self
             .get_raw(keys::BLOCK_TIMESTAMP)
@@ -82,7 +85,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid block timestamp bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(%height), err(level = Level::WARN))]
     async fn get_storage_version_by_height(&self, height: u64) -> Result<u64> {
         use astria_eyre::eyre::WrapErr as _;
 
diff --git a/crates/astria-sequencer/src/assets/query.rs b/crates/astria-sequencer/src/assets/query.rs
index ee3dad59d2..eacdd47f7c 100644
--- a/crates/astria-sequencer/src/assets/query.rs
+++ b/crates/astria-sequencer/src/assets/query.rs
@@ -11,6 +11,7 @@ use tendermint::abci::{
     response,
     Code,
 };
+use tracing::instrument;
 
 use crate::{
     app::StateReadExt as _,
@@ -21,6 +22,7 @@ use crate::{
 //
 // Example:
 // `abci-cli query --path=asset/denom/<DENOM_ID>`
+#[instrument(skip_all)]
 pub(crate) async fn denom_request(
     storage: Storage,
     request: request::Query,
diff --git a/crates/astria-sequencer/src/assets/state_ext.rs b/crates/astria-sequencer/src/assets/state_ext.rs
index b2ca7d0ca0..90ea50c65c 100644
--- a/crates/astria-sequencer/src/assets/state_ext.rs
+++ b/crates/astria-sequencer/src/assets/state_ext.rs
@@ -13,7 +13,10 @@ use cnidarium::{
     StateRead,
     StateWrite,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::storage::{
     self,
@@ -25,7 +28,7 @@ use crate::storage::StoredValue;
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_native_asset(&self) -> Result<Option<asset::TracePrefixed>> {
         let Some(bytes) = self
             .get_raw(keys::NATIVE_ASSET)
@@ -43,7 +46,7 @@ pub(crate) trait StateReadExt: StateRead {
             .map(Option::Some)
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn has_ibc_asset<'a, TAsset>(&self, asset: &'a TAsset) -> Result<bool>
     where
         TAsset: Sync,
@@ -57,7 +60,7 @@ pub(crate) trait StateReadExt: StateRead {
             .is_some())
     }
 
-    #[instrument(skip_all, fields(%asset), err)]
+    #[instrument(skip_all, fields(%asset), err(level = Level::WARN))]
     async fn map_ibc_to_trace_prefixed_asset(
         &self,
         asset: &asset::IbcPrefixed,
diff --git a/crates/astria-sequencer/src/authority/component.rs b/crates/astria-sequencer/src/authority/component.rs
index f6347dcf0c..aee3186ea1 100644
--- a/crates/astria-sequencer/src/authority/component.rs
+++ b/crates/astria-sequencer/src/authority/component.rs
@@ -13,7 +13,10 @@ use tendermint::abci::request::{
     BeginBlock,
     EndBlock,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::{
     StateReadExt,
@@ -35,7 +38,7 @@ pub(crate) struct AuthorityComponentAppState {
 impl Component for AuthorityComponent {
     type AppState = AuthorityComponentAppState;
 
-    #[instrument(name = "AuthorityComponent::init_chain", skip_all)]
+    #[instrument(name = "AuthorityComponent::init_chain", skip_all, err)]
     async fn init_chain<S: StateWriteExt>(mut state: S, app_state: &Self::AppState) -> Result<()> {
         // set sudo key and initial validator set
         state
@@ -48,7 +51,7 @@ impl Component for AuthorityComponent {
         Ok(())
     }
 
-    #[instrument(name = "AuthorityComponent::begin_block", skip_all)]
+    #[instrument(name = "AuthorityComponent::begin_block", skip_all, err(level = Level::WARN))]
     async fn begin_block<S: StateWriteExt + 'static>(
         state: &mut Arc<S>,
         begin_block: &BeginBlock,
@@ -70,7 +73,7 @@ impl Component for AuthorityComponent {
         Ok(())
     }
 
-    #[instrument(name = "AuthorityComponent::end_block", skip_all)]
+    #[instrument(name = "AuthorityComponent::end_block", skip_all, err(level = Level::WARN))]
     async fn end_block<S: StateWriteExt + StateReadExt + 'static>(
         state: &mut Arc<S>,
         _end_block: &EndBlock,
diff --git a/crates/astria-sequencer/src/authority/state_ext.rs b/crates/astria-sequencer/src/authority/state_ext.rs
index e55b81fcbf..6b565fbc0b 100644
--- a/crates/astria-sequencer/src/authority/state_ext.rs
+++ b/crates/astria-sequencer/src/authority/state_ext.rs
@@ -14,7 +14,10 @@ use cnidarium::{
     StateRead,
     StateWrite,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::{
     storage::{
@@ -30,7 +33,7 @@ use crate::{
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_sudo_address(&self) -> Result<[u8; ADDRESS_LEN]> {
         let Some(bytes) = self
             .get_raw(keys::SUDO)
@@ -46,7 +49,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid sudo key bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_validator_set(&self) -> Result<ValidatorSet> {
         let Some(bytes) = self
             .get_raw(keys::VALIDATOR_SET)
@@ -62,7 +65,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid validator set bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_validator_updates(&self) -> Result<ValidatorSet> {
         let Some(bytes) = self
             .nonverifiable_get_raw(keys::VALIDATOR_UPDATES.as_bytes())
diff --git a/crates/astria-sequencer/src/bridge/query.rs b/crates/astria-sequencer/src/bridge/query.rs
index ed5d1c5177..c4387a4756 100644
--- a/crates/astria-sequencer/src/bridge/query.rs
+++ b/crates/astria-sequencer/src/bridge/query.rs
@@ -16,8 +16,10 @@ use tendermint::abci::{
     response,
     Code,
 };
+use tracing::instrument;
 
 use crate::{
+    accounts::AddressBytes as _,
     address::StateReadExt,
     app::StateReadExt as _,
     assets::StateReadExt as _,
@@ -43,7 +45,7 @@ fn error_query_response(
 
 // FIXME (https://github.com/astriaorg/astria/issues/1582): there is a lot of code duplication due to `error_query_response`.
 // this could be significantly shortened.
-#[expect(clippy::too_many_lines, reason = "should be refactored")]
+#[instrument(skip_all, fields(address = %address.display_address()))]
 async fn get_bridge_account_info(
     snapshot: cnidarium::Snapshot,
     address: &Address,
@@ -162,6 +164,7 @@ async fn get_bridge_account_info(
     }))
 }
 
+#[instrument(skip_all)]
 pub(crate) async fn bridge_account_info_request(
     storage: Storage,
     request: request::Query,
@@ -210,6 +213,7 @@ pub(crate) async fn bridge_account_info_request(
     }
 }
 
+#[instrument(skip_all)]
 pub(crate) async fn bridge_account_last_tx_hash_request(
     storage: Storage,
     request: request::Query,
diff --git a/crates/astria-sequencer/src/bridge/state_ext.rs b/crates/astria-sequencer/src/bridge/state_ext.rs
index 68bd6c2925..e405d505fe 100644
--- a/crates/astria-sequencer/src/bridge/state_ext.rs
+++ b/crates/astria-sequencer/src/bridge/state_ext.rs
@@ -26,6 +26,7 @@ use cnidarium::{
 use tracing::{
     debug,
     instrument,
+    Level,
 };
 
 use super::storage::{
@@ -40,13 +41,13 @@ use crate::{
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(address = %address.display_address()))]
     async fn is_a_bridge_account<T: AddressBytes>(&self, address: &T) -> Result<bool> {
         let maybe_id = self.get_bridge_account_rollup_id(address).await?;
         Ok(maybe_id.is_some())
     }
 
-    #[instrument(skip_all, fields(address = %address.display_address()), err)]
+    #[instrument(skip_all, fields(address = %address.display_address()), err(level = Level::WARN))]
     async fn get_bridge_account_rollup_id<T: AddressBytes>(
         &self,
         address: &T,
@@ -68,7 +69,7 @@ pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
             .wrap_err("invalid rollup ID bytes")
     }
 
-    #[instrument(skip_all, fields(address = %address.display_address()), err)]
+    #[instrument(skip_all, fields(address = %address.display_address()), err(level = Level::WARN))]
     async fn get_bridge_account_ibc_asset<T: AddressBytes>(
         &self,
         address: &T,
@@ -86,7 +87,7 @@ pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
             .wrap_err("invalid bridge account asset ID bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(bridge_address = %bridge_address.display_address()), err(level = Level::WARN))]
     async fn get_bridge_account_sudo_address<T: AddressBytes>(
         &self,
         bridge_address: &T,
@@ -109,7 +110,7 @@ pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
             .wrap_err("invalid bridge account sudo address bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(bridge_address = %bridge_address.display_address()), err(level = Level::WARN))]
     async fn get_bridge_account_withdrawer_address<T: AddressBytes>(
         &self,
         bridge_address: &T,
@@ -138,7 +139,7 @@ pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
             .unwrap_or_default()
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(block_hash = %hex::encode(block_hash), %rollup_id), err(level = Level::WARN))]
     async fn get_deposits(
         &self,
         block_hash: &[u8; 32],
@@ -157,7 +158,7 @@ pub(crate) trait StateReadExt: StateRead + address::StateReadExt {
             .context("invalid deposits bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(address = %address.display_address()), err(level = Level::WARN))]
     async fn get_last_transaction_id_for_bridge_account<T: AddressBytes>(
         &self,
         address: &T,
@@ -249,7 +250,12 @@ pub(crate) trait StateWriteExt: StateWrite {
         Ok(())
     }
 
-    #[instrument(skip_all)]
+    #[instrument(
+        skip_all,
+        fields(address = %address.display_address(),
+        withdrawal_event_id, block_num),
+        err(level = Level::DEBUG)
+    )]
     async fn check_and_set_withdrawal_event_block_for_bridge_account<T: AddressBytes>(
         &mut self,
         address: &T,
@@ -293,7 +299,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         self.object_put(keys::DEPOSITS_EPHEMERAL, cached_deposits);
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     fn put_deposits(
         &mut self,
         block_hash: &[u8; 32],
diff --git a/crates/astria-sequencer/src/fees/component.rs b/crates/astria-sequencer/src/fees/component.rs
index e22e4480d7..d21298fbb8 100644
--- a/crates/astria-sequencer/src/fees/component.rs
+++ b/crates/astria-sequencer/src/fees/component.rs
@@ -23,7 +23,7 @@ pub(crate) struct FeesComponent;
 impl Component for FeesComponent {
     type AppState = GenesisAppState;
 
-    #[instrument(name = "FeesComponent::init_chain", skip_all)]
+    #[instrument(name = "FeesComponent::init_chain", skip_all, err)]
     async fn init_chain<S>(mut state: S, app_state: &Self::AppState) -> Result<()>
     where
         S: fees::StateWriteExt + fees::StateReadExt,
diff --git a/crates/astria-sequencer/src/fees/mod.rs b/crates/astria-sequencer/src/fees/mod.rs
index ba6557d522..85519f7cde 100644
--- a/crates/astria-sequencer/src/fees/mod.rs
+++ b/crates/astria-sequencer/src/fees/mod.rs
@@ -103,7 +103,7 @@ pub(crate) trait FeeHandler: Send {
     /// If this method returns `None`, the action is free.
     fn fee_asset(&self) -> Option<&asset::Denom>;
 
-    #[instrument(skip_all, err(level = Level::WARN))]
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn check_and_pay_fees<'a, S>(&self, mut state: S) -> eyre::Result<()>
     where
         S: StateWrite,
diff --git a/crates/astria-sequencer/src/fees/query.rs b/crates/astria-sequencer/src/fees/query.rs
index d4b28dc768..b551e50951 100644
--- a/crates/astria-sequencer/src/fees/query.rs
+++ b/crates/astria-sequencer/src/fees/query.rs
@@ -68,6 +68,7 @@ use tokio::{
 use tracing::{
     instrument,
     warn,
+    Level,
 };
 
 use crate::{
@@ -80,6 +81,7 @@ use crate::{
     storage::StoredValue,
 };
 
+#[instrument(skip_all, fields(%asset))]
 async fn find_trace_prefixed_or_return_ibc<S: StateRead>(
     state: S,
     asset: asset::IbcPrefixed,
@@ -110,6 +112,7 @@ async fn get_allowed_fee_assets<S: StateRead>(state: &S) -> Vec<Denom> {
     stream.collect::<Vec<_>>().await
 }
 
+#[instrument(skip_all)]
 pub(crate) async fn allowed_fee_assets_request(
     storage: Storage,
     request: request::Query,
@@ -275,7 +278,7 @@ pub(crate) async fn transaction_fee_request(
     }
 }
 
-#[instrument(skip_all)]
+#[instrument(skip_all, err(level = Level::DEBUG))]
 pub(crate) async fn get_fees_for_transaction<S: StateRead>(
     tx: &TransactionBody,
     state: &S,
diff --git a/crates/astria-sequencer/src/fees/state_ext.rs b/crates/astria-sequencer/src/fees/state_ext.rs
index d62d682f86..5ef3140e6e 100644
--- a/crates/astria-sequencer/src/fees/state_ext.rs
+++ b/crates/astria-sequencer/src/fees/state_ext.rs
@@ -28,7 +28,10 @@ use cnidarium::{
 use futures::Stream;
 use pin_project_lite::pin_project;
 use tendermint::abci::Event;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::{
     storage::keys::{
@@ -78,7 +81,7 @@ pub(crate) trait StateReadExt: StateRead {
         self.object_get(keys::BLOCK).unwrap_or_default()
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_fees<'a, F>(&self) -> Result<Option<FeeComponents<F>>>
     where
         F: FeeHandler + ?Sized,
@@ -102,7 +105,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid fees bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn is_allowed_fee_asset<'a, TAsset>(&self, asset: &'a TAsset) -> Result<bool>
     where
         TAsset: Sync,
@@ -128,6 +131,7 @@ impl<T: ?Sized + StateRead> StateReadExt for T {}
 
 #[async_trait]
 pub(crate) trait StateWriteExt: StateWrite {
+    // TODO(https://github.com/astriaorg/astria/issues/1845): This doesn't need to return a result
     /// Constructs and adds `Fee` object to the block fees vec.
     #[instrument(skip_all)]
     fn add_fee_to_block_fees<'a, TAsset, F: FeeHandler + ?Sized>(
@@ -162,7 +166,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         self.object_put(keys::BLOCK, new_fees);
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     fn put_fees<'a, F>(&mut self, fees: FeeComponents<F>) -> Result<()>
     where
         F: FeeHandler,
@@ -183,7 +187,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         self.delete(keys::allowed_asset(asset));
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     fn put_allowed_fee_asset<'a, TAsset>(&mut self, asset: &'a TAsset) -> Result<()>
     where
         &'a TAsset: Into<Cow<'a, asset::IbcPrefixed>>,
diff --git a/crates/astria-sequencer/src/grpc/state_ext.rs b/crates/astria-sequencer/src/grpc/state_ext.rs
index fc377d48c1..86aefcf1c7 100644
--- a/crates/astria-sequencer/src/grpc/state_ext.rs
+++ b/crates/astria-sequencer/src/grpc/state_ext.rs
@@ -20,7 +20,10 @@ use cnidarium::{
     StateRead,
     StateWrite,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use super::storage::{
     self,
@@ -30,7 +33,7 @@ use crate::storage::StoredValue;
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(%height), err(level = Level::WARN))]
     async fn get_block_hash_by_height(&self, height: u64) -> Result<[u8; 32]> {
         let Some(bytes) = self
             .nonverifiable_get_raw(keys::block_hash_by_height(height).as_bytes())
@@ -45,7 +48,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid block hash bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(hash = %hex::encode(hash)), err(level = Level::WARN))]
     async fn get_sequencer_block_header_by_hash(
         &self,
         hash: &[u8; 32],
@@ -65,7 +68,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid sequencer block header bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(hash = %hex::encode(hash)), err(level = Level::WARN))]
     async fn get_rollup_ids_by_block_hash(&self, hash: &[u8; 32]) -> Result<Vec<RollupId>> {
         let Some(bytes) = self
             .nonverifiable_get_raw(keys::rollup_ids_by_hash(hash).as_bytes())
@@ -80,7 +83,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid rollup ids bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(%height), err(level = Level::DEBUG))]
     async fn get_sequencer_block_by_height(&self, height: u64) -> Result<SequencerBlock> {
         let hash = self
             .get_block_hash_by_height(height)
@@ -91,7 +94,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("failed to get sequencer block by hash")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(hash = %hex::encode(hash), %rollup_id), err(level = Level::WARN))]
     async fn get_rollup_data(
         &self,
         hash: &[u8; 32],
@@ -116,7 +119,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid rollup transactions bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(hash = %hex::encode(hash)), err(level = Level::WARN))]
     async fn get_rollup_transactions_proof_by_block_hash(
         &self,
         hash: &[u8; 32],
@@ -134,7 +137,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid rollup transactions proof bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(hash = %hex::encode(hash)), err(level = Level::WARN))]
     async fn get_rollup_ids_proof_by_block_hash(&self, hash: &[u8; 32]) -> Result<merkle::Proof> {
         let Some(bytes) = self
             .nonverifiable_get_raw(keys::rollup_ids_proof_by_hash(hash).as_bytes())
@@ -180,7 +183,7 @@ pub(crate) trait StateWriteExt: StateWrite {
     }
 }
 
-#[instrument(skip_all)]
+#[instrument(skip_all, fields(hash = %hex::encode(hash)), err(level = Level::DEBUG))]
 async fn get_sequencer_block_by_hash<S: StateRead + ?Sized>(
     state: &S,
     hash: &[u8; 32],
diff --git a/crates/astria-sequencer/src/ibc/component.rs b/crates/astria-sequencer/src/ibc/component.rs
index f8bc969e39..8a17b30c61 100644
--- a/crates/astria-sequencer/src/ibc/component.rs
+++ b/crates/astria-sequencer/src/ibc/component.rs
@@ -13,7 +13,10 @@ use tendermint::abci::request::{
     BeginBlock,
     EndBlock,
 };
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     component::Component,
@@ -30,7 +33,7 @@ pub(crate) struct IbcComponent;
 impl Component for IbcComponent {
     type AppState = GenesisAppState;
 
-    #[instrument(name = "IbcComponent::init_chain", skip_all)]
+    #[instrument(name = "IbcComponent::init_chain", skip_all, err)]
     async fn init_chain<S: StateWriteExt>(mut state: S, app_state: &Self::AppState) -> Result<()> {
         Ibc::init_chain(
             &mut state,
@@ -53,7 +56,7 @@ impl Component for IbcComponent {
         Ok(())
     }
 
-    #[instrument(name = "IbcComponent::begin_block", skip_all)]
+    #[instrument(name = "IbcComponent::begin_block", skip_all, err(level = Level::WARN))]
     async fn begin_block<S: StateWriteExt + 'static>(
         state: &mut Arc<S>,
         begin_block: &BeginBlock,
@@ -62,7 +65,7 @@ impl Component for IbcComponent {
         Ok(())
     }
 
-    #[instrument(name = "IbcComponent::end_block", skip_all)]
+    #[instrument(name = "IbcComponent::end_block", skip_all, er(level = Level::WARN))]
     async fn end_block<S: StateWriteExt + 'static>(
         state: &mut Arc<S>,
         end_block: &EndBlock,
diff --git a/crates/astria-sequencer/src/ibc/host_interface.rs b/crates/astria-sequencer/src/ibc/host_interface.rs
index 33e76455b2..c00114c59d 100644
--- a/crates/astria-sequencer/src/ibc/host_interface.rs
+++ b/crates/astria-sequencer/src/ibc/host_interface.rs
@@ -4,6 +4,7 @@ use astria_eyre::{
 };
 use cnidarium::StateRead;
 use penumbra_ibc::component::HostInterface;
+use tracing::instrument;
 
 use crate::app::StateReadExt as _;
 
@@ -12,6 +13,7 @@ pub(crate) struct AstriaHost;
 
 #[async_trait::async_trait]
 impl HostInterface for AstriaHost {
+    #[instrument(skip_all)]
     async fn get_chain_id<S: StateRead>(state: S) -> anyhow::Result<String> {
         state
             .get_chain_id()
@@ -20,14 +22,17 @@ impl HostInterface for AstriaHost {
             .map(|s| s.to_string())
     }
 
+    #[instrument(skip_all)]
     async fn get_revision_number<S: StateRead>(state: S) -> anyhow::Result<u64> {
         state.get_revision_number().await.map_err(eyre_to_anyhow)
     }
 
+    #[instrument(skip_all)]
     async fn get_block_height<S: StateRead>(state: S) -> anyhow::Result<u64> {
         state.get_block_height().await.map_err(eyre_to_anyhow)
     }
 
+    #[instrument(skip_all)]
     async fn get_block_timestamp<S: StateRead>(state: S) -> anyhow::Result<tendermint::Time> {
         state.get_block_timestamp().await.map_err(eyre_to_anyhow)
     }
diff --git a/crates/astria-sequencer/src/ibc/ics20_transfer.rs b/crates/astria-sequencer/src/ibc/ics20_transfer.rs
index c94b87ed0f..c0bf154dc2 100644
--- a/crates/astria-sequencer/src/ibc/ics20_transfer.rs
+++ b/crates/astria-sequencer/src/ibc/ics20_transfer.rs
@@ -71,7 +71,10 @@ use penumbra_ibc::component::app_handler::{
 };
 use penumbra_proto::penumbra::core::component::ibc::v1::FungibleTokenPacketData;
 use tokio::try_join;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     accounts::StateWriteExt as _,
@@ -179,6 +182,7 @@ pub(crate) struct Ics20Transfer;
 
 #[async_trait::async_trait]
 impl AppHandlerCheck for Ics20Transfer {
+    #[instrument(skip_all)]
     async fn chan_open_init_check<S: StateRead>(
         _: S,
         msg: &MsgChannelOpenInit,
@@ -194,6 +198,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn chan_open_try_check<S: StateRead>(
         _: S,
         msg: &MsgChannelOpenTry,
@@ -209,6 +214,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn chan_open_ack_check<S: StateRead>(
         _: S,
         msg: &MsgChannelOpenAck,
@@ -220,6 +226,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn chan_open_confirm_check<S: StateRead>(
         _: S,
         _: &MsgChannelOpenConfirm,
@@ -229,6 +236,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn chan_close_init_check<S: StateRead>(
         _: S,
         _: &MsgChannelCloseInit,
@@ -236,6 +244,7 @@ impl AppHandlerCheck for Ics20Transfer {
         anyhow::bail!("ics20 always aborts on chan_close_init");
     }
 
+    #[instrument(skip_all)]
     async fn chan_close_confirm_check<S: StateRead>(
         _: S,
         _: &MsgChannelCloseConfirm,
@@ -244,6 +253,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn recv_packet_check<S: StateRead>(_: S, msg: &MsgRecvPacket) -> anyhow::Result<()> {
         // most checks performed in `execute`
         // perform stateless checks here
@@ -258,6 +268,7 @@ impl AppHandlerCheck for Ics20Transfer {
         Ok(())
     }
 
+    #[instrument(skip_all)]
     async fn timeout_packet_check<S: StateRead>(state: S, msg: &MsgTimeout) -> anyhow::Result<()> {
         refund_tokens_check(
             state,
@@ -269,6 +280,7 @@ impl AppHandlerCheck for Ics20Transfer {
         .map_err(eyre_to_anyhow)
     }
 
+    #[instrument(skip_all)]
     async fn acknowledge_packet_check<S: StateRead>(
         state: S,
         msg: &MsgAcknowledgement,
@@ -293,6 +305,7 @@ impl AppHandlerCheck for Ics20Transfer {
     }
 }
 
+#[instrument(skip_all, fields(%source_port, %source_channel), err(level = Level::DEBUG))]
 async fn refund_tokens_check<S: StateRead>(
     state: S,
     data: &[u8],
@@ -345,7 +358,7 @@ impl AppHandlerExecute for Ics20Transfer {
 
     async fn chan_close_init_execute<S: StateWrite>(_: S, _: &MsgChannelCloseInit) {}
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     async fn recv_packet_execute<S: StateWrite>(
         mut state: S,
         msg: &MsgRecvPacket,
@@ -371,7 +384,7 @@ impl AppHandlerExecute for Ics20Transfer {
             .context("failed to write acknowledgement")
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn timeout_packet_execute<S: StateWrite>(
         mut state: S,
         msg: &MsgTimeout,
@@ -381,7 +394,7 @@ impl AppHandlerExecute for Ics20Transfer {
         })
     }
 
-    #[instrument(skip_all, err)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn acknowledge_packet_execute<S: StateWrite>(
         mut state: S,
         msg: &MsgAcknowledgement,
@@ -573,7 +586,7 @@ fn does_failed_transfer_come_from_rollup(
     serde_json::from_str::<Ics20WithdrawalFromRollup>(&packet_data.memo).ok()
 }
 
-#[instrument(skip_all, fields(%recipient, %asset, amount), err)]
+#[instrument(skip_all, fields(%recipient, %asset, amount), err(level = Level::DEBUG))]
 async fn refund_tokens_to_sequencer_address<S: StateWrite>(
     mut state: S,
     recipient: &Address,
@@ -596,7 +609,7 @@ async fn refund_tokens_to_sequencer_address<S: StateWrite>(
     Ok(())
 }
 
-#[instrument(skip_all, fields(input), err)]
+#[instrument(skip_all, fields(input), err(level = Level::DEBUG))]
 async fn parse_asset<S: StateRead>(state: S, input: &str) -> Result<denom::TracePrefixed> {
     let asset = match input
         .parse::<Denom>()
@@ -614,7 +627,7 @@ async fn parse_asset<S: StateRead>(state: S, input: &str) -> Result<denom::Trace
     Ok(asset)
 }
 
-#[instrument(skip_all, fields(input), err)]
+#[instrument(skip_all, fields(input), err(level = Level::DEBUG))]
 async fn parse_address_on_sequencer<S: StateRead>(state: &S, input: &str) -> Result<Address> {
     use futures::TryFutureExt as _;
     let (base_prefix, compat_prefix) = match try_join!(
@@ -660,7 +673,7 @@ async fn parse_address_on_sequencer<S: StateRead>(state: &S, input: &str) -> Res
 
 /// Emits a deposit event signaling to the rollup that funds
 /// were added to `bridge_address`.
-#[instrument(skip_all, fields(%bridge_address, %asset, amount, memo), err)]
+#[instrument(skip_all, fields(%bridge_address, %asset, amount, memo), err(level = Level::DEBUG))]
 async fn emit_bridge_lock_deposit<S: StateWrite>(
     mut state: S,
     bridge_address: Address,
@@ -691,7 +704,7 @@ async fn emit_bridge_lock_deposit<S: StateWrite>(
     .await
 }
 
-#[instrument(skip_all, fields(%bridge_address, destination_chain_address, %asset, amount), err)]
+#[instrument(skip_all, fields(%bridge_address, destination_chain_address, %asset, amount), err(level = Level::DEBUG))]
 async fn emit_deposit<S: StateWrite>(
     mut state: S,
     bridge_address: &Address,
diff --git a/crates/astria-sequencer/src/ibc/state_ext.rs b/crates/astria-sequencer/src/ibc/state_ext.rs
index 3307e5b2e8..6a7de01af5 100644
--- a/crates/astria-sequencer/src/ibc/state_ext.rs
+++ b/crates/astria-sequencer/src/ibc/state_ext.rs
@@ -25,6 +25,7 @@ use ibc_types::core::channel::ChannelId;
 use tracing::{
     debug,
     instrument,
+    Level,
 };
 
 use super::storage::{
@@ -38,7 +39,7 @@ use crate::{
 
 #[async_trait]
 pub(crate) trait StateReadExt: StateRead {
-    #[instrument(skip_all, fields(%channel, %asset), err)]
+    #[instrument(skip_all, fields(%channel, %asset), err(level = Level::WARN))]
     async fn get_ibc_channel_balance<'a, TAsset>(
         &self,
         channel: &ChannelId,
@@ -62,7 +63,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid ibc channel balance bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn get_ibc_sudo_address(&self) -> Result<[u8; ADDRESS_LEN]> {
         let Some(bytes) = self
             .get_raw(keys::IBC_SUDO)
@@ -78,7 +79,7 @@ pub(crate) trait StateReadExt: StateRead {
             .wrap_err("invalid ibc sudo address bytes")
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn is_ibc_relayer<T: AddressBytes>(&self, address: T) -> Result<bool> {
         Ok(self
             .get_raw(&keys::ibc_relayer(&address))
@@ -93,7 +94,7 @@ impl<T: StateRead + ?Sized> StateReadExt for T {}
 
 #[async_trait]
 pub(crate) trait StateWriteExt: StateWrite {
-    #[instrument(skip_all, fields(%channel, %asset, balance), err)]
+    #[instrument(skip_all, fields(%channel, %asset, balance), err(level = Level::WARN))]
     fn put_ibc_channel_balance<'a, TAsset>(
         &mut self,
         channel: &ChannelId,
@@ -111,7 +112,7 @@ pub(crate) trait StateWriteExt: StateWrite {
         Ok(())
     }
 
-    #[instrument(skip_all, fields(%channel, %asset, amount), err)]
+    #[instrument(skip_all, fields(%channel, %asset, amount), err(level = Level::DEBUG))]
     async fn decrease_ibc_channel_balance<'a, TAsset>(
         &mut self,
         channel: &ChannelId,
diff --git a/crates/astria-sequencer/src/mempool/mempool_state.rs b/crates/astria-sequencer/src/mempool/mempool_state.rs
index e1676dcb02..69b5344089 100644
--- a/crates/astria-sequencer/src/mempool/mempool_state.rs
+++ b/crates/astria-sequencer/src/mempool/mempool_state.rs
@@ -3,14 +3,17 @@ use std::collections::HashMap;
 use astria_core::primitive::v1::asset;
 use astria_eyre::eyre::Result;
 use cnidarium::StateRead;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::accounts::{
     AddressBytes,
     StateReadExt as _,
 };
 
-#[instrument(skip_all)]
+#[instrument(skip_all, fields(address = %address.display_address()), err(level = Level::DEBUG))]
 pub(crate) async fn get_account_balances<S: StateRead, T: AddressBytes>(
     state: S,
     address: &T,
diff --git a/crates/astria-sequencer/src/mempool/mod.rs b/crates/astria-sequencer/src/mempool/mod.rs
index 68579620f4..e3391df888 100644
--- a/crates/astria-sequencer/src/mempool/mod.rs
+++ b/crates/astria-sequencer/src/mempool/mod.rs
@@ -29,6 +29,7 @@ use tokio::{
 use tracing::{
     error,
     instrument,
+    Level,
 };
 pub(crate) use transactions_container::InsertionError;
 use transactions_container::{
@@ -203,7 +204,7 @@ impl Mempool {
 
     /// Inserts a transaction into the mempool and does not allow for transaction replacement.
     /// Will return the reason for insertion failure if failure occurs.
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(tx_hash = %tx.id(), current_account_nonce), err(level = Level::DEBUG))]
     pub(crate) async fn insert(
         &self,
         tx: Arc<Transaction>,
@@ -291,6 +292,7 @@ impl Mempool {
     /// Returns a copy of all transactions and their hashes ready for execution, sorted first by the
     /// difference between a transaction and the account's current nonce and then by the time that
     /// the transaction was first seen by the appside mempool.
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     pub(crate) async fn builder_queue<S: accounts::StateReadExt>(
         &self,
         state: &S,
@@ -303,6 +305,7 @@ impl Mempool {
     ///
     /// This function should only be used to remove invalid/failing transactions and not executed
     /// transactions. Executed transactions will be removed in the `run_maintenance()` function.
+    #[instrument(skip_all, fields(tx_hash = %signed_tx.id()))]
     pub(crate) async fn remove_tx_invalid(
         &self,
         signed_tx: Arc<Transaction>,
@@ -342,13 +345,13 @@ impl Mempool {
 
     /// Checks if a transaction was flagged to be removed from the `CometBFT` mempool. Will
     /// remove the transaction from the cache if it is present.
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(tx_hash = %hex::encode(tx_hash)))]
     pub(crate) async fn check_removed_comet_bft(&self, tx_hash: [u8; 32]) -> Option<RemovalReason> {
         self.comet_bft_removal_cache.write().await.remove(tx_hash)
     }
 
     /// Returns true if the transaction is tracked as inserted.
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(tx_hash = %hex::encode(tx_hash)))]
     pub(crate) async fn is_tracked(&self, tx_hash: [u8; 32]) -> bool {
         self.contained_txs.read().await.contains(&tx_hash)
     }
@@ -482,7 +485,7 @@ impl Mempool {
     /// does not take into account gapped nonces in the parked queue. For example, if the
     /// pending queue for an account has nonces [0,1] and the parked queue has [3], [1] will be
     /// returned.
-    #[instrument(skip_all)]
+    #[instrument(skip_all, fields(address = %telemetry::display::base64(address)))]
     pub(crate) async fn pending_nonce(&self, address: &[u8; 20]) -> Option<u32> {
         self.pending.read().await.pending_nonce(address)
     }
diff --git a/crates/astria-sequencer/src/mempool/transactions_container.rs b/crates/astria-sequencer/src/mempool/transactions_container.rs
index 95d17b00c1..9dc2b5198c 100644
--- a/crates/astria-sequencer/src/mempool/transactions_container.rs
+++ b/crates/astria-sequencer/src/mempool/transactions_container.rs
@@ -26,7 +26,11 @@ use tokio::time::{
     Duration,
     Instant,
 };
-use tracing::error;
+use tracing::{
+    error,
+    instrument,
+    Level,
+};
 
 use super::RemovalReason;
 use crate::{
@@ -569,6 +573,7 @@ pub(super) trait TransactionsContainer<T: TransactionsForAccount> {
     /// Recosts transactions for an account.
     ///
     /// Logs an error if fails to recost a transaction.
+    #[instrument(skip_all, fields(address = %telemetry::display::base64(address)))]
     async fn recost_transactions<S: accounts::StateReadExt>(
         &mut self,
         address: &[u8; 20],
@@ -770,6 +775,7 @@ impl PendingTransactions {
 
     /// Returns a copy of transactions and their hashes sorted by nonce difference and then time
     /// first seen.
+    #[instrument(skip_all, err(level = Level::DEBUG))]
     pub(super) async fn builder_queue<S: accounts::StateReadExt>(
         &self,
         state: &S,
diff --git a/crates/astria-sequencer/src/sequencer.rs b/crates/astria-sequencer/src/sequencer.rs
index c5b3411d1c..92df49d470 100644
--- a/crates/astria-sequencer/src/sequencer.rs
+++ b/crates/astria-sequencer/src/sequencer.rs
@@ -29,8 +29,9 @@ use tokio::{
 use tower_abci::v038::Server;
 use tracing::{
     error,
+    error_span,
     info,
-    instrument,
+    info_span,
 };
 
 use crate::{
@@ -46,26 +47,31 @@ use crate::{
 pub struct Sequencer;
 
 impl Sequencer {
-    #[instrument(skip_all)]
+    #[expect(clippy::missing_errors_doc, reason = "not a public function")]
     pub async fn run_until_stopped(config: Config, metrics: &'static Metrics) -> Result<()> {
         cnidarium::register_metrics();
         register_histogram_global("cnidarium_get_raw_duration_seconds");
         register_histogram_global("cnidarium_nonverifiable_get_raw_duration_seconds");
+        let span = info_span!("Sequencer::run_until_stopped");
 
         if config
             .db_filepath
             .try_exists()
             .context("failed checking for existence of db storage file")?
         {
-            info!(
-                path = %config.db_filepath.display(),
-                "opening storage db"
-            );
+            span.in_scope(|| {
+                info!(
+                    path = %config.db_filepath.display(),
+                    "opening storage db"
+                );
+            });
         } else {
-            info!(
-                path = %config.db_filepath.display(),
-                "creating storage db"
-            );
+            span.in_scope(|| {
+                info!(
+                    path = %config.db_filepath.display(),
+                    "creating storage db"
+                );
+            });
         }
 
         let mut signals = spawn_signal_handler();
@@ -119,15 +125,16 @@ impl Sequencer {
             .wrap_err("failed to parse grpc_addr address")?;
         let grpc_server_handle = start_grpc_server(&storage, mempool, grpc_addr, shutdown_rx);
 
-        info!(config.listen_addr, "starting sequencer");
+        span.in_scope(|| info!(config.listen_addr, "starting sequencer"));
         let server_handle = tokio::spawn(async move {
             match server.listen_tcp(&config.listen_addr).await {
                 Ok(()) => {
                     // this shouldn't happen, as there isn't a way for the ABCI server to exit
-                    info!("ABCI server exited successfully");
+                    info_span!("abci_server").in_scope(|| info!("ABCI server exited successfully"));
                 }
                 Err(e) => {
-                    error!(err = e.as_ref(), "ABCI server exited with error");
+                    error_span!("abci_server")
+                        .in_scope(|| error!(err = e.as_ref(), "ABCI server exited with error"));
                 }
             }
             let _ = server_exit_tx.send(());
@@ -135,11 +142,11 @@ impl Sequencer {
 
         select! {
             _ = signals.stop_rx.changed() => {
-                info!("shutting down sequencer");
+                span.in_scope(|| info!("shutting down sequencer"));
             }
 
             _ = server_exit_rx => {
-                error!("ABCI server task exited, this shouldn't happen");
+                span.in_scope(|| error!("ABCI server task exited, this shouldn't happen"));
             }
         }
 
@@ -179,9 +186,9 @@ fn start_grpc_server(
         .trace_fn(|req| {
             if let Some(remote_addr) = remote_addr(req) {
                 let addr = remote_addr.to_string();
-                tracing::error_span!("grpc", addr)
+                error_span!("grpc", addr)
             } else {
-                tracing::error_span!("grpc")
+                error_span!("grpc")
             }
         })
         // (from Penumbra) Allow HTTP/1, which will be used by grpc-web connections.
diff --git a/crates/astria-sequencer/src/service/consensus.rs b/crates/astria-sequencer/src/service/consensus.rs
index a0b6a4a5ae..a5f5e8d4bb 100644
--- a/crates/astria-sequencer/src/service/consensus.rs
+++ b/crates/astria-sequencer/src/service/consensus.rs
@@ -15,9 +15,11 @@ use tokio::sync::mpsc;
 use tower_abci::BoxError;
 use tower_actor::Message;
 use tracing::{
+    debug,
     instrument,
     warn,
     Instrument,
+    Level,
 };
 
 use crate::app::App;
@@ -153,7 +155,7 @@ impl Consensus {
         })
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn handle_prepare_proposal(
         &mut self,
         prepare_proposal: request::PrepareProposal,
@@ -163,7 +165,7 @@ impl Consensus {
             .await
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err(level = Level::WARN))]
     async fn handle_process_proposal(
         &mut self,
         process_proposal: request::ProcessProposal,
@@ -171,11 +173,11 @@ impl Consensus {
         self.app
             .process_proposal(process_proposal, self.storage.clone())
             .await?;
-        tracing::debug!("proposal processed");
+        debug!("proposal processed");
         Ok(())
     }
 
-    #[instrument(skip_all)]
+    #[instrument(skip_all, err)]
     async fn finalize_block(
         &mut self,
         finalize_block: request::FinalizeBlock,
diff --git a/crates/astria-sequencer/src/service/mempool/mod.rs b/crates/astria-sequencer/src/service/mempool/mod.rs
index 47bd1dfe91..f6d47b2f6b 100644
--- a/crates/astria-sequencer/src/service/mempool/mod.rs
+++ b/crates/astria-sequencer/src/service/mempool/mod.rs
@@ -262,6 +262,7 @@ async fn handle_check_tx<S: StateRead>(
 }
 
 /// Checks if the transaction is already in the mempool.
+#[instrument(skip_all, fields(tx_hash = %hex::encode(tx_hash)))]
 async fn is_tracked(tx_hash: [u8; 32], mempool: &AppMempool, metrics: &Metrics) -> bool {
     let start_tracked_check = Instant::now();
 
@@ -276,6 +277,7 @@ async fn is_tracked(tx_hash: [u8; 32], mempool: &AppMempool, metrics: &Metrics)
 ///
 /// Returns an `Err(response::CheckTx)` with an error code and message if the transaction has been
 /// removed from the appside mempool.
+#[instrument(skip_all, fields(tx_hash = %hex::encode(tx_hash)))]
 async fn check_removed_comet_bft(
     tx_hash: [u8; 32],
     mempool: &AppMempool,
@@ -308,6 +310,7 @@ async fn check_removed_comet_bft(
 ///
 /// Returns an `Err(response::CheckTx)` if the transaction fails any of the checks.
 /// Otherwise, it returns the [`Transaction`] to be inserted into the mempool.
+#[instrument(skip_all)]
 async fn stateless_checks<S: StateRead>(
     tx: Bytes,
     state: &S,
@@ -387,6 +390,7 @@ async fn stateless_checks<S: StateRead>(
 ///
 /// Returns a `Err(response::CheckTx)` with an error code and message if the transaction fails
 /// insertion into the mempool.
+#[instrument(skip_all)]
 async fn insert_into_mempool<S: StateRead>(
     mempool: &AppMempool,
     state: &S,
diff --git a/crates/astria-sequencer/src/transaction/checks.rs b/crates/astria-sequencer/src/transaction/checks.rs
index bff86e838e..8d7943ca90 100644
--- a/crates/astria-sequencer/src/transaction/checks.rs
+++ b/crates/astria-sequencer/src/transaction/checks.rs
@@ -15,7 +15,10 @@ use astria_eyre::eyre::{
     WrapErr as _,
 };
 use cnidarium::StateRead;
-use tracing::instrument;
+use tracing::{
+    instrument,
+    Level,
+};
 
 use crate::{
     accounts::StateReadExt as _,
@@ -24,7 +27,7 @@ use crate::{
     fees::query::get_fees_for_transaction,
 };
 
-#[instrument(skip_all)]
+#[instrument(skip_all, fields(tx_hash = %tx.id()), err(level = Level::DEBUG))]
 pub(crate) async fn check_chain_id_mempool<S: StateRead>(
     tx: &Transaction,
     state: &S,
@@ -39,7 +42,7 @@ pub(crate) async fn check_chain_id_mempool<S: StateRead>(
 
 // Checks that the account has enough balance to cover the total fees and transferred values
 // for all actions in the transaction.
-#[instrument(skip_all)]
+#[instrument(skip_all, fields(tx_hash = %tx.id()), err(level = Level::DEBUG))]
 pub(crate) async fn check_balance_for_total_fees_and_transfers<S: StateRead>(
     tx: &Transaction,
     state: &S,
@@ -64,7 +67,7 @@ pub(crate) async fn check_balance_for_total_fees_and_transfers<S: StateRead>(
 
 // Returns the total cost of the transaction (fees and transferred values for all actions in the
 // transaction).
-#[instrument(skip_all)]
+#[instrument(skip_all, fields(tx_hash = %tx.id()), err(level = Level::DEBUG))]
 pub(crate) async fn get_total_transaction_cost<S: StateRead>(
     tx: &Transaction,
     state: &S,