From 9dde3e215ab94c4811c43a52f5085059a2c637ba Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 22 Nov 2023 11:38:30 +0100 Subject: [PATCH 001/135] Update readme --- README.md | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 27ceead..f6198c8 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,40 @@ A monolithized version of the [tx-sitter](https://github.com/worldcoin/tx-sitter-aws/). -## Testing locally +## Configuration +The Tx Sitter can be configured in 2 ways: +1. Using the config file, refer to `config.rs` and `config.toml` for more info +2. Using env vars. Every field in the config can also be set via an env var. + For example the following config + ```toml + [service] + escalation_interval = "1m" + + [server] + host = "127.0.0.1:3000" + disable_auth = true + + [rpc] + rpcs = ["http://127.0.0.1:8545"] + + [database] + connection_string = "postgres://postgres:postgres@127.0.0.1:5432/database" + + [keys] + kind = "local" + ``` + + Can also be expressed with env vars + ``` + TX_SITTER__SERVICE__ESCALATION_INTERVAL="1m" + TX_SITTER__SERVER__HOST="127.0.0.1:3000" + TX_SITTER__SERVER__DISABLE_AUTH="true" + TX_SITTER_EXT__RPC__RPCS="http://127.0.0.1:8545" + TX_SITTER__DATABASE__CONNECTION_STRING="postgres://postgres:postgres@127.0.0.1:5432/database" + TX_SITTER__KEYS__KIND="local" + ``` +## Testing locally Copy `.env.example` to `.env` or set `RUST_LOG=info,service=debug` to have logging. 1. Spin up the database `docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 postgres` From a64e224a763a7ced0634ec0827a4d872bec61ad8 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 22 Nov 2023 11:38:50 +0100 Subject: [PATCH 002/135] Only saved fee estimates on mined blocks --- src/app.rs | 2 +- src/db.rs | 4 ++-- src/tasks/index.rs | 25 ++++++++----------------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/app.rs b/src/app.rs index f3fffac..1910e15 100644 --- a/src/app.rs +++ b/src/app.rs @@ -114,7 +114,7 @@ async fn seed_initial_blocks( block.number.context("Missing block number")?.as_u64(), chain_id.as_u64(), &block.transactions, - &fee_estimates, + Some(&fee_estimates), BlockTxStatus::Mined, ) .await?; diff --git a/src/db.rs b/src/db.rs index 82a1e96..64250ed 100644 --- a/src/db.rs +++ b/src/db.rs @@ -255,7 +255,7 @@ impl Database { block_number: u64, chain_id: u64, txs: &[H256], - fee_estimates: &FeesEstimate, + fee_estimates: Option<&FeesEstimate>, status: BlockTxStatus, ) -> eyre::Result<()> { let mut db_tx = self.pool.begin().await?; @@ -271,7 +271,7 @@ impl Database { ) .bind(block_number as i64) .bind(chain_id as i64) - .bind(sqlx::types::Json(fee_estimates)) + .bind(fee_estimates.map(sqlx::types::Json)) .bind(status) .fetch_one(db_tx.as_mut()) .await?; diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 77fd7f5..d07c4e8 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -35,7 +35,7 @@ pub async fn index_blocks(app: Arc) -> eyre::Result<()> { block_number, chain_id.as_u64(), &block.transactions, - &fee_estimates, + Some(&fee_estimates), BlockTxStatus::Mined, ) .await?; @@ -59,20 +59,17 @@ pub async fn index_blocks(app: Arc) -> eyre::Result<()> { } if block_number > TRAILING_BLOCK_OFFSET { - let (block, fee_estimates) = - fetch_block_with_fee_estimates( - rpc, - block_number - TRAILING_BLOCK_OFFSET, - ) - .await? - .context("Missing trailing block")?; + let block = + fetch_block(rpc, block_number - TRAILING_BLOCK_OFFSET) + .await? + .context("Missing trailing block")?; app.db .save_block( block_number, chain_id.as_u64(), &block.transactions, - &fee_estimates, + None, BlockTxStatus::Finalized, ) .await?; @@ -111,7 +108,7 @@ pub async fn fetch_block_with_fee_estimates( pub async fn fetch_block( rpc: &Provider, block_id: impl Into, -) -> eyre::Result, FeesEstimate)>> { +) -> eyre::Result>> { let block_id = block_id.into(); let block = rpc.get_block(block_id).await?; @@ -120,11 +117,5 @@ pub async fn fetch_block( return Ok(None); }; - let fee_history = rpc - .fee_history(BLOCK_FEE_HISTORY_SIZE, block_id, &FEE_PERCENTILES) - .await?; - - let fee_estimates = estimate_percentile_fees(&fee_history)?; - - Ok(Some((block, fee_estimates))) + Ok(Some(block)) } From 366a5f607e110fcaa7d9e50d12330d4d26ec595a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 22 Nov 2023 13:27:25 +0100 Subject: [PATCH 003/135] Fixes & improvements --- Cargo.lock | 8 ++++ Cargo.toml | 2 + TODO.md | 7 ++- db/migrations/001_init.sql | 1 + src/app.rs | 9 ++++ src/db.rs | 75 ++++++++++++++++------------- src/db/data.rs | 32 ++++++++++++- src/tasks/index.rs | 98 ++++++++++++++++++++++++++++---------- 8 files changed, 171 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 984c18f..7e5b559 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4017,6 +4017,7 @@ dependencies = [ "atoi", "byteorder", "bytes", + "chrono", "crc", "crossbeam-queue", "dotenvy", @@ -4043,6 +4044,7 @@ dependencies = [ "smallvec", "sqlformat", "thiserror", + "time", "tokio", "tokio-stream", "tracing", @@ -4100,6 +4102,7 @@ dependencies = [ "bitflags 2.4.1", "byteorder", "bytes", + "chrono", "crc", "digest", "dotenvy", @@ -4127,6 +4130,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", + "time", "tracing", "whoami", ] @@ -4141,6 +4145,7 @@ dependencies = [ "base64 0.21.5", "bitflags 2.4.1", "byteorder", + "chrono", "crc", "dotenvy", "etcetera", @@ -4166,6 +4171,7 @@ dependencies = [ "sqlx-core", "stringprep", "thiserror", + "time", "tracing", "whoami", ] @@ -4177,6 +4183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59dc83cf45d89c555a577694534fcd1b55c545a816c816ce51f20bbe56a4f3f" dependencies = [ "atoi", + "chrono", "flume", "futures-channel", "futures-core", @@ -4188,6 +4195,7 @@ dependencies = [ "percent-encoding", "serde", "sqlx-core", + "time", "tracing", "url", ] diff --git a/Cargo.toml b/Cargo.toml index ffcd4c4..c50a76b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,8 @@ sha3 = "0.10.8" config = "0.13.3" toml = "0.8.8" sqlx = { version = "0.7.2", features = [ + "time", + "chrono", "runtime-tokio", "tls-rustls", "postgres", diff --git a/TODO.md b/TODO.md index 2258c9d..5999f14 100644 --- a/TODO.md +++ b/TODO.md @@ -12,4 +12,9 @@ 2. [ ] Basic with contracts 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) -9. [ ] Parallelization in a few places +9. [ ] Parallelization: + 1. [ ] Parallelize block indexing + 2. [x] Parallelize nonce updating + 3. [ ] Parallelize broadcast per chain id +10. [x] No need to insert all block txs into DB +11. [ ] Prune block info diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index c043647..481a42a 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -75,6 +75,7 @@ CREATE TABLE blocks ( id BIGSERIAL PRIMARY KEY, block_number BIGINT NOT NULL, chain_id BIGINT NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, -- mined | finalized status block_tx_status NOT NULL, fee_estimate JSON diff --git a/src/app.rs b/src/app.rs index 1910e15..45d55c0 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; +use chrono::NaiveDateTime; use ethers::middleware::SignerMiddleware; use ethers::providers::{Http, Middleware, Provider}; use ethers::types::{BlockNumber, U256}; @@ -110,9 +111,17 @@ async fn seed_initial_blocks( .await? .context("Missing latest block")?; + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = NaiveDateTime::from_timestamp_opt( + block_timestamp_seconds as i64, + 0, + ) + .context("Invalid timestamp")?; + db.save_block( block.number.context("Missing block number")?.as_u64(), chain_id.as_u64(), + block_timestamp, &block.transactions, Some(&fee_estimates), BlockTxStatus::Mined, diff --git a/src/db.rs b/src/db.rs index 64250ed..0669f32 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use chrono::NaiveDateTime; use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; use sqlx::{Pool, Postgres, Row}; @@ -9,7 +10,7 @@ use crate::config::DatabaseConfig; pub mod data; -use self::data::{AddressWrapper, ReadTxData}; +use self::data::{AddressWrapper, H256Wrapper, NextBlock, ReadTxData}; pub use self::data::{BlockTxStatus, TxForEscalation, UnsentTx}; // Statically link in migration files @@ -207,27 +208,29 @@ impl Database { Ok(item.map(|json_fee_estimate| json_fee_estimate.0)) } - pub async fn get_next_block_numbers( - &self, - ) -> eyre::Result> { - let rows: Vec<(i64, i64)> = sqlx::query_as( + pub async fn get_next_block_numbers(&self) -> eyre::Result> { + Ok(sqlx::query_as( r#" - SELECT MAX(block_number) + 1, chain_id - FROM blocks - WHERE status = $1 - GROUP BY chain_id + WITH LatestBlocks AS ( + SELECT + block_number, + chain_id, + timestamp, + ROW_NUMBER() OVER (PARTITION BY chain_id ORDER BY block_number DESC) AS rn + FROM blocks + WHERE status = $1 + ) + SELECT + block_number + 1 AS next_block_number, + chain_id, + timestamp as prev_block_timestamp + FROM LatestBlocks + WHERE rn = 1 "#, ) .bind(BlockTxStatus::Mined) .fetch_all(&self.pool) - .await?; - - Ok(rows - .into_iter() - .map(|(block_number, chain_id)| { - (block_number as u64, chain_id as u64) - }) - .collect()) + .await?) } pub async fn has_blocks_for_chain( @@ -254,40 +257,46 @@ impl Database { &self, block_number: u64, chain_id: u64, + timestamp: NaiveDateTime, txs: &[H256], fee_estimates: Option<&FeesEstimate>, status: BlockTxStatus, ) -> eyre::Result<()> { let mut db_tx = self.pool.begin().await?; - // let fee_estimates = serde_json::to_string(&fee_estimates)?; - let (block_id,): (i64,) = sqlx::query_as( r#" - INSERT INTO blocks (block_number, chain_id, fee_estimate, status) - VALUES ($1, $2, $3, $4) + INSERT INTO blocks (block_number, chain_id, timestamp, fee_estimate, status) + VALUES ($1, $2, $3, $4, $5) RETURNING id "#, ) .bind(block_number as i64) .bind(chain_id as i64) + .bind(timestamp) .bind(fee_estimates.map(sqlx::types::Json)) .bind(status) .fetch_one(db_tx.as_mut()) .await?; - for tx_hash in txs { - sqlx::query( - r#" - INSERT INTO block_txs (block_id, tx_hash) - VALUES ($1, $2) - "#, - ) - .bind(block_id) - .bind(tx_hash.as_bytes()) - .execute(db_tx.as_mut()) - .await?; - } + let txs: Vec<_> = txs.iter().map(|tx| H256Wrapper(*tx)).collect(); + + sqlx::query( + r#" + INSERT INTO block_txs (block_id, tx_hash) + SELECT $1, unnested.tx_hash + FROM UNNEST($2::BYTEA[]) AS unnested(tx_hash) + WHERE EXISTS ( + SELECT 1 + FROM tx_hashes + WHERE tx_hashes.tx_hash = unnested.tx_hash + ); + "#, + ) + .bind(block_id) + .bind(&txs[..]) + .execute(db_tx.as_mut()) + .await?; db_tx.commit().await?; diff --git a/src/db/data.rs b/src/db/data.rs index 0513bba..515c568 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -1,6 +1,8 @@ +use chrono::Utc; use ethers::types::{Address, H256, U256}; use serde::{Deserialize, Serialize}; -use sqlx::database::HasValueRef; +use sqlx::database::{HasArguments, HasValueRef}; +use sqlx::postgres::{PgHasArrayType, PgTypeInfo}; use sqlx::prelude::FromRow; use sqlx::Database; @@ -51,6 +53,15 @@ pub struct ReadTxData { pub status: Option, } +#[derive(Debug, Clone, FromRow)] +pub struct NextBlock { + #[sqlx(try_from = "i64")] + pub next_block_number: u64, + #[sqlx(try_from = "i64")] + pub chain_id: u64, + pub prev_block_timestamp: chrono::DateTime, +} + #[derive(Debug, Clone)] pub struct AddressWrapper(pub Address); #[derive(Debug, Clone)] @@ -133,6 +144,25 @@ where } } +impl<'q, DB> sqlx::Encode<'q, DB> for H256Wrapper +where + DB: Database, + [u8; 32]: sqlx::Encode<'q, DB>, +{ + fn encode_by_ref( + &self, + buf: &mut >::ArgumentBuffer, + ) -> sqlx::encode::IsNull { + <[u8; 32] as sqlx::Encode>::encode_by_ref(&self.0 .0, buf) + } +} + +impl PgHasArrayType for H256Wrapper { + fn array_type_info() -> PgTypeInfo { + <[u8; 32] as PgHasArrayType>::array_type_info() + } +} + impl sqlx::Type for H256Wrapper where [u8; 32]: sqlx::Type, diff --git a/src/tasks/index.rs b/src/tasks/index.rs index d07c4e8..d9e89b8 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -1,9 +1,12 @@ use std::sync::Arc; use std::time::Duration; +use chrono::NaiveDateTime; use ethers::providers::{Http, Middleware, Provider}; use ethers::types::{Block, BlockNumber, H256, U256}; use eyre::ContextCompat; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use crate::app::App; use crate::broadcast_utils::gas_estimation::{ @@ -19,21 +22,32 @@ pub async fn index_blocks(app: Arc) -> eyre::Result<()> { loop { let next_block_numbers = app.db.get_next_block_numbers().await?; - // TODO: Parallelize - for (block_number, chain_id) in next_block_numbers { - let chain_id = U256::from(chain_id); + for next_block in next_block_numbers { + let chain_id = U256::from(next_block.chain_id); let rpc = app .rpcs .get(&chain_id) .context("Missing RPC for chain id")?; if let Some((block, fee_estimates)) = - fetch_block_with_fee_estimates(rpc, block_number).await? + fetch_block_with_fee_estimates( + rpc, + next_block.next_block_number, + ) + .await? { + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = NaiveDateTime::from_timestamp_opt( + block_timestamp_seconds as i64, + 0, + ) + .context("Invalid timestamp")?; + app.db .save_block( - block_number, + next_block.next_block_number, chain_id.as_u64(), + block_timestamp, &block.transactions, Some(&fee_estimates), BlockTxStatus::Mined, @@ -43,47 +57,79 @@ pub async fn index_blocks(app: Arc) -> eyre::Result<()> { let relayer_addresses = app.db.fetch_relayer_addresses(chain_id.as_u64()).await?; - // TODO: Parallelize - for relayer_address in relayer_addresses { - let tx_count = rpc - .get_transaction_count(relayer_address, None) - .await?; + update_relayer_nonces(relayer_addresses, &app, rpc, chain_id) + .await?; - app.db - .update_relayer_nonce( - chain_id.as_u64(), - relayer_address, - tx_count.as_u64(), - ) - .await?; - } + if next_block.next_block_number > TRAILING_BLOCK_OFFSET { + let block = fetch_block( + rpc, + next_block.next_block_number - TRAILING_BLOCK_OFFSET, + ) + .await? + .context("Missing trailing block")?; - if block_number > TRAILING_BLOCK_OFFSET { - let block = - fetch_block(rpc, block_number - TRAILING_BLOCK_OFFSET) - .await? - .context("Missing trailing block")?; + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = NaiveDateTime::from_timestamp_opt( + block_timestamp_seconds as i64, + 0, + ) + .context("Invalid timestamp")?; app.db .save_block( - block_number, + next_block.next_block_number, chain_id.as_u64(), + block_timestamp, &block.transactions, None, BlockTxStatus::Finalized, ) .await?; } - } else { - tokio::time::sleep(Duration::from_secs(5)).await; } } app.db.update_transactions(BlockTxStatus::Mined).await?; app.db.update_transactions(BlockTxStatus::Finalized).await?; + + tokio::time::sleep(Duration::from_secs(2)).await; } } +async fn update_relayer_nonces( + relayer_addresses: Vec, + app: &Arc, + rpc: &Provider, + chain_id: U256, +) -> Result<(), eyre::Error> { + let mut futures = FuturesUnordered::new(); + + for relayer_address in relayer_addresses { + let app = app.clone(); + + futures.push(async move { + let tx_count = + rpc.get_transaction_count(relayer_address, None).await?; + + app.db + .update_relayer_nonce( + chain_id.as_u64(), + relayer_address, + tx_count.as_u64(), + ) + .await?; + + Result::<(), eyre::Report>::Ok(()) + }) + } + + while let Some(result) = futures.next().await { + result?; + } + + Ok(()) +} + pub async fn fetch_block_with_fee_estimates( rpc: &Provider, block_id: impl Into, From bd5ac6819d8249f4ff5d8bfdc55b4aa53792eb87 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 23 Nov 2023 22:47:14 +0100 Subject: [PATCH 004/135] Update stuff --- src/lib.rs | 2 +- src/service.rs | 2 +- src/{task_backoff.rs => task_runner.rs} | 17 +-- src/tasks/index.rs | 144 +++++++++++++----------- 4 files changed, 90 insertions(+), 75 deletions(-) rename src/{task_backoff.rs => task_runner.rs} (89%) diff --git a/src/lib.rs b/src/lib.rs index b52f53c..f64234f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ pub mod config; pub mod db; pub mod server; pub mod service; -pub mod task_backoff; +pub mod task_runner; pub mod tasks; pub mod broadcast_utils; diff --git a/src/service.rs b/src/service.rs index b07c6c4..4f8d1c7 100644 --- a/src/service.rs +++ b/src/service.rs @@ -5,7 +5,7 @@ use tokio::task::JoinHandle; use crate::app::App; use crate::config::Config; -use crate::task_backoff::TaskRunner; +use crate::task_runner::TaskRunner; use crate::tasks; pub struct Service { diff --git a/src/task_backoff.rs b/src/task_runner.rs similarity index 89% rename from src/task_backoff.rs rename to src/task_runner.rs index 4099e36..4108ba3 100644 --- a/src/task_backoff.rs +++ b/src/task_runner.rs @@ -3,23 +3,26 @@ use std::time::{Duration, Instant}; use futures::Future; -use crate::app::App; - const FAILURE_MONITORING_PERIOD: Duration = Duration::from_secs(60); -pub struct TaskRunner { - app: Arc, +pub struct TaskRunner { + app: Arc, } -impl TaskRunner { - pub fn new(app: Arc) -> Self { +impl TaskRunner { + pub fn new(app: Arc) -> Self { Self { app } } +} +impl TaskRunner +where + T: Send + Sync + 'static, +{ pub fn add_task(&self, label: S, task: C) where S: ToString, - C: Fn(Arc) -> F + Send + Sync + 'static, + C: Fn(Arc) -> F + Send + Sync + 'static, F: Future> + Send + 'static, { let app = self.app.clone(); diff --git a/src/tasks/index.rs b/src/tasks/index.rs index d9e89b8..5089bf8 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -12,6 +12,7 @@ use crate::app::App; use crate::broadcast_utils::gas_estimation::{ estimate_percentile_fees, FeesEstimate, }; +use crate::db::data::NextBlock; use crate::db::BlockTxStatus; const BLOCK_FEE_HISTORY_SIZE: usize = 10; @@ -23,79 +24,90 @@ pub async fn index_blocks(app: Arc) -> eyre::Result<()> { let next_block_numbers = app.db.get_next_block_numbers().await?; for next_block in next_block_numbers { - let chain_id = U256::from(next_block.chain_id); - let rpc = app - .rpcs - .get(&chain_id) - .context("Missing RPC for chain id")?; - - if let Some((block, fee_estimates)) = - fetch_block_with_fee_estimates( - rpc, - next_block.next_block_number, - ) - .await? - { - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = NaiveDateTime::from_timestamp_opt( - block_timestamp_seconds as i64, - 0, - ) - .context("Invalid timestamp")?; - - app.db - .save_block( - next_block.next_block_number, - chain_id.as_u64(), - block_timestamp, - &block.transactions, - Some(&fee_estimates), - BlockTxStatus::Mined, - ) - .await?; - - let relayer_addresses = - app.db.fetch_relayer_addresses(chain_id.as_u64()).await?; - - update_relayer_nonces(relayer_addresses, &app, rpc, chain_id) - .await?; - - if next_block.next_block_number > TRAILING_BLOCK_OFFSET { - let block = fetch_block( - rpc, - next_block.next_block_number - TRAILING_BLOCK_OFFSET, - ) - .await? - .context("Missing trailing block")?; - - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = NaiveDateTime::from_timestamp_opt( - block_timestamp_seconds as i64, - 0, - ) - .context("Invalid timestamp")?; - - app.db - .save_block( - next_block.next_block_number, - chain_id.as_u64(), - block_timestamp, - &block.transactions, - None, - BlockTxStatus::Finalized, - ) - .await?; - } - } + update_block(app.clone(), next_block).await?; } - app.db.update_transactions(BlockTxStatus::Mined).await?; - app.db.update_transactions(BlockTxStatus::Finalized).await?; + let (update_mined, update_finalized) = tokio::join!( + app.db.update_transactions(BlockTxStatus::Mined), + app.db.update_transactions(BlockTxStatus::Finalized) + ); + + update_mined?; + update_finalized?; tokio::time::sleep(Duration::from_secs(2)).await; } } +async fn update_block( + app: Arc, + next_block: NextBlock, +) -> eyre::Result<()> { + let chain_id = U256::from(next_block.chain_id); + let rpc = app + .rpcs + .get(&chain_id) + .context("Missing RPC for chain id")?; + + let block = + fetch_block_with_fee_estimates(rpc, next_block.next_block_number) + .await?; + + let Some((block, fee_estimates)) = block else { + return Ok(()); + }; + + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = + NaiveDateTime::from_timestamp_opt(block_timestamp_seconds as i64, 0) + .context("Invalid timestamp")?; + + app.db + .save_block( + next_block.next_block_number, + chain_id.as_u64(), + block_timestamp, + &block.transactions, + Some(&fee_estimates), + BlockTxStatus::Mined, + ) + .await?; + + let relayer_addresses = + app.db.fetch_relayer_addresses(chain_id.as_u64()).await?; + + update_relayer_nonces(relayer_addresses, &app, rpc, chain_id).await?; + + if next_block.next_block_number > TRAILING_BLOCK_OFFSET { + let block = fetch_block( + rpc, + next_block.next_block_number - TRAILING_BLOCK_OFFSET, + ) + .await? + .context("Missing trailing block")?; + + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = NaiveDateTime::from_timestamp_opt( + block_timestamp_seconds as i64, + 0, + ) + .context("Invalid timestamp")?; + + app.db + .save_block( + next_block.next_block_number, + chain_id.as_u64(), + block_timestamp, + &block.transactions, + None, + BlockTxStatus::Finalized, + ) + .await?; + } + + Ok(()) +} + async fn update_relayer_nonces( relayer_addresses: Vec, app: &Arc, From 19055474b10df8b9f482f521f205adab292f5623 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 23 Nov 2023 23:15:03 +0100 Subject: [PATCH 005/135] Prune blocks --- TODO.md | 6 ++-- src/app.rs | 4 +-- src/db.rs | 90 ++++++++++++++++++++++++++++++++++++++++++++-- src/db/data.rs | 6 ++-- src/service.rs | 1 + src/tasks.rs | 2 ++ src/tasks/index.rs | 12 +++---- src/tasks/prune.rs | 35 ++++++++++++++++++ 8 files changed, 139 insertions(+), 17 deletions(-) create mode 100644 src/tasks/prune.rs diff --git a/TODO.md b/TODO.md index 5999f14..400d294 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,7 @@ # TODO 1. [ ] Handling reorgs -2. [ ] Per network settings (i.e. max inflight txs, max gas price) +2. [ ] Per network settings (i.e. max inflight txs, max gas price, block time) 3. [ ] Multiple RPCs support 4. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) 5. [ ] Transaction priority @@ -13,8 +13,8 @@ 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) 9. [ ] Parallelization: - 1. [ ] Parallelize block indexing + 1. [ ] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating 3. [ ] Parallelize broadcast per chain id 10. [x] No need to insert all block txs into DB -11. [ ] Prune block info +11. [x] Prune block info diff --git a/src/app.rs b/src/app.rs index 45d55c0..e3cda73 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::Arc; -use chrono::NaiveDateTime; +use chrono::{DateTime, Utc}; use ethers::middleware::SignerMiddleware; use ethers::providers::{Http, Middleware, Provider}; use ethers::types::{BlockNumber, U256}; @@ -112,7 +112,7 @@ async fn seed_initial_blocks( .context("Missing latest block")?; let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = NaiveDateTime::from_timestamp_opt( + let block_timestamp = DateTime::::from_timestamp( block_timestamp_seconds as i64, 0, ) diff --git a/src/db.rs b/src/db.rs index 0669f32..e000813 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use chrono::NaiveDateTime; +use chrono::{DateTime, NaiveDateTime, Utc}; use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; use sqlx::{Pool, Postgres, Row}; @@ -257,7 +257,7 @@ impl Database { &self, block_number: u64, chain_id: u64, - timestamp: NaiveDateTime, + timestamp: DateTime, txs: &[H256], fee_estimates: Option<&FeesEstimate>, status: BlockTxStatus, @@ -470,10 +470,46 @@ impl Database { Ok(()) } + + pub async fn prune_blocks( + &self, + timestamp: DateTime, + ) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + sqlx::query( + r#" + DELETE FROM block_txs + WHERE block_id IN ( + SELECT id + FROM blocks + WHERE timestamp < $1 + ) + "#, + ) + .bind(timestamp) + .execute(tx.as_mut()) + .await?; + + sqlx::query( + r#" + DELETE FROM blocks + WHERE timestamp < $1 + "#, + ) + .bind(timestamp) + .execute(tx.as_mut()) + .await?; + + tx.commit().await?; + + Ok(()) + } } #[cfg(test)] mod tests { + use chrono::NaiveDate; use postgres_docker_utils::DockerContainerGuard; use super::*; @@ -500,4 +536,54 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn save_and_prune_blocks() -> eyre::Result<()> { + let (db, _db_container) = setup_db().await?; + + let block_timestamp = NaiveDate::from_ymd_opt(2023, 11, 23) + .unwrap() + .and_hms_opt(12, 32, 2) + .unwrap() + .and_utc(); + + let prune_timestamp = NaiveDate::from_ymd_opt(2023, 11, 23) + .unwrap() + .and_hms_opt(13, 32, 23) + .unwrap() + .and_utc(); + + let tx_hashes = vec![ + H256::from_low_u64_be(1), + H256::from_low_u64_be(2), + H256::from_low_u64_be(3), + ]; + + db.save_block( + 1, + 1, + block_timestamp.clone(), + &tx_hashes, + None, + BlockTxStatus::Mined, + ) + .await?; + + let next_blocks = db.get_next_block_numbers().await?; + let expected = vec![NextBlock { + next_block_number: 2, + chain_id: 1, + prev_block_timestamp: block_timestamp, + }]; + + assert_eq!(next_blocks, expected, "Should return next block"); + + db.prune_blocks(prune_timestamp).await?; + + let next_blocks = db.get_next_block_numbers().await?; + + assert!(next_blocks.is_empty(), "Should return no blocks"); + + Ok(()) + } } diff --git a/src/db/data.rs b/src/db/data.rs index 515c568..9666226 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -1,4 +1,4 @@ -use chrono::Utc; +use chrono::{DateTime, Utc}; use ethers::types::{Address, H256, U256}; use serde::{Deserialize, Serialize}; use sqlx::database::{HasArguments, HasValueRef}; @@ -53,13 +53,13 @@ pub struct ReadTxData { pub status: Option, } -#[derive(Debug, Clone, FromRow)] +#[derive(Debug, Clone, FromRow, PartialEq, Eq)] pub struct NextBlock { #[sqlx(try_from = "i64")] pub next_block_number: u64, #[sqlx(try_from = "i64")] pub chain_id: u64, - pub prev_block_timestamp: chrono::DateTime, + pub prev_block_timestamp: DateTime, } #[derive(Debug, Clone)] diff --git a/src/service.rs b/src/service.rs index 4f8d1c7..118dbb8 100644 --- a/src/service.rs +++ b/src/service.rs @@ -22,6 +22,7 @@ impl Service { task_runner.add_task("Broadcast transactions", tasks::broadcast_txs); task_runner.add_task("Index transactions", tasks::index_blocks); task_runner.add_task("Escalate transactions", tasks::escalate_txs); + task_runner.add_task("Prune blocks", tasks::prune_blocks); let server = crate::server::spawn_server(app.clone()).await?; let local_addr = server.local_addr(); diff --git a/src/tasks.rs b/src/tasks.rs index 8839ce6..8f99ac1 100644 --- a/src/tasks.rs +++ b/src/tasks.rs @@ -1,7 +1,9 @@ pub mod broadcast; pub mod escalate; pub mod index; +pub mod prune; pub use self::broadcast::broadcast_txs; pub use self::escalate::escalate_txs; pub use self::index::index_blocks; +pub use self::prune::prune_blocks; diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 5089bf8..92333ce 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use chrono::NaiveDateTime; +use chrono::{DateTime, Utc}; use ethers::providers::{Http, Middleware, Provider}; use ethers::types::{Block, BlockNumber, H256, U256}; use eyre::ContextCompat; @@ -59,7 +59,7 @@ async fn update_block( let block_timestamp_seconds = block.timestamp.as_u64(); let block_timestamp = - NaiveDateTime::from_timestamp_opt(block_timestamp_seconds as i64, 0) + DateTime::::from_timestamp(block_timestamp_seconds as i64, 0) .context("Invalid timestamp")?; app.db @@ -87,11 +87,9 @@ async fn update_block( .context("Missing trailing block")?; let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = NaiveDateTime::from_timestamp_opt( - block_timestamp_seconds as i64, - 0, - ) - .context("Invalid timestamp")?; + let block_timestamp = + DateTime::::from_timestamp(block_timestamp_seconds as i64, 0) + .context("Invalid timestamp")?; app.db .save_block( diff --git a/src/tasks/prune.rs b/src/tasks/prune.rs new file mode 100644 index 0000000..7651cf0 --- /dev/null +++ b/src/tasks/prune.rs @@ -0,0 +1,35 @@ +use std::sync::Arc; +use std::time::Duration; + +use chrono::Utc; + +use crate::app::App; + +const PRUNING_INTERVAL: Duration = Duration::from_secs(60); + +const fn minutes(seconds: i64) -> i64 { + seconds * 60 +} + +const fn hours(seconds: i64) -> i64 { + minutes(seconds) * 60 +} + +const fn days(seconds: i64) -> i64 { + hours(seconds) * 24 +} + +const BLOCK_PRUNE_AGE_SECONDS: i64 = days(7); + +pub async fn prune_blocks(app: Arc) -> eyre::Result<()> { + loop { + let prune_age = chrono::Duration::seconds(BLOCK_PRUNE_AGE_SECONDS); + let block_prune_timestamp = Utc::now() - prune_age; + + tracing::info!(?block_prune_timestamp, "Pruning blocks"); + + app.db.prune_blocks(block_prune_timestamp).await?; + + tokio::time::sleep(PRUNING_INTERVAL).await; + } +} From b1fa082a2fa738faf6ce22f8c282ebc406953c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Mon, 27 Nov 2023 16:49:25 +0100 Subject: [PATCH 006/135] Networks in DB - Handling reorgs (#2) --- Cargo.lock | 3 +- Cargo.toml | 4 +- TODO.md | 6 +- crates/fake-rpc/src/lib.rs | 4 + db/migrations/001_init.sql | 81 +++-- manual_test.nu | 15 +- src/app.rs | 110 ++---- src/aws/ethers_signer.rs | 2 +- src/client.rs | 63 ++++ src/config.rs | 30 +- src/db.rs | 633 ++++++++++++++++++++++++++++------- src/db/data.rs | 41 +-- src/keys.rs | 4 +- src/keys/universal_signer.rs | 3 +- src/lib.rs | 9 +- src/server.rs | 11 +- src/server/data.rs | 4 +- src/server/routes.rs | 1 + src/server/routes/network.rs | 74 ++++ src/service.rs | 5 +- src/task_runner.rs | 35 ++ src/tasks.rs | 8 +- src/tasks/broadcast.rs | 2 +- src/tasks/finalize.rs | 25 ++ src/tasks/handle_reorgs.rs | 34 ++ src/tasks/index.rs | 157 +++------ src/tasks/prune.rs | 22 +- tests/common/mod.rs | 42 ++- tests/create_relayer.rs | 15 +- tests/send_many_txs.rs | 33 +- tests/send_tx.rs | 34 +- 31 files changed, 1046 insertions(+), 464 deletions(-) create mode 100644 src/server/routes.rs create mode 100644 src/server/routes/network.rs create mode 100644 src/tasks/finalize.rs create mode 100644 src/tasks/handle_reorgs.rs diff --git a/Cargo.lock b/Cargo.lock index 7e5b559..13a5a89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1569,6 +1569,7 @@ dependencies = [ "const-hex", "enr", "ethers-core", + "futures-channel", "futures-core", "futures-timer", "futures-util", @@ -3801,7 +3802,6 @@ dependencies = [ "config", "dotenv", "ethers", - "ethers-signers", "eyre", "fake-rpc", "futures", @@ -3828,6 +3828,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "uuid 0.8.2", ] diff --git a/Cargo.toml b/Cargo.toml index c50a76b..8144505 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,7 @@ humantime-serde = "1.1.1" hyper = "0.14.27" dotenv = "0.15.0" clap = { version = "4.3.0", features = ["env", "derive"] } -ethers = { version = "2.0.11" } -ethers-signers = { version = "2.0.11" } +ethers = { version = "2.0.11", features = ["ws"] } eyre = "0.6.5" hex = "0.4.3" hex-literal = "0.4.1" @@ -54,6 +53,7 @@ rand = "0.8.5" sha3 = "0.10.8" config = "0.13.3" toml = "0.8.8" +url = "2.4.1" sqlx = { version = "0.7.2", features = [ "time", "chrono", diff --git a/TODO.md b/TODO.md index 400d294..e92ecdb 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,6 @@ # TODO -1. [ ] Handling reorgs +1. [x] Handling reorgs 2. [ ] Per network settings (i.e. max inflight txs, max gas price, block time) 3. [ ] Multiple RPCs support 4. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) @@ -12,8 +12,8 @@ 2. [ ] Basic with contracts 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) -9. [ ] Parallelization: - 1. [ ] Parallelize block indexing - depends on per network settings +9. [x] Parallelization: + 1. [x] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating 3. [ ] Parallelize broadcast per chain id 10. [x] No need to insert all block txs into DB diff --git a/crates/fake-rpc/src/lib.rs b/crates/fake-rpc/src/lib.rs index 2a9dbc0..1ec8738 100644 --- a/crates/fake-rpc/src/lib.rs +++ b/crates/fake-rpc/src/lib.rs @@ -51,6 +51,10 @@ impl DoubleAnvil { self.auto_advance .store(auto_advance, std::sync::atomic::Ordering::SeqCst); } + + pub async fn ws_endpoint(&self) -> String { + self.main_anvil.lock().await.ws_endpoint() + } } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 481a42a..dc25580 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -1,20 +1,18 @@ -CREATE TYPE block_tx_status AS ENUM ( - 'pending', - 'mined', - 'finalized' -); +CREATE TYPE tx_status AS ENUM ('pending', 'mined', 'finalized'); --- create table networks ( --- id BIGSERIAL PRIMARY KEY, --- name VARCHAR(255) NOT NULL, --- chain_id BIGINT NOT NULL --- ); +CREATE TYPE rpc_kind AS ENUM ('http', 'ws'); --- create table rpcs ( --- id BIGSERIAL PRIMARY KEY, --- network_id BIGINT NOT NULL REFERENCES networks(id), --- url VARCHAR(255) NOT NULL --- ); +create table networks ( + chain_id BIGINT PRIMARY KEY, + name VARCHAR(255) NOT NULL +); + +create table rpcs ( + id BIGSERIAL PRIMARY KEY, + chain_id BIGINT NOT NULL REFERENCES networks(chain_id), + url VARCHAR(255) NOT NULL, + kind rpc_kind NOT NULL +); CREATE TABLE relayers ( id VARCHAR(255) PRIMARY KEY, @@ -45,45 +43,46 @@ CREATE TABLE transactions ( relayer_id VARCHAR(255) NOT NULL REFERENCES relayers(id) ); --- Dynamic tx data & data used for escalations -CREATE TABLE sent_transactions ( - tx_id VARCHAR(255) PRIMARY KEY REFERENCES transactions(id), - initial_max_fee_per_gas BYTEA NOT NULL, - initial_max_priority_fee_per_gas BYTEA NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - first_submitted_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - mined_at TIMESTAMP, - escalation_count BIGINT NOT NULL DEFAULT 0, - last_escalation TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - reorg BOOL NOT NULL DEFAULT FALSE -); - -- Sent transaction attempts CREATE TABLE tx_hashes ( tx_hash BYTEA PRIMARY KEY, - tx_id VARCHAR(255) NOT NULL REFERENCES transactions(id), + tx_id VARCHAR(255) NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, max_fee_per_gas BYTEA NOT NULL, max_priority_fee_per_gas BYTEA NOT NULL, - escalated BOOL NOT NULL DEFAULT FALSE, - -- pending | mined | finalized - status block_tx_status NOT NULL DEFAULT 'pending' + escalated BOOL NOT NULL DEFAULT FALSE +); + +-- Dynamic tx data & data used for escalations +CREATE TABLE sent_transactions ( + tx_id VARCHAR(255) PRIMARY KEY REFERENCES transactions(id) ON DELETE CASCADE, + initial_max_fee_per_gas BYTEA NOT NULL, + initial_max_priority_fee_per_gas BYTEA NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + first_submitted_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + mined_at TIMESTAMP, + escalation_count BIGINT NOT NULL DEFAULT 0, + last_escalation TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + reorg BOOL NOT NULL DEFAULT FALSE, + status tx_status NOT NULL DEFAULT 'pending', + -- If the status is mined or finalized this should be the actual tx hash that is mined or finalized + valid_tx_hash BYTEA NOT NULL REFERENCES tx_hashes(tx_hash) ); CREATE TABLE blocks ( - id BIGSERIAL PRIMARY KEY, block_number BIGINT NOT NULL, chain_id BIGINT NOT NULL, timestamp TIMESTAMPTZ NOT NULL, - -- mined | finalized - status block_tx_status NOT NULL, - fee_estimate JSON + fee_estimate JSON, + PRIMARY KEY (block_number, chain_id) ); CREATE TABLE block_txs ( - block_id BIGINT REFERENCES blocks(id), + block_number BIGINT NOT NULL, + chain_id BIGINT NOT NULL, tx_hash BYTEA NOT NULL, - PRIMARY KEY (block_id, tx_hash) -); - + PRIMARY KEY (block_number, chain_id, tx_hash), + FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE, + FOREIGN KEY (tx_hash) REFERENCES tx_hashes (tx_hash) +); \ No newline at end of file diff --git a/manual_test.nu b/manual_test.nu index 9fa8df3..0b045fa 100644 --- a/manual_test.nu +++ b/manual_test.nu @@ -1,11 +1,20 @@ +let txSitter = "http://127.0.0.1:3000" +let anvilSocket = "127.0.0.1:8545" + +http post -t application/json $"($txSitter)/1/network/31337" { + name: "Anvil network", + httpRpc: $"http://($anvilSocket)", + wsRpc: $"ws://($anvilSocket)" +} + echo "Creating relayer" -let relayer = http post -t application/json http://127.0.0.1:3000/1/relayer/create { "name": "My Relayer", "chainId": 31337 } +let relayer = http post -t application/json $"($txSitter)/1/relayer/create" { "name": "My Relayer", "chainId": 31337 } echo "Funding relayer" cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' echo "Sending transaction" -let tx = http post -t application/json http://127.0.0.1:3000/1/tx/send { +let tx = http post -t application/json $"($txSitter)/1/tx/send" { "relayerId": $relayer.relayerId, "to": $relayer.address, "value": "10", @@ -15,7 +24,7 @@ let tx = http post -t application/json http://127.0.0.1:3000/1/tx/send { echo "Wait until tx is mined" for i in 0..100 { - let txResponse = http get http://127.0.0.1:3000/1/tx/($tx.txId) + let txResponse = http get $"($txSitter)/1/tx/($tx.txId)" if ($txResponse | get -i status) == "mined" { echo $txResponse diff --git a/src/app.rs b/src/app.rs index e3cda73..b67db93 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,25 +1,20 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use chrono::{DateTime, Utc}; use ethers::middleware::SignerMiddleware; -use ethers::providers::{Http, Middleware, Provider}; -use ethers::types::{BlockNumber, U256}; -use ethers_signers::Signer; -use eyre::{Context, ContextCompat}; +use ethers::providers::{Http, Provider, Ws}; +use ethers::signers::Signer; +use eyre::Context; use crate::config::{Config, KeysConfig}; -use crate::db::{BlockTxStatus, Database}; +use crate::db::data::RpcKind; +use crate::db::Database; use crate::keys::{KeysSource, KmsKeys, LocalKeys, UniversalSigner}; -use crate::tasks::index::fetch_block_with_fee_estimates; -pub type AppMiddleware = SignerMiddleware>, UniversalSigner>; +pub type AppGenericMiddleware = + SignerMiddleware, UniversalSigner>; +pub type AppMiddleware = AppGenericMiddleware; pub struct App { pub config: Config, - pub rpcs: HashMap>>, - pub keys_source: Box, pub db: Database, @@ -27,32 +22,45 @@ pub struct App { impl App { pub async fn new(config: Config) -> eyre::Result { - let rpcs = init_rpcs(&config).await?; let keys_source = init_keys_source(&config).await?; let db = Database::new(&config.database).await?; - seed_initial_blocks(&rpcs, &db).await?; - Ok(Self { config, - rpcs, keys_source, db, }) } + pub async fn fetch_http_provider( + &self, + chain_id: u64, + ) -> eyre::Result> { + let url = self.db.get_network_rpc(chain_id, RpcKind::Http).await?; + + let provider = Provider::::try_from(url.as_str())?; + + Ok(provider) + } + + pub async fn fetch_ws_provider( + &self, + chain_id: u64, + ) -> eyre::Result> { + let url = self.db.get_network_rpc(chain_id, RpcKind::Ws).await?; + + let ws = Ws::connect(url.as_str()).await?; + let provider = Provider::new(ws); + + Ok(provider) + } + pub async fn fetch_signer_middleware( &self, - chain_id: impl Into, + chain_id: u64, key_id: String, ) -> eyre::Result { - let chain_id: U256 = chain_id.into(); - - let rpc = self - .rpcs - .get(&chain_id) - .context("Missing RPC for chain id")? - .clone(); + let rpc = self.fetch_http_provider(chain_id).await?; let wallet = self .keys_source @@ -60,7 +68,7 @@ impl App { .await .context("Missing signer")?; - let wallet = wallet.with_chain_id(chain_id.as_u64()); + let wallet = wallet.with_chain_id(chain_id); let middlware = SignerMiddleware::new(rpc, wallet); @@ -82,53 +90,3 @@ async fn init_keys_source( Ok(keys_source) } - -async fn init_rpcs( - config: &Config, -) -> eyre::Result>>> { - let mut providers = HashMap::new(); - - for rpc_url in &config.rpc.rpcs { - let provider = Provider::::try_from(rpc_url.as_str())?; - let chain_id = provider.get_chainid().await?; - - providers.insert(chain_id, Arc::new(provider)); - } - - Ok(providers) -} - -async fn seed_initial_blocks( - rpcs: &HashMap>>, - db: &Database, -) -> eyre::Result<()> { - for (chain_id, rpc) in rpcs { - tracing::info!("Seeding block for chain id {chain_id}"); - - if !db.has_blocks_for_chain(chain_id.as_u64()).await? { - let (block, fee_estimates) = - fetch_block_with_fee_estimates(rpc, BlockNumber::Latest) - .await? - .context("Missing latest block")?; - - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = DateTime::::from_timestamp( - block_timestamp_seconds as i64, - 0, - ) - .context("Invalid timestamp")?; - - db.save_block( - block.number.context("Missing block number")?.as_u64(), - chain_id.as_u64(), - block_timestamp, - &block.transactions, - Some(&fee_estimates), - BlockTxStatus::Mined, - ) - .await?; - } - } - - Ok(()) -} diff --git a/src/aws/ethers_signer.rs b/src/aws/ethers_signer.rs index 3296679..b4530d2 100644 --- a/src/aws/ethers_signer.rs +++ b/src/aws/ethers_signer.rs @@ -34,7 +34,7 @@ use utils::{apply_eip155, verifying_key_to_address}; /// use rusoto_core::Client; /// use rusoto_kms::{Kms, KmsClient}; /// -/// user ethers_signers::Signer; +/// user ethers::signers::Signer; /// /// let client = Client::new_with( /// EnvironmentProvider::default(), diff --git a/src/client.rs b/src/client.rs index 8b13789..4c6ce5a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1 +1,64 @@ +use crate::server::data::{ + CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, SendTxResponse, +}; +use crate::server::routes::network::NewNetworkInfo; +pub struct TxSitterClient { + client: reqwest::Client, + url: String, +} + +impl TxSitterClient { + pub fn new(url: impl ToString) -> Self { + Self { + client: reqwest::Client::new(), + url: url.to_string(), + } + } + + pub async fn create_relayer( + &self, + req: &CreateRelayerRequest, + ) -> eyre::Result { + let response = self + .client + .post(&format!("{}/1/relayer/create", self.url)) + .json(req) + .send() + .await?; + + let response: CreateRelayerResponse = response.json().await?; + + Ok(response) + } + + pub async fn send_tx( + &self, + req: &SendTxRequest, + ) -> eyre::Result { + let response = self + .client + .post(&format!("{}/1/tx/send", self.url)) + .json(req) + .send() + .await?; + + Ok(response.json().await?) + } + + pub async fn create_network( + &self, + chain_id: u64, + req: &NewNetworkInfo, + ) -> eyre::Result<()> { + self.client + .post(&format!("{}/1/network/{}", self.url, chain_id)) + .json(req) + .send() + .await?; + + // TODO: Handle status? + + Ok(()) + } +} diff --git a/src/config.rs b/src/config.rs index 03af392..1c5e218 100644 --- a/src/config.rs +++ b/src/config.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Serialize}; pub struct Config { pub service: TxSitterConfig, pub server: ServerConfig, - pub rpc: RpcConfig, pub database: DatabaseConfig, pub keys: KeysConfig, } @@ -29,13 +28,6 @@ pub struct ServerConfig { pub disable_auth: bool, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct RpcConfig { - #[serde(default)] - pub rpcs: Vec, -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct DatabaseConfig { @@ -61,8 +53,25 @@ pub struct LocalKeysConfig {} #[cfg(test)] mod tests { + use indoc::indoc; + use super::*; + const SAMPLE: &str = indoc! {r#" + [service] + escalation_interval = "1h" + + [server] + host = "127.0.0.1:3000" + disable_auth = false + + [database] + connection_string = "postgres://postgres:postgres@127.0.0.1:52804/database" + + [keys] + kind = "local" + "#}; + #[test] fn sample() { let config = Config { @@ -73,9 +82,6 @@ mod tests { host: SocketAddr::from(([127, 0, 0, 1], 3000)), disable_auth: false, }, - rpc: RpcConfig { - rpcs: vec!["hello".to_string()], - }, database: DatabaseConfig { connection_string: "postgres://postgres:postgres@127.0.0.1:52804/database" @@ -86,6 +92,6 @@ mod tests { let toml = toml::to_string_pretty(&config).unwrap(); - println!("{}", toml); + assert_eq!(toml, SAMPLE); } } diff --git a/src/db.rs b/src/db.rs index e000813..629642e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{DateTime, Utc}; use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; use sqlx::{Pool, Postgres, Row}; @@ -10,8 +10,8 @@ use crate::config::DatabaseConfig; pub mod data; -use self::data::{AddressWrapper, H256Wrapper, NextBlock, ReadTxData}; -pub use self::data::{BlockTxStatus, TxForEscalation, UnsentTx}; +use self::data::{AddressWrapper, H256Wrapper, ReadTxData, RpcKind}; +pub use self::data::{TxForEscalation, TxStatus, UnsentTx}; // Statically link in migration files static MIGRATOR: Migrator = sqlx::migrate!("db/migrations"); @@ -122,12 +122,12 @@ impl Database { ) -> eyre::Result> { Ok(sqlx::query_as( r#" - SELECT t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id - FROM transactions t - LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + SELECT t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id + FROM transactions t + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) INNER JOIN relayers r ON (t.relayer_id = r.id) - WHERE s.tx_id IS NULL - AND (t.nonce - r.current_nonce < $1); + WHERE s.tx_id IS NULL + AND (t.nonce - r.current_nonce < $1); "#, ) .bind(max_inflight_txs as i64) @@ -154,27 +154,28 @@ impl Database { sqlx::query( r#" - INSERT INTO sent_transactions (tx_id, initial_max_fee_per_gas, initial_max_priority_fee_per_gas) - VALUES ($1, $2, $3) - "# + INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) + VALUES ($1, $2, $3, $4) + "#, ) .bind(tx_id) + .bind(tx_hash.as_bytes()) .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) - .execute(tx.as_mut()).await?; + .execute(tx.as_mut()) + .await?; sqlx::query( r#" - INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) + INSERT INTO sent_transactions (tx_id, initial_max_fee_per_gas, initial_max_priority_fee_per_gas, valid_tx_hash) VALUES ($1, $2, $3, $4) - "#, + "# ) .bind(tx_id) - .bind(tx_hash.as_bytes()) .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) - .execute(tx.as_mut()) - .await?; + .bind(tx_hash.as_bytes()) + .execute(tx.as_mut()).await?; tx.commit().await?; @@ -190,14 +191,12 @@ impl Database { SELECT fee_estimate FROM blocks WHERE chain_id = $1 - AND status = $2 AND fee_estimate IS NOT NULL ORDER BY block_number DESC LIMIT 1 "#, ) .bind(chain_id as i64) - .bind(BlockTxStatus::Mined) .fetch_optional(&self.pool) .await?; @@ -208,31 +207,6 @@ impl Database { Ok(item.map(|json_fee_estimate| json_fee_estimate.0)) } - pub async fn get_next_block_numbers(&self) -> eyre::Result> { - Ok(sqlx::query_as( - r#" - WITH LatestBlocks AS ( - SELECT - block_number, - chain_id, - timestamp, - ROW_NUMBER() OVER (PARTITION BY chain_id ORDER BY block_number DESC) AS rn - FROM blocks - WHERE status = $1 - ) - SELECT - block_number + 1 AS next_block_number, - chain_id, - timestamp as prev_block_timestamp - FROM LatestBlocks - WHERE rn = 1 - "#, - ) - .bind(BlockTxStatus::Mined) - .fetch_all(&self.pool) - .await?) - } - pub async fn has_blocks_for_chain( &self, chain_id: u64, @@ -260,32 +234,47 @@ impl Database { timestamp: DateTime, txs: &[H256], fee_estimates: Option<&FeesEstimate>, - status: BlockTxStatus, ) -> eyre::Result<()> { let mut db_tx = self.pool.begin().await?; - let (block_id,): (i64,) = sqlx::query_as( + // Prune block txs + sqlx::query( + r#" + DELETE + FROM block_txs + WHERE block_number = $1 + AND chain_id = $2 + "#, + ) + .bind(block_number as i64) + .bind(chain_id as i64) + .execute(db_tx.as_mut()) + .await?; + + // Insert new block or update + sqlx::query( r#" - INSERT INTO blocks (block_number, chain_id, timestamp, fee_estimate, status) - VALUES ($1, $2, $3, $4, $5) - RETURNING id + INSERT INTO blocks (block_number, chain_id, timestamp, fee_estimate) + VALUES ($1, $2, $3, $4) + ON CONFLICT (block_number, chain_id) DO UPDATE + SET timestamp = $3, + fee_estimate = $4 "#, ) .bind(block_number as i64) .bind(chain_id as i64) .bind(timestamp) .bind(fee_estimates.map(sqlx::types::Json)) - .bind(status) - .fetch_one(db_tx.as_mut()) + .execute(db_tx.as_mut()) .await?; let txs: Vec<_> = txs.iter().map(|tx| H256Wrapper(*tx)).collect(); sqlx::query( r#" - INSERT INTO block_txs (block_id, tx_hash) - SELECT $1, unnested.tx_hash - FROM UNNEST($2::BYTEA[]) AS unnested(tx_hash) + INSERT INTO block_txs (block_number, chain_id, tx_hash) + SELECT $1, $2, unnested.tx_hash + FROM UNNEST($3::BYTEA[]) AS unnested(tx_hash) WHERE EXISTS ( SELECT 1 FROM tx_hashes @@ -293,7 +282,8 @@ impl Database { ); "#, ) - .bind(block_id) + .bind(block_number as i64) + .bind(chain_id as i64) .bind(&txs[..]) .execute(db_tx.as_mut()) .await?; @@ -303,29 +293,202 @@ impl Database { Ok(()) } - pub async fn update_transactions( + pub async fn handle_soft_reorgs(&self) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + // Fetch txs which have valid tx hash different than what is actually mined + let items: Vec<(String, H256Wrapper)> = sqlx::query_as( + r#" + SELECT t.id, h.tx_hash + FROM transactions t + JOIN sent_transactions s ON t.id = s.tx_id + JOIN tx_hashes h ON t.id = h.tx_id + JOIN block_txs bt ON h.tx_hash = bt.tx_hash + WHERE h.tx_hash <> s.valid_tx_hash + AND s.status = $1 + "#, + ) + .bind(TxStatus::Mined) + .fetch_all(tx.as_mut()) + .await?; + + let (tx_ids, tx_hashes): (Vec<_>, Vec<_>) = items.into_iter().unzip(); + + sqlx::query( + r#" + UPDATE sent_transactions s + SET valid_tx_hash = mined.tx_hash + FROM transactions t, + UNNEST($1::TEXT[], $2::BYTEA[]) AS mined(tx_id, tx_hash) + WHERE t.id = mined.tx_id + AND t.id = s.tx_id + "#, + ) + .bind(&tx_ids) + .bind(&tx_hashes) + .execute(tx.as_mut()) + .await?; + + tx.commit().await?; + + Ok(()) + } + + pub async fn handle_hard_reorgs(&self) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + // Fetch txs which are marked as mined + // but none of the associated tx hashes are present in block txs + let items: Vec<(String,)> = sqlx::query_as( + r#" + WITH fdsa AS ( + SELECT t.id, h.tx_hash, bt.chain_id + FROM transactions t + JOIN sent_transactions s ON t.id = s.tx_id + JOIN tx_hashes h ON t.id = h.tx_id + LEFT JOIN block_txs bt ON h.tx_hash = bt.tx_hash + WHERE s.status = $1 + ) + SELECT t.id + FROM fdsa t + GROUP BY t.id + HAVING COUNT(t.chain_id) = 0 + "#, + ) + .bind(TxStatus::Mined) + .fetch_all(tx.as_mut()) + .await?; + + let tx_ids: Vec<_> = items.into_iter().map(|(x,)| x).collect(); + + // Set status to pending + // and set valid tx hash to the latest tx hash + sqlx::query( + r#" + UPDATE sent_transactions s + SET status = $1, + valid_tx_hash = ( + SELECT tx_hash + FROM tx_hashes h + WHERE h.tx_id = s.tx_id + ORDER BY created_at DESC + LIMIT 1 + ), + mined_at = NULL + FROM transactions t, UNNEST($2::TEXT[]) AS reorged(tx_id) + WHERE t.id = reorged.tx_id + AND t.id = s.tx_id + "#, + ) + .bind(TxStatus::Pending) + .bind(&tx_ids) + .execute(tx.as_mut()) + .await?; + + tx.commit().await?; + + Ok(()) + } + + pub async fn mine_txs(&self, chain_id: u64) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + // Fetch txs which are marked as pending but have an associated tx + // present in in one of the block txs + let items: Vec<(String, H256Wrapper, DateTime)> = sqlx::query_as( + r#" + SELECT t.id, h.tx_hash, b.timestamp + FROM transactions t + JOIN sent_transactions s ON t.id = s.tx_id + JOIN tx_hashes h ON t.id = h.tx_id + JOIN block_txs bt ON h.tx_hash = bt.tx_hash + JOIN blocks b ON bt.block_number = b.block_number AND bt.chain_id = b.chain_id + WHERE s.status = $1 + AND b.chain_id = $2 + "#, + ) + .bind(TxStatus::Pending) + .bind(chain_id as i64) + .fetch_all(tx.as_mut()) + .await?; + + let mut tx_ids = Vec::new(); + let mut tx_hashes = Vec::new(); + let mut timestamps = Vec::new(); + + for (tx_id, tx_hash, timestamp) in items { + tx_ids.push(tx_id); + tx_hashes.push(tx_hash); + timestamps.push(timestamp); + } + + sqlx::query( + r#" + UPDATE sent_transactions s + SET status = $1, + valid_tx_hash = mined.tx_hash, + mined_at = mined.timestamp + FROM transactions t, + UNNEST($2::TEXT[], $3::BYTEA[], $4::TIMESTAMPTZ[]) AS mined(tx_id, tx_hash, timestamp) + WHERE t.id = mined.tx_id + AND t.id = s.tx_id + "#, + ) + .bind(TxStatus::Mined) + .bind(&tx_ids) + .bind(&tx_hashes) + .bind(×tamps) + .execute(tx.as_mut()) + .await?; + + tx.commit().await?; + + Ok(()) + } + + pub async fn finalize_txs( &self, - status: BlockTxStatus, + finalization_timestmap: DateTime, ) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + // Fetch txs which are marked as mined, but the associated valid tx hash + // is present in a block which is older than the given timestamp + let items: Vec<(String,)> = sqlx::query_as( + r#" + SELECT s.tx_id + FROM sent_transactions s + JOIN tx_hashes h ON s.valid_tx_hash = h.tx_hash + JOIN block_txs bt ON h.tx_hash = bt.tx_hash + JOIN blocks b ON bt.block_number = b.block_number AND bt.chain_id = b.chain_id + WHERE s.status = $1 + AND b.timestamp < $2 + "#, + ) + .bind(TxStatus::Mined) + .bind(finalization_timestmap) + .fetch_all(tx.as_mut()) + .await?; + + let tx_ids: Vec<_> = items.into_iter().map(|(x,)| x).collect(); + + // Set status to finalized sqlx::query( r#" - UPDATE tx_hashes h + UPDATE sent_transactions s SET status = $1 - FROM transactions t, block_txs bt, blocks b, relayers r - WHERE t.id = h.tx_id - AND b.id = bt.block_id - AND h.tx_hash = bt.tx_hash - AND r.chain_id = b.chain_id - AND r.id = t.relayer_id - AND h.status = $2 - AND b.status = $1 + FROM transactions t, UNNEST($2::TEXT[]) AS finalized(tx_id) + WHERE t.id = finalized.tx_id + AND t.id = s.tx_id "#, ) - .bind(status) - .bind(status.previous()) - .execute(&self.pool) + .bind(TxStatus::Finalized) + .bind(&tx_ids) + .execute(tx.as_mut()) .await?; + tx.commit().await?; + Ok(()) } @@ -343,12 +506,12 @@ impl Database { JOIN tx_hashes h ON t.id = h.tx_id JOIN relayers r ON t.relayer_id = r.id WHERE now() - h.created_at > $1 - AND h.status = $2 + AND s.status = $2 AND NOT h.escalated "#, ) .bind(escalation_interval) - .bind(BlockTxStatus::Pending) + .bind(TxStatus::Pending) .fetch_all(&self.pool) .await?) } @@ -373,17 +536,6 @@ impl Database { .execute(tx.as_mut()) .await?; - sqlx::query( - r#" - UPDATE sent_transactions - SET escalation_count = escalation_count + 1 - WHERE tx_id = $1 - "#, - ) - .bind(tx_id) - .execute(tx.as_mut()) - .await?; - let mut max_fee_per_gas_bytes = [0u8; 32]; max_fee_per_gas.to_big_endian(&mut max_fee_per_gas_bytes); @@ -404,6 +556,19 @@ impl Database { .execute(tx.as_mut()) .await?; + sqlx::query( + r#" + UPDATE sent_transactions + SET escalation_count = escalation_count + 1, + valid_tx_hash = $2 + WHERE tx_id = $1 + "#, + ) + .bind(tx_id) + .bind(tx_hash.as_bytes()) + .execute(tx.as_mut()) + .await?; + tx.commit().await?; Ok(()) @@ -416,12 +581,11 @@ impl Database { Ok(sqlx::query_as( r#" SELECT t.id as tx_id, t.tx_to as to, t.data, t.value, t.gas_limit, t.nonce, - h.tx_hash, h.status + h.tx_hash, s.status FROM transactions t - LEFT JOIN tx_hashes h ON t.id = h.tx_id + LEFT JOIN sent_transactions s ON t.id = s.tx_id + LEFT JOIN tx_hashes h ON s.valid_tx_hash = h.tx_hash WHERE t.id = $1 - ORDER BY h.created_at DESC, h.status DESC - LIMIT 1 "#, ) .bind(tx_id) @@ -475,29 +639,74 @@ impl Database { &self, timestamp: DateTime, ) -> eyre::Result<()> { - let mut tx = self.pool.begin().await?; + sqlx::query( + r#" + DELETE FROM blocks + WHERE timestamp < $1 + "#, + ) + .bind(timestamp) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn prune_txs( + &self, + timestamp: DateTime, + ) -> eyre::Result<()> { sqlx::query( r#" - DELETE FROM block_txs - WHERE block_id IN ( - SELECT id - FROM blocks - WHERE timestamp < $1 - ) + DELETE + FROM transactions t + USING sent_transactions s + WHERE t.id = s.tx_id + AND s.mined_at < $1 + AND s.status = $2 "#, ) .bind(timestamp) + .bind(TxStatus::Finalized) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn create_network( + &self, + chain_id: u64, + name: &str, + http_rpc: &str, + ws_rpc: &str, + ) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + sqlx::query( + r#" + INSERT INTO networks (chain_id, name) + VALUES ($1, $2) + "#, + ) + .bind(chain_id as i64) + .bind(name) .execute(tx.as_mut()) .await?; sqlx::query( r#" - DELETE FROM blocks - WHERE timestamp < $1 + INSERT INTO rpcs (chain_id, url, kind) + VALUES + ($1, $2, $3), + ($1, $4, $5) "#, ) - .bind(timestamp) + .bind(chain_id as i64) + .bind(http_rpc) + .bind(RpcKind::Http) + .bind(ws_rpc) + .bind(RpcKind::Ws) .execute(tx.as_mut()) .await?; @@ -505,12 +714,38 @@ impl Database { Ok(()) } + + pub async fn get_network_rpc( + &self, + chain_id: u64, + rpc_kind: RpcKind, + ) -> eyre::Result { + let row: (String,) = sqlx::query_as( + r#" + SELECT url + FROM rpcs + WHERE chain_id = $1 + AND kind = $2 + "#, + ) + .bind(chain_id as i64) + .bind(rpc_kind) + .fetch_one(&self.pool) + .await?; + + Ok(row.0) + } } #[cfg(test)] mod tests { use chrono::NaiveDate; + use eyre::ContextCompat; use postgres_docker_utils::DockerContainerGuard; + use tracing::level_filters::LevelFilter; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + use tracing_subscriber::EnvFilter; use super::*; @@ -528,6 +763,21 @@ mod tests { Ok((db, db_container)) } + async fn full_update( + db: &Database, + chain_id: u64, + finalization_timestamp: DateTime, + ) -> eyre::Result<()> { + db.mine_txs(chain_id).await?; + + db.handle_soft_reorgs().await?; + db.handle_hard_reorgs().await?; + + db.finalize_txs(finalization_timestamp).await?; + + Ok(()) + } + #[tokio::test] async fn basic() -> eyre::Result<()> { let (_db, _db_container) = setup_db().await?; @@ -559,31 +809,188 @@ mod tests { H256::from_low_u64_be(3), ]; - db.save_block( - 1, - 1, - block_timestamp.clone(), - &tx_hashes, - None, - BlockTxStatus::Mined, + db.save_block(1, 1, block_timestamp, &tx_hashes, None) + .await?; + + assert!(db.has_blocks_for_chain(1).await?, "Should have blocks"); + + db.prune_blocks(prune_timestamp).await?; + + assert!(!db.has_blocks_for_chain(1).await?, "Should not have blocks"); + + Ok(()) + } + + #[tokio::test] + async fn tx_lifecycle() -> eyre::Result<()> { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().pretty().compact()) + .with( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + // Logging from fake_rpc can get very messy so we set it to warn only + .parse_lossy("info"), + ) + .init(); + + let (db, _db_container) = setup_db().await?; + + let chain_id = 123; + let network_name = "network_name"; + let http_rpc = "http_rpc"; + let ws_rpc = "ws_rpc"; + + db.create_network(chain_id, network_name, http_rpc, ws_rpc) + .await?; + + let relayer_id = "relayer_id"; + let relayer_name = "relayer_name"; + let key_id = "key_id"; + let relayer_address = Address::from_low_u64_be(1); + + db.create_relayer( + relayer_id, + relayer_name, + chain_id, + key_id, + relayer_address, ) .await?; - let next_blocks = db.get_next_block_numbers().await?; - let expected = vec![NextBlock { - next_block_number: 2, - chain_id: 1, - prev_block_timestamp: block_timestamp, - }]; + let tx_id = "tx_id"; + let to = Address::from_low_u64_be(1); + let data: &[u8] = &[]; + let value = U256::from(0); + let gas_limit = U256::from(0); + + let tx = db.read_tx(tx_id).await?; + assert!(tx.is_none(), "Tx has not been sent yet"); + + db.create_transaction(tx_id, to, data, value, gas_limit, relayer_id) + .await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + assert_eq!(tx.tx_id, tx_id); + assert_eq!(tx.to.0, to); + assert_eq!(tx.data, data); + assert_eq!(tx.value.0, value); + assert_eq!(tx.gas_limit.0, gas_limit); + assert_eq!(tx.nonce, 0); + assert_eq!(tx.tx_hash, None); + + let tx_hash_1 = H256::from_low_u64_be(1); + let tx_hash_2 = H256::from_low_u64_be(2); + let initial_max_fee_per_gas = U256::from(1); + let initial_max_priority_fee_per_gas = U256::from(1); + + db.insert_tx_broadcast( + tx_id, + tx_hash_1, + initial_max_fee_per_gas, + initial_max_priority_fee_per_gas, + ) + .await?; - assert_eq!(next_blocks, expected, "Should return next block"); + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; - db.prune_blocks(prune_timestamp).await?; + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_1); + assert_eq!(tx.status, Some(TxStatus::Pending)); - let next_blocks = db.get_next_block_numbers().await?; + db.escalate_tx( + tx_id, + tx_hash_2, + initial_max_fee_per_gas, + initial_max_priority_fee_per_gas, + ) + .await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + // By default we take the latest tx + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_2); + assert_eq!(tx.status, Some(TxStatus::Pending)); + + // Do an update + let finalized_timestamp = ymd_hms(2023, 11, 23, 10, 32, 3); + full_update(&db, chain_id, finalized_timestamp).await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + // Nothing should have changed + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_2); + assert_eq!(tx.status, Some(TxStatus::Pending)); + + // Save block + let block_number = 1; + let timestamp = ymd_hms(2023, 11, 23, 12, 32, 2); + let txs = &[tx_hash_1]; + + db.save_block(block_number, chain_id, timestamp, txs, None) + .await?; + + full_update(&db, chain_id, finalized_timestamp).await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_1); + assert_eq!(tx.status, Some(TxStatus::Mined)); + + // Reorg + let txs = &[tx_hash_2]; + + db.save_block(block_number, chain_id, timestamp, txs, None) + .await?; + + full_update(&db, chain_id, finalized_timestamp).await?; - assert!(next_blocks.is_empty(), "Should return no blocks"); + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_2); + assert_eq!(tx.status, Some(TxStatus::Mined)); + + // Destructive reorg + let txs = &[]; + + db.save_block(block_number, chain_id, timestamp, txs, None) + .await?; + + full_update(&db, chain_id, finalized_timestamp).await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_2); + assert_eq!(tx.status, Some(TxStatus::Pending)); + + // Finalization + let txs = &[tx_hash_2]; + + db.save_block(block_number, chain_id, timestamp, txs, None) + .await?; + + let finalized_timestamp = ymd_hms(2023, 11, 23, 22, 0, 0); + full_update(&db, chain_id, finalized_timestamp).await?; + + let tx = db.read_tx(tx_id).await?.context("Missing tx")?; + + assert_eq!(tx.tx_hash.unwrap().0, tx_hash_2); + assert_eq!(tx.status, Some(TxStatus::Finalized)); Ok(()) } + + fn ymd_hms( + year: i32, + month: u32, + day: u32, + hour: u32, + minute: u32, + second: u32, + ) -> DateTime { + NaiveDate::from_ymd_opt(year, month, day) + .unwrap() + .and_hms_opt(hour, minute, second) + .unwrap() + .and_utc() + } } diff --git a/src/db/data.rs b/src/db/data.rs index 9666226..d0a2563 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -1,4 +1,3 @@ -use chrono::{DateTime, Utc}; use ethers::types::{Address, H256, U256}; use serde::{Deserialize, Serialize}; use sqlx::database::{HasArguments, HasValueRef}; @@ -50,24 +49,15 @@ pub struct ReadTxData { // Sent tx data pub tx_hash: Option, - pub status: Option, + pub status: Option, } -#[derive(Debug, Clone, FromRow, PartialEq, Eq)] -pub struct NextBlock { - #[sqlx(try_from = "i64")] - pub next_block_number: u64, - #[sqlx(try_from = "i64")] - pub chain_id: u64, - pub prev_block_timestamp: DateTime, -} - -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct AddressWrapper(pub Address); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct U256Wrapper(pub U256); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct H256Wrapper(pub H256); impl<'r, DB> sqlx::Decode<'r, DB> for AddressWrapper @@ -180,15 +170,15 @@ where Debug, Clone, Serialize, Deserialize, Copy, PartialEq, Eq, sqlx::Type, )] #[sqlx(rename_all = "camelCase")] -#[sqlx(type_name = "block_tx_status")] +#[sqlx(type_name = "tx_status")] #[serde(rename_all = "camelCase")] -pub enum BlockTxStatus { - Pending = 0, - Mined = 1, - Finalized = 2, +pub enum TxStatus { + Pending, + Mined, + Finalized, } -impl BlockTxStatus { +impl TxStatus { pub fn previous(self) -> Self { match self { Self::Pending => Self::Pending, @@ -197,3 +187,14 @@ impl BlockTxStatus { } } } + +#[derive( + Debug, Clone, Serialize, Deserialize, Copy, PartialEq, Eq, sqlx::Type, +)] +#[sqlx(rename_all = "camelCase")] +#[sqlx(type_name = "rpc_kind")] +#[serde(rename_all = "camelCase")] +pub enum RpcKind { + Http, + Ws, +} diff --git a/src/keys.rs b/src/keys.rs index 62f7155..9814aa8 100644 --- a/src/keys.rs +++ b/src/keys.rs @@ -1,7 +1,7 @@ use aws_sdk_kms::types::{KeySpec, KeyUsageType}; use aws_types::region::Region; use ethers::core::k256::ecdsa::SigningKey; -use ethers_signers::Wallet; +use ethers::signers::Wallet; use eyre::{Context, ContextCompat}; pub use universal_signer::UniversalSigner; @@ -110,7 +110,7 @@ impl KeysSource for LocalKeys { #[cfg(test)] mod tests { - use ethers_signers::Signer; + use ethers::signers::Signer; use super::*; diff --git a/src/keys/universal_signer.rs b/src/keys/universal_signer.rs index 226cbcf..6bfd718 100644 --- a/src/keys/universal_signer.rs +++ b/src/keys/universal_signer.rs @@ -2,8 +2,7 @@ use ethers::core::k256::ecdsa::SigningKey; use ethers::core::types::transaction::eip2718::TypedTransaction; use ethers::core::types::transaction::eip712::Eip712; use ethers::core::types::{Address, Signature as EthSig}; -use ethers::signers::Signer; -use ethers_signers::{Wallet, WalletError}; +use ethers::signers::{Signer, Wallet, WalletError}; use thiserror::Error; use crate::aws::ethers_signer::AwsSigner; diff --git a/src/lib.rs b/src/lib.rs index f64234f..393030c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,12 @@ pub mod app; pub mod aws; +pub mod broadcast_utils; +pub mod client; pub mod config; pub mod db; +pub mod keys; +pub mod serde_utils; pub mod server; pub mod service; pub mod task_runner; pub mod tasks; - -pub mod broadcast_utils; -pub mod client; -pub mod keys; -pub mod serde_utils; diff --git a/src/server.rs b/src/server.rs index cfe06ee..bb80b7f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -5,7 +5,7 @@ use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::{get, post, IntoMakeService}; use axum::{Router, TypedHeader}; -use ethers_signers::Signer; +use ethers::signers::Signer; use eyre::Result; use hyper::server::conn::AddrIncoming; use middleware::AuthorizedRelayer; @@ -19,6 +19,7 @@ use crate::app::App; pub mod data; mod middleware; +pub mod routes; #[derive(Debug, Error)] pub enum ApiError { @@ -165,12 +166,16 @@ pub async fn spawn_server( .route("/:relayer_id", get(get_relayer)) .with_state(app.clone()); - // let network_routes = Router::new() - // .route("/"); + let network_routes = Router::new() + // .route("/", get(routes::network::get_networks)) + // .route("/:chain_id", get(routes::network::get_network)) + .route("/:chain_id", post(routes::network::create_network)) + .with_state(app.clone()); let router = Router::new() .nest("/1/tx", tx_routes) .nest("/1/relayer", relayer_routes) + .nest("/1/network", network_routes) .layer(tower_http::trace::TraceLayer::new_for_http()) .layer(axum::middleware::from_fn(middleware::log_response)); diff --git a/src/server/data.rs b/src/server/data.rs index 22b2485..44fc65b 100644 --- a/src/server/data.rs +++ b/src/server/data.rs @@ -1,7 +1,7 @@ use ethers::types::{Address, Bytes, H256, U256}; use serde::{Deserialize, Serialize}; -use crate::db::BlockTxStatus; +use crate::db::TxStatus; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -41,7 +41,7 @@ pub struct GetTxResponse { #[serde(default, skip_serializing_if = "Option::is_none")] pub tx_hash: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, + pub status: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/server/routes.rs b/src/server/routes.rs new file mode 100644 index 0000000..a61610b --- /dev/null +++ b/src/server/routes.rs @@ -0,0 +1 @@ +pub mod network; diff --git a/src/server/routes/network.rs b/src/server/routes/network.rs new file mode 100644 index 0000000..e43e219 --- /dev/null +++ b/src/server/routes/network.rs @@ -0,0 +1,74 @@ +use std::sync::Arc; + +use axum::extract::{Json, Path, State}; +use eyre::Result; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::app::App; +use crate::server::ApiError; +use crate::task_runner::TaskRunner; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewNetworkInfo { + pub name: String, + pub http_rpc: String, + pub ws_rpc: String, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NetworkInfo { + pub chain_id: u64, + pub name: String, + pub http_rpc: String, + pub ws_rpc: String, +} + +pub async fn create_network( + State(app): State>, + Path(chain_id): Path, + Json(network): Json, +) -> Result<(), ApiError> { + let http_url: Url = network.http_rpc.parse().map_err(|err| { + tracing::error!("Invalid http rpc url: {}", err); + ApiError::InvalidFormat + })?; + + let ws_url: Url = network.ws_rpc.parse().map_err(|err| { + tracing::error!("Invalid ws rpc url: {}", err); + ApiError::InvalidFormat + })?; + + app.db + .create_network( + chain_id, + &network.name, + http_url.as_str(), + ws_url.as_str(), + ) + .await?; + + let task_runner = TaskRunner::new(app.clone()); + + task_runner.add_task(format!("index_block_{}", chain_id), move |app| { + crate::tasks::index::index_chain(app, chain_id) + }); + + Ok(()) +} + +pub async fn _get_network( + State(_app): State>, + Path(_chain_id): Path, +) -> &'static str { + "Hello, World!" +} + +pub async fn _get_networks( + State(_app): State>, + Path(_chain_id): Path, +) -> &'static str { + "Hello, World!" +} diff --git a/src/service.rs b/src/service.rs index 118dbb8..b901beb 100644 --- a/src/service.rs +++ b/src/service.rs @@ -20,9 +20,12 @@ impl Service { let task_runner = TaskRunner::new(app.clone()); task_runner.add_task("Broadcast transactions", tasks::broadcast_txs); - task_runner.add_task("Index transactions", tasks::index_blocks); task_runner.add_task("Escalate transactions", tasks::escalate_txs); task_runner.add_task("Prune blocks", tasks::prune_blocks); + task_runner.add_task("Prune transactions", tasks::prune_txs); + task_runner.add_task("Finalize transactions", tasks::finalize_txs); + task_runner.add_task("Handle soft reorgs", tasks::handle_soft_reorgs); + task_runner.add_task("Handle hard reorgs", tasks::handle_hard_reorgs); let server = crate::server::spawn_server(app.clone()).await?; let local_addr = server.local_addr(); diff --git a/src/task_runner.rs b/src/task_runner.rs index 4108ba3..e2d8e4d 100644 --- a/src/task_runner.rs +++ b/src/task_runner.rs @@ -52,6 +52,41 @@ where } }); } + + pub fn add_task_with_args(&self, label: S, task: C, args: A) + where + A: Clone + Send + 'static, + S: ToString, + C: Fn(Arc, A) -> F + Send + Sync + 'static, + F: Future> + Send + 'static, + { + let app = self.app.clone(); + let label = label.to_string(); + + tokio::spawn(async move { + let mut failures = vec![]; + + loop { + tracing::info!(label, "Running task"); + + let result = task(app.clone(), args.clone()).await; + + if let Err(err) = result { + tracing::error!(label, error = ?err, "Task failed"); + + failures.push(Instant::now()); + let backoff = determine_backoff(&failures); + + tokio::time::sleep(backoff).await; + + prune_failures(&mut failures); + } else { + tracing::info!(label, "Task finished"); + break; + } + } + }); + } } fn determine_backoff(failures: &[Instant]) -> Duration { diff --git a/src/tasks.rs b/src/tasks.rs index 8f99ac1..e15f411 100644 --- a/src/tasks.rs +++ b/src/tasks.rs @@ -1,9 +1,13 @@ pub mod broadcast; pub mod escalate; +pub mod finalize; +pub mod handle_reorgs; pub mod index; pub mod prune; pub use self::broadcast::broadcast_txs; pub use self::escalate::escalate_txs; -pub use self::index::index_blocks; -pub use self::prune::prune_blocks; +pub use self::finalize::finalize_txs; +pub use self::handle_reorgs::{handle_hard_reorgs, handle_soft_reorgs}; +pub use self::index::index_chain; +pub use self::prune::{prune_blocks, prune_txs}; diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index b7c7f21..d6e1d3d 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -79,6 +79,6 @@ pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { .await?; } - tokio::time::sleep(Duration::from_secs(5)).await; + tokio::time::sleep(Duration::from_secs(1)).await; } } diff --git a/src/tasks/finalize.rs b/src/tasks/finalize.rs new file mode 100644 index 0000000..4695153 --- /dev/null +++ b/src/tasks/finalize.rs @@ -0,0 +1,25 @@ +use std::sync::Arc; +use std::time::Duration; + +use crate::app::App; + +const TIME_BETWEEN_FINALIZATIONS_SECONDS: i64 = 60; + +pub async fn finalize_txs(app: Arc) -> eyre::Result<()> { + loop { + let finalization_timestmap = + chrono::Utc::now() - chrono::Duration::seconds(60 * 60); + + tracing::info!( + "Finalizing txs mined before {}", + finalization_timestmap + ); + + app.db.finalize_txs(finalization_timestmap).await?; + + tokio::time::sleep(Duration::from_secs( + TIME_BETWEEN_FINALIZATIONS_SECONDS as u64, + )) + .await; + } +} diff --git a/src/tasks/handle_reorgs.rs b/src/tasks/handle_reorgs.rs new file mode 100644 index 0000000..699ef5c --- /dev/null +++ b/src/tasks/handle_reorgs.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; +use std::time::Duration; + +use crate::app::App; + +// TODO: Make this configurable +const TIME_BETWEEN_HARD_REORGS_SECONDS: i64 = 60 * 60; // Once every hour +const TIME_BETWEEN_SOFT_REORGS_SECONDS: i64 = 60; // Once every minute + +pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { + loop { + tracing::info!("Handling hard reorgs"); + + app.db.handle_hard_reorgs().await?; + + tokio::time::sleep(Duration::from_secs( + TIME_BETWEEN_HARD_REORGS_SECONDS as u64, + )) + .await; + } +} + +pub async fn handle_soft_reorgs(app: Arc) -> eyre::Result<()> { + loop { + tracing::info!("Handling soft reorgs"); + + app.db.handle_soft_reorgs().await?; + + tokio::time::sleep(Duration::from_secs( + TIME_BETWEEN_SOFT_REORGS_SECONDS as u64, + )) + .await; + } +} diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 92333ce..eb67680 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -1,10 +1,9 @@ use std::sync::Arc; -use std::time::Duration; use chrono::{DateTime, Utc}; use ethers::providers::{Http, Middleware, Provider}; -use ethers::types::{Block, BlockNumber, H256, U256}; -use eyre::ContextCompat; +use ethers::types::BlockNumber; +use eyre::{Context, ContextCompat}; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -12,105 +11,62 @@ use crate::app::App; use crate::broadcast_utils::gas_estimation::{ estimate_percentile_fees, FeesEstimate, }; -use crate::db::data::NextBlock; -use crate::db::BlockTxStatus; const BLOCK_FEE_HISTORY_SIZE: usize = 10; -const TRAILING_BLOCK_OFFSET: u64 = 5; const FEE_PERCENTILES: [f64; 5] = [5.0, 25.0, 50.0, 75.0, 95.0]; -pub async fn index_blocks(app: Arc) -> eyre::Result<()> { +pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { loop { - let next_block_numbers = app.db.get_next_block_numbers().await?; + let ws_rpc = app.fetch_ws_provider(chain_id).await?; + let rpc = app.fetch_http_provider(chain_id).await?; - for next_block in next_block_numbers { - update_block(app.clone(), next_block).await?; - } - - let (update_mined, update_finalized) = tokio::join!( - app.db.update_transactions(BlockTxStatus::Mined), - app.db.update_transactions(BlockTxStatus::Finalized) - ); + let mut blocks_stream = ws_rpc.subscribe_blocks().await?; - update_mined?; - update_finalized?; + while let Some(block) = blocks_stream.next().await { + let block_number = + block.number.context("Missing block number")?.as_u64(); - tokio::time::sleep(Duration::from_secs(2)).await; - } -} + tracing::info!(block_number, "Indexing block"); -async fn update_block( - app: Arc, - next_block: NextBlock, -) -> eyre::Result<()> { - let chain_id = U256::from(next_block.chain_id); - let rpc = app - .rpcs - .get(&chain_id) - .context("Missing RPC for chain id")?; - - let block = - fetch_block_with_fee_estimates(rpc, next_block.next_block_number) - .await?; - - let Some((block, fee_estimates)) = block else { - return Ok(()); - }; - - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = - DateTime::::from_timestamp(block_timestamp_seconds as i64, 0) + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = DateTime::::from_timestamp( + block_timestamp_seconds as i64, + 0, + ) .context("Invalid timestamp")?; - app.db - .save_block( - next_block.next_block_number, - chain_id.as_u64(), - block_timestamp, - &block.transactions, - Some(&fee_estimates), - BlockTxStatus::Mined, - ) - .await?; + // TODO: We don't need to do this for every block for a given chain + // Add a separate task to do this periodically for the latest block + let fee_estimates = fetch_block_fee_estimates(&rpc, block_number) + .await + .context("Failed to fetch fee estimates")?; - let relayer_addresses = - app.db.fetch_relayer_addresses(chain_id.as_u64()).await?; - - update_relayer_nonces(relayer_addresses, &app, rpc, chain_id).await?; - - if next_block.next_block_number > TRAILING_BLOCK_OFFSET { - let block = fetch_block( - rpc, - next_block.next_block_number - TRAILING_BLOCK_OFFSET, - ) - .await? - .context("Missing trailing block")?; - - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = - DateTime::::from_timestamp(block_timestamp_seconds as i64, 0) - .context("Invalid timestamp")?; - - app.db - .save_block( - next_block.next_block_number, - chain_id.as_u64(), - block_timestamp, - &block.transactions, - None, - BlockTxStatus::Finalized, - ) - .await?; - } + app.db + .save_block( + block.number.unwrap().as_u64(), + chain_id, + block_timestamp, + &block.transactions, + Some(&fee_estimates), + ) + .await?; - Ok(()) + app.db.mine_txs(chain_id).await?; + + let relayer_addresses = + app.db.fetch_relayer_addresses(chain_id).await?; + + update_relayer_nonces(relayer_addresses, &app, &rpc, chain_id) + .await?; + } + } } async fn update_relayer_nonces( relayer_addresses: Vec, app: &Arc, rpc: &Provider, - chain_id: U256, + chain_id: u64, ) -> Result<(), eyre::Error> { let mut futures = FuturesUnordered::new(); @@ -121,9 +77,15 @@ async fn update_relayer_nonces( let tx_count = rpc.get_transaction_count(relayer_address, None).await?; + // tracing::info!( + // nonce = ?tx_count, + // ?relayer_address, + // "Updating relayer nonce" + // ); + app.db .update_relayer_nonce( - chain_id.as_u64(), + chain_id, relayer_address, tx_count.as_u64(), ) @@ -140,38 +102,17 @@ async fn update_relayer_nonces( Ok(()) } -pub async fn fetch_block_with_fee_estimates( +pub async fn fetch_block_fee_estimates( rpc: &Provider, block_id: impl Into, -) -> eyre::Result, FeesEstimate)>> { +) -> eyre::Result { let block_id = block_id.into(); - let block = rpc.get_block(block_id).await?; - - let Some(block) = block else { - return Ok(None); - }; - let fee_history = rpc .fee_history(BLOCK_FEE_HISTORY_SIZE, block_id, &FEE_PERCENTILES) .await?; let fee_estimates = estimate_percentile_fees(&fee_history)?; - Ok(Some((block, fee_estimates))) -} - -pub async fn fetch_block( - rpc: &Provider, - block_id: impl Into, -) -> eyre::Result>> { - let block_id = block_id.into(); - - let block = rpc.get_block(block_id).await?; - - let Some(block) = block else { - return Ok(None); - }; - - Ok(Some(block)) + Ok(fee_estimates) } diff --git a/src/tasks/prune.rs b/src/tasks/prune.rs index 7651cf0..c505eb0 100644 --- a/src/tasks/prune.rs +++ b/src/tasks/prune.rs @@ -5,7 +5,8 @@ use chrono::Utc; use crate::app::App; -const PRUNING_INTERVAL: Duration = Duration::from_secs(60); +const BLOCK_PRUNING_INTERVAL: Duration = Duration::from_secs(60); +const TX_PRUNING_INTERVAL: Duration = Duration::from_secs(60); const fn minutes(seconds: i64) -> i64 { seconds * 60 @@ -19,7 +20,11 @@ const fn days(seconds: i64) -> i64 { hours(seconds) * 24 } +// TODO: This should be a per network setting const BLOCK_PRUNE_AGE_SECONDS: i64 = days(7); +// NOTE: We must prune txs earlier than blocks +// as a missing block tx indicates a hard reorg +const TX_PRUNE_AGE_SECONDS: i64 = days(5); pub async fn prune_blocks(app: Arc) -> eyre::Result<()> { loop { @@ -30,6 +35,19 @@ pub async fn prune_blocks(app: Arc) -> eyre::Result<()> { app.db.prune_blocks(block_prune_timestamp).await?; - tokio::time::sleep(PRUNING_INTERVAL).await; + tokio::time::sleep(BLOCK_PRUNING_INTERVAL).await; + } +} + +pub async fn prune_txs(app: Arc) -> eyre::Result<()> { + loop { + let prune_age = chrono::Duration::seconds(TX_PRUNE_AGE_SECONDS); + let tx_prune_timestamp = Utc::now() - prune_age; + + tracing::info!(?tx_prune_timestamp, "Pruning txs"); + + app.db.prune_txs(tx_prune_timestamp).await?; + + tokio::time::sleep(TX_PRUNING_INTERVAL).await; } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 0b25bcc..e183663 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,15 +7,16 @@ use std::time::Duration; use ethers::core::k256::ecdsa::SigningKey; use ethers::middleware::SignerMiddleware; use ethers::providers::{Http, Middleware, Provider}; -use ethers::signers::LocalWallet; +use ethers::signers::{LocalWallet, Signer}; use ethers::types::{Address, Eip1559TransactionRequest, H160, U256}; -use ethers_signers::Signer; use fake_rpc::DoubleAnvil; use postgres_docker_utils::DockerContainerGuard; +use service::client::TxSitterClient; use service::config::{ - Config, DatabaseConfig, KeysConfig, LocalKeysConfig, RpcConfig, - ServerConfig, TxSitterConfig, + Config, DatabaseConfig, KeysConfig, LocalKeysConfig, ServerConfig, + TxSitterConfig, }; +use service::server::routes::network::NewNetworkInfo; use service::service::Service; use tokio::task::JoinHandle; use tracing::level_filters::LevelFilter; @@ -41,6 +42,7 @@ pub const DEFAULT_ANVIL_CHAIN_ID: u64 = 31337; pub struct DoubleAnvilHandle { pub double_anvil: Arc, + ws_addr: String, local_addr: SocketAddr, server_handle: JoinHandle>, } @@ -49,6 +51,10 @@ impl DoubleAnvilHandle { pub fn local_addr(&self) -> String { self.local_addr.to_string() } + + pub fn ws_addr(&self) -> String { + self.ws_addr.clone() + } } pub fn setup_tracing() { @@ -100,19 +106,22 @@ pub async fn setup_double_anvil() -> eyre::Result { .await? .await?; + let ws_addr = double_anvil.ws_endpoint().await; + Ok(DoubleAnvilHandle { double_anvil, + ws_addr, local_addr, server_handle, }) } pub async fn setup_service( - rpc_url: &str, + anvil_handle: &DoubleAnvilHandle, db_connection_url: &str, escalation_interval: Duration, -) -> eyre::Result { - println!("rpc_url.to_string() = {}", rpc_url); +) -> eyre::Result<(Service, TxSitterClient)> { + let rpc_url = anvil_handle.local_addr(); let config = Config { service: TxSitterConfig { @@ -125,9 +134,6 @@ pub async fn setup_service( )), disable_auth: true, }, - rpc: RpcConfig { - rpcs: vec![format!("http://{}", rpc_url.to_string())], - }, database: DatabaseConfig { connection_string: db_connection_url.to_string(), }, @@ -136,7 +142,21 @@ pub async fn setup_service( let service = Service::new(config).await?; - Ok(service) + let client = + TxSitterClient::new(format!("http://{}", service.local_addr())); + + client + .create_network( + DEFAULT_ANVIL_CHAIN_ID, + &NewNetworkInfo { + name: "Anvil".to_string(), + http_rpc: format!("http://{}", rpc_url), + ws_rpc: anvil_handle.ws_addr(), + }, + ) + .await?; + + Ok((service, client)) } pub async fn setup_middleware( diff --git a/tests/create_relayer.rs b/tests/create_relayer.rs index 4a0acf6..e8ab75e 100644 --- a/tests/create_relayer.rs +++ b/tests/create_relayer.rs @@ -15,22 +15,15 @@ async fn create_relayer() -> eyre::Result<()> { let (db_url, _db_container) = setup_db().await?; let double_anvil = setup_double_anvil().await?; - let service = - setup_service(&double_anvil.local_addr(), &db_url, ESCALATION_INTERVAL) - .await?; + let (_service, client) = + setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let addr = service.local_addr(); - - let response = reqwest::Client::new() - .post(&format!("http://{}/1/relayer/create", addr)) - .json(&CreateRelayerRequest { + let CreateRelayerResponse { .. } = client + .create_relayer(&CreateRelayerRequest { name: "Test relayer".to_string(), chain_id: DEFAULT_ANVIL_CHAIN_ID, }) - .send() .await?; - let _response: CreateRelayerResponse = response.json().await?; - Ok(()) } diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 6e2c23f..4ca70cd 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -4,7 +4,7 @@ use ethers::providers::Middleware; use ethers::types::{Eip1559TransactionRequest, U256}; use ethers::utils::parse_units; use service::server::data::{ - CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, SendTxResponse, + CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, }; mod common; @@ -20,23 +20,19 @@ async fn send_many_txs() -> eyre::Result<()> { let (db_url, _db_container) = setup_db().await?; let double_anvil = setup_double_anvil().await?; - let service = - setup_service(&double_anvil.local_addr(), &db_url, ESCALATION_INTERVAL) - .await?; - - let addr = service.local_addr(); + let (_service, client) = + setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let response = reqwest::Client::new() - .post(&format!("http://{}/1/relayer/create", addr)) - .json(&CreateRelayerRequest { + let CreateRelayerResponse { + address: relayer_address, + relayer_id, + } = client + .create_relayer(&CreateRelayerRequest { name: "Test relayer".to_string(), chain_id: DEFAULT_ANVIL_CHAIN_ID, }) - .send() .await?; - let response: CreateRelayerResponse = response.json().await?; - // Fund the relayer let middleware = setup_middleware( format!("http://{}", double_anvil.local_addr()), @@ -49,7 +45,7 @@ async fn send_many_txs() -> eyre::Result<()> { middleware .send_transaction( Eip1559TransactionRequest { - to: Some(response.address.into()), + to: Some(relayer_address.into()), value: Some(amount), ..Default::default() }, @@ -60,28 +56,23 @@ async fn send_many_txs() -> eyre::Result<()> { let provider = middleware.provider(); - let current_balance = provider.get_balance(response.address, None).await?; + let current_balance = provider.get_balance(relayer_address, None).await?; assert_eq!(current_balance, amount); // Send a transaction let value: U256 = parse_units("10", "ether")?.into(); let num_transfers = 10; - let relayer_id = response.relayer_id; for _ in 0..num_transfers { - let response = reqwest::Client::new() - .post(&format!("http://{}/1/tx/send", addr)) - .json(&SendTxRequest { + client + .send_tx(&SendTxRequest { relayer_id: relayer_id.clone(), to: ARBITRARY_ADDRESS, value, gas_limit: U256::from(21_000), ..Default::default() }) - .send() .await?; - - let _response: SendTxResponse = response.json().await?; } let expected_balance = value * num_transfers; diff --git a/tests/send_tx.rs b/tests/send_tx.rs index a524a33..da4ece5 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -4,7 +4,7 @@ use ethers::providers::Middleware; use ethers::types::{Eip1559TransactionRequest, U256}; use ethers::utils::parse_units; use service::server::data::{ - CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, SendTxResponse, + CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, }; mod common; @@ -20,23 +20,19 @@ async fn send_tx() -> eyre::Result<()> { let (db_url, _db_container) = setup_db().await?; let double_anvil = setup_double_anvil().await?; - let service = - setup_service(&double_anvil.local_addr(), &db_url, ESCALATION_INTERVAL) - .await?; + let (_service, client) = + setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let addr = service.local_addr(); - - let response = reqwest::Client::new() - .post(&format!("http://{}/1/relayer/create", addr)) - .json(&CreateRelayerRequest { + let CreateRelayerResponse { + address: relayer_address, + relayer_id, + } = client + .create_relayer(&CreateRelayerRequest { name: "Test relayer".to_string(), chain_id: DEFAULT_ANVIL_CHAIN_ID, }) - .send() .await?; - let response: CreateRelayerResponse = response.json().await?; - // Fund the relayer let middleware = setup_middleware( format!("http://{}", double_anvil.local_addr()), @@ -49,7 +45,7 @@ async fn send_tx() -> eyre::Result<()> { middleware .send_transaction( Eip1559TransactionRequest { - to: Some(response.address.into()), + to: Some(relayer_address.into()), value: Some(amount), ..Default::default() }, @@ -60,25 +56,21 @@ async fn send_tx() -> eyre::Result<()> { let provider = middleware.provider(); - let current_balance = provider.get_balance(response.address, None).await?; + let current_balance = provider.get_balance(relayer_address, None).await?; assert_eq!(current_balance, amount); // Send a transaction let value: U256 = parse_units("1", "ether")?.into(); - let response = reqwest::Client::new() - .post(&format!("http://{}/1/tx/send", addr)) - .json(&SendTxRequest { - relayer_id: response.relayer_id, + client + .send_tx(&SendTxRequest { + relayer_id, to: ARBITRARY_ADDRESS, value, gas_limit: U256::from(21_000), ..Default::default() }) - .send() .await?; - let _response: SendTxResponse = response.json().await?; - for _ in 0..10 { let balance = provider.get_balance(ARBITRARY_ADDRESS, None).await?; From 73a9d9cef1a9c9bbe5db4870cb25aeba696b01f4 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 28 Nov 2023 13:59:04 +0100 Subject: [PATCH 007/135] Update TODO --- TODO.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/TODO.md b/TODO.md index e92ecdb..d398829 100644 --- a/TODO.md +++ b/TODO.md @@ -2,19 +2,22 @@ 1. [x] Handling reorgs 2. [ ] Per network settings (i.e. max inflight txs, max gas price, block time) -3. [ ] Multiple RPCs support -4. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) -5. [ ] Transaction priority -6. [ ] Metrics -7. [ ] Tracing (add telemetry-batteries) -8. [ ] Automated testing +3. IN PROGRESS [ ] Per relayer settings +4. [ ] Multiple RPCs support +5. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) +6. [ ] Transaction priority +7. [ ] Metrics +8. [ ] Tracing (add telemetry-batteries) +9. [ ] Automated testing 1. [x] Basic 2. [ ] Basic with contracts 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) -9. [x] Parallelization: +10. [x] Parallelization: 1. [x] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating 3. [ ] Parallelize broadcast per chain id -10. [x] No need to insert all block txs into DB -11. [x] Prune block info +11. [x] No need to insert all block txs into DB +12. [x] Prune block info +13. [ ] Authentication + From f80384794ee27cd35cacf2832f211a96805c7ce2 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 28 Nov 2023 14:01:47 +0100 Subject: [PATCH 008/135] Update TODO --- TODO.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index d398829..6f30a06 100644 --- a/TODO.md +++ b/TODO.md @@ -1,8 +1,8 @@ # TODO 1. [x] Handling reorgs -2. [ ] Per network settings (i.e. max inflight txs, max gas price, block time) -3. IN PROGRESS [ ] Per relayer settings +2. [ ] Per network settings - is this still needed? +3. IN PROGRESS [ ] Per relayer settings (i.e. max inflight txs, max gas price) 4. [ ] Multiple RPCs support 5. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) 6. [ ] Transaction priority From decd1dd13e9ba7ad31cb0e60ba9b2dc19e8df377 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 28 Nov 2023 15:23:54 +0100 Subject: [PATCH 009/135] Allow updating relayers - max inflight txs in relayer --- db/migrations/001_init.sql | 47 +++++----- src/db.rs | 170 ++++++++++++++++++++++++++++++++++--- src/db/data.rs | 23 ++++- src/lib.rs | 1 + src/server.rs | 24 ++++-- src/tasks/broadcast.rs | 4 +- src/types.rs | 108 +++++++++++++++++++++++ 7 files changed, 333 insertions(+), 44 deletions(-) create mode 100644 src/types.rs diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index dc25580..06ec0e0 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -2,12 +2,12 @@ CREATE TYPE tx_status AS ENUM ('pending', 'mined', 'finalized'); CREATE TYPE rpc_kind AS ENUM ('http', 'ws'); -create table networks ( +CREATE TABLE networks ( chain_id BIGINT PRIMARY KEY, name VARCHAR(255) NOT NULL ); -create table rpcs ( +CREATE TABLE rpcs ( id BIGSERIAL PRIMARY KEY, chain_id BIGINT NOT NULL REFERENCES networks(chain_id), url VARCHAR(255) NOT NULL, @@ -15,32 +15,35 @@ create table rpcs ( ); CREATE TABLE relayers ( - id VARCHAR(255) PRIMARY KEY, - name VARCHAR(255) NOT NULL, - chain_id BIGINT NOT NULL, - key_id VARCHAR(255) NOT NULL, - address BYTEA NOT NULL, + -- The relayer's ID is UUID v4 - always 36 characters (including 4 dashes) + id CHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + chain_id BIGINT NOT NULL, + key_id VARCHAR(255) NOT NULL, + address BYTEA NOT NULL, -- The local nonce value - nonce BIGINT NOT NULL, + nonce BIGINT NOT NULL DEFAULT 0, -- The confirmed nonce value - current_nonce BIGINT NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); + current_nonce BIGINT NOT NULL DEFAULT 0, --- CREATE TABLE relayer_deps ( --- relayer_id VARCHAR(255) NOT NULL REFERENCES relayers(id), --- ); + -- Settings + max_inflight_txs BIGINT NOT NULL DEFAULT 5, + gas_limits JSON NOT NULL DEFAULT '[]', + + -- Time keeping fields + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); -- Constant tx data - once a tx is created, this data should never change CREATE TABLE transactions ( - id VARCHAR(255) PRIMARY KEY, - tx_to BYTEA NOT NULL, - data BYTEA NOT NULL, - value BYTEA NOT NULL, - gas_limit BYTEA NOT NULL, - nonce BIGINT NOT NULL, - relayer_id VARCHAR(255) NOT NULL REFERENCES relayers(id) + id VARCHAR(255) PRIMARY KEY, + tx_to BYTEA NOT NULL, + data BYTEA NOT NULL, + value BYTEA NOT NULL, + gas_limit BYTEA NOT NULL, + nonce BIGINT NOT NULL, + relayer_id CHAR(36) NOT NULL REFERENCES relayers(id) ); -- Sent transaction attempts diff --git a/src/db.rs b/src/db.rs index 629642e..1e8389c 100644 --- a/src/db.rs +++ b/src/db.rs @@ -3,10 +3,12 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; +use sqlx::types::Json; use sqlx::{Pool, Postgres, Row}; use crate::broadcast_utils::gas_estimation::FeesEstimate; use crate::config::DatabaseConfig; +use crate::types::{RelayerInfo, RelayerUpdate}; pub mod data; @@ -53,8 +55,8 @@ impl Database { ) -> eyre::Result<()> { sqlx::query( r#" - INSERT INTO relayers (id, name, chain_id, key_id, address, nonce, current_nonce) - VALUES ($1, $2, $3, $4, $5, 0, 0) + INSERT INTO relayers (id, name, chain_id, key_id, address) + VALUES ($1, $2, $3, $4, $5) "#, ) .bind(id) @@ -68,6 +70,72 @@ impl Database { Ok(()) } + pub async fn update_relayer( + &self, + id: &str, + update: &RelayerUpdate, + ) -> eyre::Result<()> { + let mut tx = self.pool.begin().await?; + + if let Some(name) = &update.relayer_name { + sqlx::query( + r#" + UPDATE relayers + SET name = $2 + WHERE id = $1 + "#, + ) + .bind(id) + .bind(name) + .execute(tx.as_mut()) + .await?; + } + + if let Some(max_inflight_txs) = update.max_inflight_txs { + sqlx::query( + r#" + UPDATE relayers + SET max_inflight_txs = $2 + WHERE id = $1 + "#, + ) + .bind(id) + .bind(max_inflight_txs as i64) + .execute(tx.as_mut()) + .await?; + } + + if let Some(gas_limits) = &update.gas_limits { + sqlx::query( + r#" + UPDATE relayers + SET gas_limits = $2 + WHERE id = $1 + "#, + ) + .bind(id) + .bind(Json(gas_limits)) + .execute(tx.as_mut()) + .await?; + } + + tx.commit().await?; + + Ok(()) + } + + pub async fn get_relayer(&self, id: &str) -> eyre::Result { + Ok(sqlx::query_as( + r#" + SELECT id, name, chain_id, key_id, address, nonce, current_nonce, max_inflight_txs, gas_limits + FROM relayers + WHERE id = $1 + "#) + .bind(id) + .fetch_one(&self.pool).await? + ) + } + pub async fn create_transaction( &self, tx_id: &str, @@ -116,10 +184,7 @@ impl Database { Ok(()) } - pub async fn get_unsent_txs( - &self, - max_inflight_txs: usize, - ) -> eyre::Result> { + pub async fn get_unsent_txs(&self) -> eyre::Result> { Ok(sqlx::query_as( r#" SELECT t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id @@ -127,10 +192,9 @@ impl Database { LEFT JOIN sent_transactions s ON (t.id = s.tx_id) INNER JOIN relayers r ON (t.relayer_id = r.id) WHERE s.tx_id IS NULL - AND (t.nonce - r.current_nonce < $1); + AND (t.nonce - r.current_nonce < r.max_inflight_txs); "#, ) - .bind(max_inflight_txs as i64) .fetch_all(&self.pool) .await?) } @@ -201,7 +265,7 @@ impl Database { .await?; let item = row - .map(|row| row.try_get::, _>(0)) + .map(|row| row.try_get::, _>(0)) .transpose()?; Ok(item.map(|json_fee_estimate| json_fee_estimate.0)) @@ -264,7 +328,7 @@ impl Database { .bind(block_number as i64) .bind(chain_id as i64) .bind(timestamp) - .bind(fee_estimates.map(sqlx::types::Json)) + .bind(fee_estimates.map(Json)) .execute(db_tx.as_mut()) .await?; @@ -748,6 +812,8 @@ mod tests { use tracing_subscriber::EnvFilter; use super::*; + use crate::db::data::U256Wrapper; + use crate::types::RelayerGasLimit; async fn setup_db() -> eyre::Result<(Database, DockerContainerGuard)> { let db_container = postgres_docker_utils::setup().await?; @@ -822,7 +888,7 @@ mod tests { } #[tokio::test] - async fn tx_lifecycle() -> eyre::Result<()> { + async fn relayer_methods() -> eyre::Result<()> { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().pretty().compact()) .with( @@ -843,7 +909,83 @@ mod tests { db.create_network(chain_id, network_name, http_rpc, ws_rpc) .await?; - let relayer_id = "relayer_id"; + let relayer_id = uuid(); + let relayer_id = relayer_id.as_str(); + + let relayer_name = "relayer_name"; + let key_id = "key_id"; + let relayer_address = Address::from_low_u64_be(1); + + db.create_relayer( + relayer_id, + relayer_name, + chain_id, + key_id, + relayer_address, + ) + .await?; + + let relayer = db.get_relayer(relayer_id).await?; + + assert_eq!(relayer.id, relayer_id); + assert_eq!(relayer.name, relayer_name); + assert_eq!(relayer.chain_id, chain_id); + assert_eq!(relayer.key_id, key_id); + assert_eq!(relayer.address.0, relayer_address); + assert_eq!(relayer.nonce, 0); + assert_eq!(relayer.current_nonce, 0); + assert_eq!(relayer.max_inflight_txs, 5); + assert_eq!(relayer.gas_limits.0, vec![]); + + db.update_relayer( + relayer_id, + &RelayerUpdate { + relayer_name: None, + max_inflight_txs: Some(10), + gas_limits: Some(vec![RelayerGasLimit { + chain_id: 1, + value: U256Wrapper(U256::from(10_000u64)), + }]), + }, + ) + .await?; + + let relayer = db.get_relayer(relayer_id).await?; + + assert_eq!(relayer.id, relayer_id); + assert_eq!(relayer.name, relayer_name); + assert_eq!(relayer.chain_id, chain_id); + assert_eq!(relayer.key_id, key_id); + assert_eq!(relayer.address.0, relayer_address); + assert_eq!(relayer.nonce, 0); + assert_eq!(relayer.current_nonce, 0); + assert_eq!(relayer.max_inflight_txs, 10); + assert_eq!( + relayer.gas_limits.0, + vec![RelayerGasLimit { + chain_id: 1, + value: U256Wrapper(U256::from(10_000u64)), + }] + ); + + Ok(()) + } + + #[tokio::test] + async fn tx_lifecycle() -> eyre::Result<()> { + let (db, _db_container) = setup_db().await?; + + let chain_id = 123; + let network_name = "network_name"; + let http_rpc = "http_rpc"; + let ws_rpc = "ws_rpc"; + + db.create_network(chain_id, network_name, http_rpc, ws_rpc) + .await?; + + let relayer_id = uuid(); + let relayer_id = relayer_id.as_str(); + let relayer_name = "relayer_name"; let key_id = "key_id"; let relayer_address = Address::from_low_u64_be(1); @@ -993,4 +1135,8 @@ mod tests { .unwrap() .and_utc() } + + fn uuid() -> String { + uuid::Uuid::new_v4().to_string() + } } diff --git a/src/db/data.rs b/src/db/data.rs index d0a2563..6e5f4a7 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -52,9 +52,12 @@ pub struct ReadTxData { pub status: Option, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] pub struct AddressWrapper(pub Address); -#[derive(Debug, Clone, PartialEq, Eq)] + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] pub struct U256Wrapper(pub U256); #[derive(Debug, Clone, PartialEq, Eq)] @@ -118,6 +121,22 @@ where } } +impl<'q, DB> sqlx::Encode<'q, DB> for U256Wrapper +where + DB: Database, + [u8; 32]: sqlx::Encode<'q, DB>, +{ + fn encode_by_ref( + &self, + buf: &mut >::ArgumentBuffer, + ) -> sqlx::encode::IsNull { + let mut bytes = [0u8; 32]; + self.0.to_big_endian(&mut bytes); + + <[u8; 32] as sqlx::Encode>::encode_by_ref(&bytes, buf) + } +} + impl<'r, DB> sqlx::Decode<'r, DB> for H256Wrapper where DB: Database, diff --git a/src/lib.rs b/src/lib.rs index 393030c..9afe4b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,3 +10,4 @@ pub mod server; pub mod service; pub mod task_runner; pub mod tasks; +pub mod types; \ No newline at end of file diff --git a/src/server.rs b/src/server.rs index bb80b7f..96e0be6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -16,6 +16,7 @@ use self::data::{ SendTxResponse, }; use crate::app::App; +use crate::types::{RelayerInfo, RelayerUpdate}; pub mod data; mod middleware; @@ -132,11 +133,23 @@ async fn create_relayer( })) } +async fn update_relayer( + State(app): State>, + Path(relayer_id): Path, + Json(req): Json, +) -> Result<(), ApiError> { + app.db.update_relayer(&relayer_id, &req).await?; + + Ok(()) +} + async fn get_relayer( - State(_app): State>, - Path(_relayer_id): Path, -) -> &'static str { - "Hello, World!" + State(app): State>, + Path(relayer_id): Path, +) -> Result, ApiError> { + let relayer_info = app.db.get_relayer(&relayer_id).await?; + + Ok(Json(relayer_info)) } pub async fn serve(app: Arc) -> eyre::Result<()> { @@ -162,7 +175,8 @@ pub async fn spawn_server( .with_state(app.clone()); let relayer_routes = Router::new() - .route("/create", post(create_relayer)) + .route("/", post(create_relayer)) + .route("/:relayer_id", post(update_relayer)) .route("/:relayer_id", get(get_relayer)) .with_state(app.clone()); diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index d6e1d3d..b088133 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -12,11 +12,9 @@ use crate::broadcast_utils::{ calculate_gas_fees_from_estimates, calculate_max_base_fee_per_gas, }; -const MAX_IN_FLIGHT_TXS: usize = 5; - pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { loop { - let txs = app.db.get_unsent_txs(MAX_IN_FLIGHT_TXS).await?; + let txs = app.db.get_unsent_txs().await?; // TODO: Parallelize per chain id? for tx in txs { diff --git a/src/types.rs b/src/types.rs new file mode 100644 index 0000000..e86b6ea --- /dev/null +++ b/src/types.rs @@ -0,0 +1,108 @@ +use serde::{Deserialize, Serialize}; +use sqlx::prelude::FromRow; +use sqlx::types::Json; + +use crate::db::data::{AddressWrapper, U256Wrapper}; + +#[derive(Deserialize, Serialize, Debug, Clone, Copy, Default, sqlx::Type)] +#[serde(rename_all = "camelCase")] +#[sqlx(type_name = "transaction_priority", rename_all = "camelCase")] +pub enum TransactionPriority { + // 5th percentile + Slowest, + // 25th percentile + Slow, + // 50th percentile + #[default] + Regular, + // 75th percentile + Fast, + // 95th percentile + Fastest, +} + +#[derive(Deserialize, Serialize, Debug, Clone, FromRow)] +#[serde(rename_all = "camelCase")] +pub struct RelayerInfo { + pub id: String, + pub name: String, + #[sqlx(try_from = "i64")] + pub chain_id: u64, + pub key_id: String, + pub address: AddressWrapper, + #[sqlx(try_from = "i64")] + pub nonce: u64, + #[sqlx(try_from = "i64")] + pub current_nonce: u64, + #[sqlx(try_from = "i64")] + pub max_inflight_txs: u64, + pub gas_limits: Json>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct RelayerUpdate { + #[serde(default)] + pub relayer_name: Option, + + #[serde(default)] + pub max_inflight_txs: Option, + + #[serde(default)] + pub gas_limits: Option>, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct RelayerGasLimit { + pub value: U256Wrapper, + pub chain_id: i64, +} + +#[cfg(test)] +mod tests { + use ethers::types::{Address, U256}; + + use super::*; + + #[test] + fn relayer_info_serialize() { + let info = RelayerInfo { + id: "id".to_string(), + name: "name".to_string(), + chain_id: 1, + key_id: "key_id".to_string(), + address: AddressWrapper(Address::zero()), + nonce: 0, + current_nonce: 0, + max_inflight_txs: 0, + gas_limits: Json(vec![RelayerGasLimit { + value: U256Wrapper(U256::zero()), + chain_id: 1, + }]), + }; + + let json = serde_json::to_string_pretty(&info).unwrap(); + + let expected = indoc::indoc! {r#" + { + "id": "id", + "name": "name", + "chainId": 1, + "keyId": "key_id", + "address": "0x0000000000000000000000000000000000000000", + "nonce": 0, + "currentNonce": 0, + "maxInflightTxs": 0, + "gasLimits": [ + { + "value": "0x0", + "chainId": 1 + } + ] + } + "#}; + + assert_eq!(json.trim(), expected.trim()); + } +} From f4790a969c457b937e43081462cc5a6e82be7150 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 28 Nov 2023 15:53:30 +0100 Subject: [PATCH 010/135] Start indexing chains at startup --- src/server/routes/network.rs | 6 ++---- src/service.rs | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/server/routes/network.rs b/src/server/routes/network.rs index e43e219..7b8027b 100644 --- a/src/server/routes/network.rs +++ b/src/server/routes/network.rs @@ -7,6 +7,7 @@ use url::Url; use crate::app::App; use crate::server::ApiError; +use crate::service::Service; use crate::task_runner::TaskRunner; #[derive(Debug, Default, Clone, Serialize, Deserialize)] @@ -51,10 +52,7 @@ pub async fn create_network( .await?; let task_runner = TaskRunner::new(app.clone()); - - task_runner.add_task(format!("index_block_{}", chain_id), move |app| { - crate::tasks::index::index_chain(app, chain_id) - }); + Service::index_chain_for_id(&task_runner, chain_id); Ok(()) } diff --git a/src/service.rs b/src/service.rs index b901beb..746f66c 100644 --- a/src/service.rs +++ b/src/service.rs @@ -18,6 +18,8 @@ impl Service { pub async fn new(config: Config) -> eyre::Result { let app = Arc::new(App::new(config).await?); + let chain_ids = app.db.get_network_chain_ids().await?; + let task_runner = TaskRunner::new(app.clone()); task_runner.add_task("Broadcast transactions", tasks::broadcast_txs); task_runner.add_task("Escalate transactions", tasks::escalate_txs); @@ -27,6 +29,10 @@ impl Service { task_runner.add_task("Handle soft reorgs", tasks::handle_soft_reorgs); task_runner.add_task("Handle hard reorgs", tasks::handle_hard_reorgs); + for chain_id in chain_ids { + Self::index_chain_for_id(&task_runner, chain_id); + } + let server = crate::server::spawn_server(app.clone()).await?; let local_addr = server.local_addr(); let server_handle = tokio::spawn(async move { @@ -41,6 +47,17 @@ impl Service { }) } + pub fn index_chain_for_id( + task_runner: &TaskRunner, + chain_id: u64, + ) -> eyre::Result<()> { + task_runner.add_task(format!("index_block_{}", chain_id), move |app| { + crate::tasks::index::index_chain(app, chain_id) + }); + + Ok(()) + } + pub fn local_addr(&self) -> SocketAddr { self.local_addr } From a56faad316515e83c5447cd1e2894a24c43d356e Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 28 Nov 2023 23:44:23 +0100 Subject: [PATCH 011/135] Estimate block fees in a separate task --- Cargo.lock | 30 ++++++ Cargo.toml | 3 + db/migrations/001_init.sql | 12 ++- src/client.rs | 2 +- src/db.rs | 181 +++++++++++++++++++++++++++++------ src/server.rs | 9 +- src/server/routes/network.rs | 2 +- src/service.rs | 16 +++- src/tasks/index.rs | 41 ++++++-- 9 files changed, 246 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13a5a89..5eb388e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -638,6 +638,30 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bigdecimal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06619be423ea5bb86c95f087d5707942791a08a85530df0db2209a3ecfb8bc9" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -3797,6 +3821,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "axum", + "bigdecimal 0.4.2", "chrono", "clap", "config", @@ -3812,6 +3837,7 @@ dependencies = [ "humantime-serde", "hyper", "indoc", + "num-bigint", "postgres-docker-utils", "rand", "reqwest", @@ -4016,6 +4042,7 @@ checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d" dependencies = [ "ahash 0.8.6", "atoi", + "bigdecimal 0.3.1", "byteorder", "bytes", "chrono", @@ -4100,6 +4127,7 @@ checksum = "864b869fdf56263f4c95c45483191ea0af340f9f3e3e7b4d57a61c7c87a970db" dependencies = [ "atoi", "base64 0.21.5", + "bigdecimal 0.3.1", "bitflags 2.4.1", "byteorder", "bytes", @@ -4144,6 +4172,7 @@ checksum = "eb7ae0e6a97fb3ba33b23ac2671a5ce6e3cabe003f451abd5a56e7951d975624" dependencies = [ "atoi", "base64 0.21.5", + "bigdecimal 0.3.1", "bitflags 2.4.1", "byteorder", "chrono", @@ -4162,6 +4191,7 @@ dependencies = [ "log", "md-5", "memchr", + "num-bigint", "once_cell", "rand", "serde", diff --git a/Cargo.toml b/Cargo.toml index 8144505..2a680e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,10 @@ sqlx = { version = "0.7.2", features = [ "tls-rustls", "postgres", "migrate", + "bigdecimal", ] } +num-bigint = "0.4.4" +bigdecimal = "0.4.2" spki = "0.7.2" async-trait = "0.1.74" diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 06ec0e0..9144ffe 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -77,7 +77,6 @@ CREATE TABLE blocks ( block_number BIGINT NOT NULL, chain_id BIGINT NOT NULL, timestamp TIMESTAMPTZ NOT NULL, - fee_estimate JSON, PRIMARY KEY (block_number, chain_id) ); @@ -88,4 +87,13 @@ CREATE TABLE block_txs ( PRIMARY KEY (block_number, chain_id, tx_hash), FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE, FOREIGN KEY (tx_hash) REFERENCES tx_hashes (tx_hash) -); \ No newline at end of file +); + +CREATE TABLE block_fees ( + block_number BIGINT NOT NULL, + chain_id BIGINT NOT NULL, + gas_price NUMERIC(78, 0) NOT NULL, + fee_estimate JSON NOT NULL, + PRIMARY KEY (block_number, chain_id), + FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE +); diff --git a/src/client.rs b/src/client.rs index 4c6ce5a..f3fd0c7 100644 --- a/src/client.rs +++ b/src/client.rs @@ -22,7 +22,7 @@ impl TxSitterClient { ) -> eyre::Result { let response = self .client - .post(&format!("{}/1/relayer/create", self.url)) + .post(&format!("{}/1/relayer", self.url)) .json(req) .send() .await?; diff --git a/src/db.rs b/src/db.rs index 1e8389c..1b1836d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -3,7 +3,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; -use sqlx::types::Json; +use sqlx::types::{BigDecimal, Json}; use sqlx::{Pool, Postgres, Row}; use crate::broadcast_utils::gas_estimation::FeesEstimate; @@ -246,17 +246,58 @@ impl Database { Ok(()) } + pub async fn get_latest_block_number( + &self, + chain_id: u64, + ) -> eyre::Result { + let (block_number,): (i64,) = sqlx::query_as( + r#" + SELECT block_number + FROM blocks + WHERE chain_id = $1 + ORDER BY block_number DESC + LIMIT 1 + "#, + ) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + Ok(block_number as u64) + } + pub async fn get_latest_block_fees_by_chain_id( &self, chain_id: u64, ) -> eyre::Result> { - let row = sqlx::query( + let row: Option<(Json,)> = sqlx::query_as( r#" - SELECT fee_estimate - FROM blocks - WHERE chain_id = $1 - AND fee_estimate IS NOT NULL - ORDER BY block_number DESC + SELECT bf.fee_estimate + FROM blocks b + JOIN block_fees bf ON (b.block_number = bf.block_number AND b.chain_id = bf.chain_id) + WHERE b.chain_id = $1 + ORDER BY b.block_number DESC + LIMIT 1 + "#, + ) + .bind(chain_id as i64) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|(json_fee_estimate,)| json_fee_estimate.0)) + } + + pub async fn get_latest_gas_price_by_chain_id( + &self, + chain_id: u64, + ) -> eyre::Result> { + let row: Option<(BigDecimal,)> = sqlx::query_as( + r#" + SELECT bf.gas_price + FROM blocks b + JOIN block_fees bf ON (b.block_number = bf.block_number AND b.chain_id = bf.chain_id) + WHERE b.chain_id = $1 + ORDER BY b.block_number DESC LIMIT 1 "#, ) @@ -264,11 +305,14 @@ impl Database { .fetch_optional(&self.pool) .await?; - let item = row - .map(|row| row.try_get::, _>(0)) + let gas_price = row + .map(|(gas_price,)| { + let gas_price_str = gas_price.to_string(); + U256::from_dec_str(&gas_price_str) + }) .transpose()?; - Ok(item.map(|json_fee_estimate| json_fee_estimate.0)) + Ok(gas_price) } pub async fn has_blocks_for_chain( @@ -297,17 +341,16 @@ impl Database { chain_id: u64, timestamp: DateTime, txs: &[H256], - fee_estimates: Option<&FeesEstimate>, ) -> eyre::Result<()> { let mut db_tx = self.pool.begin().await?; - // Prune block txs + // Prune previously inserted block sqlx::query( r#" DELETE - FROM block_txs - WHERE block_number = $1 - AND chain_id = $2 + FROM blocks + WHERE block_number = $1 + AND chain_id = $2 "#, ) .bind(block_number as i64) @@ -315,20 +358,17 @@ impl Database { .execute(db_tx.as_mut()) .await?; - // Insert new block or update + // Insert new block + // There can be no conflict since we remove the previous one sqlx::query( r#" - INSERT INTO blocks (block_number, chain_id, timestamp, fee_estimate) - VALUES ($1, $2, $3, $4) - ON CONFLICT (block_number, chain_id) DO UPDATE - SET timestamp = $3, - fee_estimate = $4 + INSERT INTO blocks (block_number, chain_id, timestamp) + VALUES ($1, $2, $3) "#, ) .bind(block_number as i64) .bind(chain_id as i64) .bind(timestamp) - .bind(fee_estimates.map(Json)) .execute(db_tx.as_mut()) .await?; @@ -357,6 +397,31 @@ impl Database { Ok(()) } + pub async fn save_block_fees( + &self, + block_number: u64, + chain_id: u64, + fee_estimates: &FeesEstimate, + gas_price: U256, + ) -> eyre::Result<()> { + // TODO: Figure out how to do this without parsing + let gas_price: BigDecimal = gas_price.to_string().parse()?; + + sqlx::query( + r#" + INSERT INTO block_fees (block_number, chain_id, fee_estimate, gas_price) + VALUES ($1, $2, $3, $4) + "#, + ) + .bind(block_number as i64) + .bind(chain_id as i64) + .bind(Json(fee_estimates)) + .bind(gas_price) + .execute(&self.pool) + .await?; + Ok(()) + } + pub async fn handle_soft_reorgs(&self) -> eyre::Result<()> { let mut tx = self.pool.begin().await?; @@ -799,6 +864,19 @@ impl Database { Ok(row.0) } + + pub async fn get_network_chain_ids(&self) -> eyre::Result> { + let items: Vec<(i64,)> = sqlx::query_as( + r#" + SELECT chain_id + FROM networks + "#, + ) + .fetch_all(&self.pool) + .await?; + + Ok(items.into_iter().map(|(x,)| x as u64).collect()) + } } #[cfg(test)] @@ -845,11 +923,9 @@ mod tests { } #[tokio::test] - async fn basic() -> eyre::Result<()> { + async fn migration() -> eyre::Result<()> { let (_db, _db_container) = setup_db().await?; - // db.create_relayer().await?; - Ok(()) } @@ -875,8 +951,7 @@ mod tests { H256::from_low_u64_be(3), ]; - db.save_block(1, 1, block_timestamp, &tx_hashes, None) - .await?; + db.save_block(1, 1, block_timestamp, &tx_hashes).await?; assert!(db.has_blocks_for_chain(1).await?, "Should have blocks"); @@ -1068,7 +1143,7 @@ mod tests { let timestamp = ymd_hms(2023, 11, 23, 12, 32, 2); let txs = &[tx_hash_1]; - db.save_block(block_number, chain_id, timestamp, txs, None) + db.save_block(block_number, chain_id, timestamp, txs) .await?; full_update(&db, chain_id, finalized_timestamp).await?; @@ -1081,7 +1156,7 @@ mod tests { // Reorg let txs = &[tx_hash_2]; - db.save_block(block_number, chain_id, timestamp, txs, None) + db.save_block(block_number, chain_id, timestamp, txs) .await?; full_update(&db, chain_id, finalized_timestamp).await?; @@ -1094,7 +1169,7 @@ mod tests { // Destructive reorg let txs = &[]; - db.save_block(block_number, chain_id, timestamp, txs, None) + db.save_block(block_number, chain_id, timestamp, txs) .await?; full_update(&db, chain_id, finalized_timestamp).await?; @@ -1107,7 +1182,7 @@ mod tests { // Finalization let txs = &[tx_hash_2]; - db.save_block(block_number, chain_id, timestamp, txs, None) + db.save_block(block_number, chain_id, timestamp, txs) .await?; let finalized_timestamp = ymd_hms(2023, 11, 23, 22, 0, 0); @@ -1121,6 +1196,50 @@ mod tests { Ok(()) } + #[tokio::test] + async fn blocks() -> eyre::Result<()> { + let (db, _db_container) = setup_db().await?; + + let block_number = 1; + let chain_id = 1; + let timestamp = ymd_hms(2023, 11, 23, 12, 32, 2); + let txs = &[ + H256::from_low_u64_be(1), + H256::from_low_u64_be(2), + H256::from_low_u64_be(3), + ]; + + db.save_block(block_number, chain_id, timestamp, txs) + .await?; + + let fee_estimates = FeesEstimate { + base_fee_per_gas: U256::from(13_132), + percentile_fees: vec![U256::from(516)], + }; + + let gas_price = U256::from(12_352); + + db.save_block_fees(block_number, chain_id, &fee_estimates, gas_price) + .await?; + + let latest_fees = + db.get_latest_block_fees_by_chain_id(chain_id).await?; + let latest_gas_price = + db.get_latest_gas_price_by_chain_id(chain_id).await?; + + let latest_fees = latest_fees.context("Missing fees")?; + let latest_gas_price = latest_gas_price.context("Missing gas price")?; + + assert_eq!( + latest_fees.base_fee_per_gas, + fee_estimates.base_fee_per_gas + ); + assert_eq!(latest_fees.percentile_fees, fee_estimates.percentile_fees); + assert_eq!(latest_gas_price, gas_price); + + Ok(()) + } + fn ymd_hms( year: i32, month: u32, diff --git a/src/server.rs b/src/server.rs index 96e0be6..30fb487 100644 --- a/src/server.rs +++ b/src/server.rs @@ -186,10 +186,13 @@ pub async fn spawn_server( .route("/:chain_id", post(routes::network::create_network)) .with_state(app.clone()); + let v1_routes = Router::new() + .nest("/tx", tx_routes) + .nest("/relayer", relayer_routes) + .nest("/network", network_routes); + let router = Router::new() - .nest("/1/tx", tx_routes) - .nest("/1/relayer", relayer_routes) - .nest("/1/network", network_routes) + .nest("/1", v1_routes) .layer(tower_http::trace::TraceLayer::new_for_http()) .layer(axum::middleware::from_fn(middleware::log_response)); diff --git a/src/server/routes/network.rs b/src/server/routes/network.rs index 7b8027b..3a6c8b7 100644 --- a/src/server/routes/network.rs +++ b/src/server/routes/network.rs @@ -52,7 +52,7 @@ pub async fn create_network( .await?; let task_runner = TaskRunner::new(app.clone()); - Service::index_chain_for_id(&task_runner, chain_id); + Service::spawn_chain_tasks(&task_runner, chain_id)?; Ok(()) } diff --git a/src/service.rs b/src/service.rs index 746f66c..cef7e49 100644 --- a/src/service.rs +++ b/src/service.rs @@ -30,7 +30,7 @@ impl Service { task_runner.add_task("Handle hard reorgs", tasks::handle_hard_reorgs); for chain_id in chain_ids { - Self::index_chain_for_id(&task_runner, chain_id); + Self::spawn_chain_tasks(&task_runner, chain_id)?; } let server = crate::server::spawn_server(app.clone()).await?; @@ -47,13 +47,19 @@ impl Service { }) } - pub fn index_chain_for_id( + pub fn spawn_chain_tasks( task_runner: &TaskRunner, chain_id: u64, ) -> eyre::Result<()> { - task_runner.add_task(format!("index_block_{}", chain_id), move |app| { - crate::tasks::index::index_chain(app, chain_id) - }); + task_runner.add_task( + format!("Index blocks (chain id: {})", chain_id), + move |app| crate::tasks::index::index_chain(app, chain_id), + ); + + task_runner.add_task( + format!("Estimate fees (chain id: {})", chain_id), + move |app| crate::tasks::index::estimate_gas(app, chain_id), + ); Ok(()) } diff --git a/src/tasks/index.rs b/src/tasks/index.rs index eb67680..78837fd 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use chrono::{DateTime, Utc}; use ethers::providers::{Http, Middleware, Provider}; @@ -14,6 +15,7 @@ use crate::broadcast_utils::gas_estimation::{ const BLOCK_FEE_HISTORY_SIZE: usize = 10; const FEE_PERCENTILES: [f64; 5] = [5.0, 25.0, 50.0, 75.0, 95.0]; +const TIME_BETWEEN_FEE_ESTIMATION_SECONDS: u64 = 30; pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { loop { @@ -35,19 +37,12 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { ) .context("Invalid timestamp")?; - // TODO: We don't need to do this for every block for a given chain - // Add a separate task to do this periodically for the latest block - let fee_estimates = fetch_block_fee_estimates(&rpc, block_number) - .await - .context("Failed to fetch fee estimates")?; - app.db .save_block( block.number.unwrap().as_u64(), chain_id, block_timestamp, &block.transactions, - Some(&fee_estimates), ) .await?; @@ -62,6 +57,38 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { } } +pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { + let rpc = app.fetch_http_provider(chain_id).await?; + + loop { + let latest_block_number = + app.db.get_latest_block_number(chain_id).await?; + + tracing::info!(block_number = latest_block_number, "Estimating fees"); + + let fee_estimates = + fetch_block_fee_estimates(&rpc, latest_block_number) + .await + .context("Failed to fetch fee estimates")?; + + let gas_price = rpc.get_gas_price().await?; + + app.db + .save_block_fees( + latest_block_number, + chain_id, + &fee_estimates, + gas_price, + ) + .await?; + + tokio::time::sleep(Duration::from_secs( + TIME_BETWEEN_FEE_ESTIMATION_SECONDS, + )) + .await; + } +} + async fn update_relayer_nonces( relayer_addresses: Vec, app: &Arc, From 426a50cb96e778cd99e7721d1dc3f45234964c05 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 00:54:12 +0100 Subject: [PATCH 012/135] Add support for cross-chain gas price limits --- TODO.md | 23 +++++----- manual_test_gas_prices.nu | 68 ++++++++++++++++++++++++++++ src/app.rs | 8 ++-- src/broadcast_utils.rs | 32 +++++++++++++ src/db.rs | 94 +++++++++++++++++++-------------------- src/db/data.rs | 10 +++++ src/lib.rs | 2 +- src/tasks/broadcast.rs | 14 ++++-- src/tasks/escalate.rs | 18 +++++--- src/tasks/index.rs | 17 ++++--- 10 files changed, 204 insertions(+), 82 deletions(-) create mode 100644 manual_test_gas_prices.nu diff --git a/TODO.md b/TODO.md index 6f30a06..6f8ed2c 100644 --- a/TODO.md +++ b/TODO.md @@ -1,23 +1,24 @@ # TODO 1. [x] Handling reorgs -2. [ ] Per network settings - is this still needed? -3. IN PROGRESS [ ] Per relayer settings (i.e. max inflight txs, max gas price) +2. [ ] Per network settings (max blocks age/number - for pruning) +3. [x] Per relayer settings (i.e. max inflight txs, max gas price) 4. [ ] Multiple RPCs support -5. [ ] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) -6. [ ] Transaction priority -7. [ ] Metrics -8. [ ] Tracing (add telemetry-batteries) -9. [ ] Automated testing +5. IN PROGRESS [ ] RPC Access +6. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) +7. [ ] Transaction priority +8. [ ] Metrics +9. [ ] Tracing (add telemetry-batteries) +10. [ ] Automated testing 1. [x] Basic 2. [ ] Basic with contracts 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) -10. [x] Parallelization: +11. [x] Parallelization: 1. [x] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating 3. [ ] Parallelize broadcast per chain id -11. [x] No need to insert all block txs into DB -12. [x] Prune block info -13. [ ] Authentication +12. [x] No need to insert all block txs into DB +13. [x] Prune block info +14. [ ] Authentication diff --git a/manual_test_gas_prices.nu b/manual_test_gas_prices.nu new file mode 100644 index 0000000..ca5ec70 --- /dev/null +++ b/manual_test_gas_prices.nu @@ -0,0 +1,68 @@ +# Setup dependencies in different terminals: +# DB +psql postgres://postgres:postgres@127.0.0.1:5432/database + +# Nodes +anvil --chain-id 31337 -p 8545 --block-time 1 +anvil --chain-id 31338 -p 8546 --block-time 1 + +# TxSitter +cargo watch -x run +# or just +cargo run + +let txSitter = "http://127.0.0.1:3000" +let anvilSocket = "127.0.0.1:8545" +let anvilSocket2 = "127.0.0.1:8546" + +http post -t application/json $"($txSitter)/1/network/31337" { + name: "Anvil network", + httpRpc: $"http://($anvilSocket)", + wsRpc: $"ws://($anvilSocket)" +} + +http post -t application/json $"($txSitter)/1/network/31338" { + name: "Secondary Anvil network", + httpRpc: $"http://($anvilSocket2)", + wsRpc: $"ws://($anvilSocket2)" +} + +echo "Creating relayer" +let relayer = http post -t application/json $"($txSitter)/1/relayer" { "name": "My Relayer", "chainId": 31337 } + +echo "Update relayer - with secondary chain gas limit dependency" +http post -t application/json $"($txSitter)/1/relayer/($relayer.relayerId)" { + gasLimits: [ + { + chainId: 31338, + # Note that this value is hexadecimal + value: "3B9ACA00" + } + ] +} + +echo "Funding relayer" +cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' + +echo "Sending transaction" +let tx = http post -t application/json $"($txSitter)/1/tx/send" { + "relayerId": $relayer.relayerId, + "to": $relayer.address, + "value": "10", + "data": "" + "gasLimit": "150000" +} + +echo "Wait until tx is mined" +for i in 0..100 { + let txResponse = http get $"($txSitter)/1/tx/($tx.txId)" + + if ($txResponse | get -i status) == "mined" { + echo $txResponse + break + } else { + sleep 1sec + } +} + +echo "Success!" diff --git a/src/app.rs b/src/app.rs index b67db93..2535001 100644 --- a/src/app.rs +++ b/src/app.rs @@ -32,7 +32,7 @@ impl App { }) } - pub async fn fetch_http_provider( + pub async fn http_provider( &self, chain_id: u64, ) -> eyre::Result> { @@ -43,7 +43,7 @@ impl App { Ok(provider) } - pub async fn fetch_ws_provider( + pub async fn ws_provider( &self, chain_id: u64, ) -> eyre::Result> { @@ -55,12 +55,12 @@ impl App { Ok(provider) } - pub async fn fetch_signer_middleware( + pub async fn signer_middleware( &self, chain_id: u64, key_id: String, ) -> eyre::Result { - let rpc = self.fetch_http_provider(chain_id).await?; + let rpc = self.http_provider(chain_id).await?; let wallet = self .keys_source diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index e4172d9..e719597 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -1,6 +1,8 @@ use ethers::types::{Eip1559TransactionRequest, U256}; +use eyre::ContextCompat; use self::gas_estimation::FeesEstimate; +use crate::app::App; pub mod gas_estimation; @@ -75,3 +77,33 @@ pub fn escalate_priority_fee( tx.max_fee_per_gas = Some(new_max_fee_per_gas); tx.max_priority_fee_per_gas = Some(new_max_priority_fee_per_gas); } + +pub async fn should_send_transaction( + app: &App, + relayer_id: &str, +) -> eyre::Result { + let relayer = app.db.get_relayer(relayer_id).await?; + + for gas_limit in &relayer.gas_limits.0 { + let chain_fees = app + .db + .get_latest_block_fees_by_chain_id(relayer.chain_id) + .await? + .context("Missing block")?; + + tracing::info!(?chain_fees, gas_limit = ?gas_limit.value.0, "Checking gas price",); + + if chain_fees.gas_price > gas_limit.value.0 { + tracing::warn!( + chain_id = relayer.chain_id, + gas_price = ?chain_fees.gas_price, + gas_limit = ?gas_limit.value.0, + "Gas price is too high for relayer" + ); + + return Ok(false); + } + } + + Ok(true) +} diff --git a/src/db.rs b/src/db.rs index 1b1836d..2d9a215 100644 --- a/src/db.rs +++ b/src/db.rs @@ -12,7 +12,7 @@ use crate::types::{RelayerInfo, RelayerUpdate}; pub mod data; -use self::data::{AddressWrapper, H256Wrapper, ReadTxData, RpcKind}; +use self::data::{AddressWrapper, BlockFees, H256Wrapper, ReadTxData, RpcKind}; pub use self::data::{TxForEscalation, TxStatus, UnsentTx}; // Statically link in migration files @@ -187,7 +187,7 @@ impl Database { pub async fn get_unsent_txs(&self) -> eyre::Result> { Ok(sqlx::query_as( r#" - SELECT t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id + SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id FROM transactions t LEFT JOIN sent_transactions s ON (t.id = s.tx_id) INNER JOIN relayers r ON (t.relayer_id = r.id) @@ -269,10 +269,10 @@ impl Database { pub async fn get_latest_block_fees_by_chain_id( &self, chain_id: u64, - ) -> eyre::Result> { - let row: Option<(Json,)> = sqlx::query_as( + ) -> eyre::Result> { + let row: Option<(Json, BigDecimal)> = sqlx::query_as( r#" - SELECT bf.fee_estimate + SELECT bf.fee_estimate, bf.gas_price FROM blocks b JOIN block_fees bf ON (b.block_number = bf.block_number AND b.chain_id = bf.chain_id) WHERE b.chain_id = $1 @@ -284,35 +284,19 @@ impl Database { .fetch_optional(&self.pool) .await?; - Ok(row.map(|(json_fee_estimate,)| json_fee_estimate.0)) - } + let Some((fees, gas_price)) = row else { + return Ok(None); + }; - pub async fn get_latest_gas_price_by_chain_id( - &self, - chain_id: u64, - ) -> eyre::Result> { - let row: Option<(BigDecimal,)> = sqlx::query_as( - r#" - SELECT bf.gas_price - FROM blocks b - JOIN block_fees bf ON (b.block_number = bf.block_number AND b.chain_id = bf.chain_id) - WHERE b.chain_id = $1 - ORDER BY b.block_number DESC - LIMIT 1 - "#, - ) - .bind(chain_id as i64) - .fetch_optional(&self.pool) - .await?; + let fee_estimates = fees.0; - let gas_price = row - .map(|(gas_price,)| { - let gas_price_str = gas_price.to_string(); - U256::from_dec_str(&gas_price_str) - }) - .transpose()?; + let gas_price_str = gas_price.to_string(); + let gas_price = U256::from_dec_str(&gas_price_str)?; - Ok(gas_price) + Ok(Some(BlockFees { + fee_estimates, + gas_price, + })) } pub async fn has_blocks_for_chain( @@ -407,6 +391,14 @@ impl Database { // TODO: Figure out how to do this without parsing let gas_price: BigDecimal = gas_price.to_string().parse()?; + tracing::info!( + block_number, + chain_id, + ?fee_estimates, + ?gas_price, + "Saving block fees" + ); + sqlx::query( r#" INSERT INTO block_fees (block_number, chain_id, fee_estimate, gas_price) @@ -621,13 +613,13 @@ impl Database { Ok(()) } - pub async fn fetch_txs_for_escalation( + pub async fn get_txs_for_escalation( &self, escalation_interval: Duration, ) -> eyre::Result> { Ok(sqlx::query_as( r#" - SELECT t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, + SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id, s.initial_max_fee_per_gas, s.initial_max_priority_fee_per_gas, s.escalation_count FROM transactions t @@ -722,7 +714,7 @@ impl Database { .await?) } - pub async fn fetch_relayer_addresses( + pub async fn get_relayer_addresses( &self, chain_id: u64, ) -> eyre::Result> { @@ -1019,7 +1011,7 @@ mod tests { max_inflight_txs: Some(10), gas_limits: Some(vec![RelayerGasLimit { chain_id: 1, - value: U256Wrapper(U256::from(10_000u64)), + value: U256Wrapper(U256::from(10_123u64)), }]), }, ) @@ -1039,7 +1031,7 @@ mod tests { relayer.gas_limits.0, vec![RelayerGasLimit { chain_id: 1, - value: U256Wrapper(U256::from(10_000u64)), + value: U256Wrapper(U256::from(10_123u64)), }] ); @@ -1214,28 +1206,27 @@ mod tests { let fee_estimates = FeesEstimate { base_fee_per_gas: U256::from(13_132), - percentile_fees: vec![U256::from(516)], + percentile_fees: vec![U256::from(0)], }; - let gas_price = U256::from(12_352); + let gas_price = U256::from(1_000_000_007); db.save_block_fees(block_number, chain_id, &fee_estimates, gas_price) .await?; - let latest_fees = - db.get_latest_block_fees_by_chain_id(chain_id).await?; - let latest_gas_price = - db.get_latest_gas_price_by_chain_id(chain_id).await?; + let block_fees = db.get_latest_block_fees_by_chain_id(chain_id).await?; - let latest_fees = latest_fees.context("Missing fees")?; - let latest_gas_price = latest_gas_price.context("Missing gas price")?; + let block_fees = block_fees.context("Missing fees")?; assert_eq!( - latest_fees.base_fee_per_gas, + block_fees.fee_estimates.base_fee_per_gas, fee_estimates.base_fee_per_gas ); - assert_eq!(latest_fees.percentile_fees, fee_estimates.percentile_fees); - assert_eq!(latest_gas_price, gas_price); + assert_eq!( + block_fees.fee_estimates.percentile_fees, + fee_estimates.percentile_fees + ); + assert_eq!(block_fees.gas_price, gas_price); Ok(()) } @@ -1258,4 +1249,13 @@ mod tests { fn uuid() -> String { uuid::Uuid::new_v4().to_string() } + + #[test] + fn bignum_conversion() { + let bd = BigDecimal::from(123u32); + let un = U256::from_dec_str("123").unwrap(); + + println!("bd = {bd}"); + println!("un = {un}"); + } } diff --git a/src/db/data.rs b/src/db/data.rs index 6e5f4a7..a700a0d 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -5,8 +5,11 @@ use sqlx::postgres::{PgHasArrayType, PgTypeInfo}; use sqlx::prelude::FromRow; use sqlx::Database; +use crate::broadcast_utils::gas_estimation::FeesEstimate; + #[derive(Debug, Clone, FromRow)] pub struct UnsentTx { + pub relayer_id: String, pub id: String, pub tx_to: AddressWrapper, pub data: Vec, @@ -21,6 +24,7 @@ pub struct UnsentTx { #[derive(Debug, Clone, FromRow)] pub struct TxForEscalation { + pub relayer_id: String, pub id: String, pub tx_to: AddressWrapper, pub data: Vec, @@ -52,6 +56,12 @@ pub struct ReadTxData { pub status: Option, } +#[derive(Debug, Clone)] +pub struct BlockFees { + pub fee_estimates: FeesEstimate, + pub gas_price: U256, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(transparent)] pub struct AddressWrapper(pub Address); diff --git a/src/lib.rs b/src/lib.rs index 9afe4b9..c897aa2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,4 +10,4 @@ pub mod server; pub mod service; pub mod task_runner; pub mod tasks; -pub mod types; \ No newline at end of file +pub mod types; diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index b088133..9aaed68 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -10,6 +10,7 @@ use eyre::ContextCompat; use crate::app::App; use crate::broadcast_utils::{ calculate_gas_fees_from_estimates, calculate_max_base_fee_per_gas, + should_send_transaction, }; pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { @@ -20,22 +21,27 @@ pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { for tx in txs { tracing::info!(tx.id, "Sending tx"); + if !should_send_transaction(&app, &tx.relayer_id).await? { + tracing::warn!(id = tx.id, "Skipping transaction broadcast"); + continue; + } + let middleware = app - .fetch_signer_middleware(tx.chain_id, tx.key_id.clone()) + .signer_middleware(tx.chain_id, tx.key_id.clone()) .await?; - let estimates = app + let fees = app .db .get_latest_block_fees_by_chain_id(tx.chain_id) .await? .context("Missing block")?; let max_base_fee_per_gas = - calculate_max_base_fee_per_gas(&estimates)?; + calculate_max_base_fee_per_gas(&fees.fee_estimates)?; let (max_fee_per_gas, max_priority_fee_per_gas) = calculate_gas_fees_from_estimates( - &estimates, + &fees.fee_estimates, 2, // Priority - 50th percentile max_base_fee_per_gas, )?; diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index b840144..05f65cd 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -7,24 +7,30 @@ use ethers::types::{Address, Eip1559TransactionRequest, NameOrAddress, U256}; use eyre::ContextCompat; use crate::app::App; +use crate::broadcast_utils::should_send_transaction; pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { loop { let txs_for_escalation = app .db - .fetch_txs_for_escalation(app.config.service.escalation_interval) + .get_txs_for_escalation(app.config.service.escalation_interval) .await?; for tx in txs_for_escalation { tracing::info!(tx.id, "Escalating tx"); - let middleware = app - .fetch_signer_middleware(tx.chain_id, tx.key_id.clone()) - .await?; + if !should_send_transaction(&app, &tx.relayer_id).await? { + tracing::warn!(id = tx.id, "Skipping transaction broadcast"); + continue; + } let escalation = tx.escalation_count + 1; - let estimates = app + let middleware = app + .signer_middleware(tx.chain_id, tx.key_id.clone()) + .await?; + + let fees = app .db .get_latest_block_fees_by_chain_id(tx.chain_id) .await? @@ -47,7 +53,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { + max_priority_fee_per_gas_increase; let max_fee_per_gas = - estimates.base_fee_per_gas + max_priority_fee_per_gas; + fees.fee_estimates.base_fee_per_gas + max_priority_fee_per_gas; let eip1559_tx = Eip1559TransactionRequest { from: None, diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 78837fd..9db8a58 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -19,8 +19,8 @@ const TIME_BETWEEN_FEE_ESTIMATION_SECONDS: u64 = 30; pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { loop { - let ws_rpc = app.fetch_ws_provider(chain_id).await?; - let rpc = app.fetch_http_provider(chain_id).await?; + let ws_rpc = app.ws_provider(chain_id).await?; + let rpc = app.http_provider(chain_id).await?; let mut blocks_stream = ws_rpc.subscribe_blocks().await?; @@ -49,7 +49,7 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { app.db.mine_txs(chain_id).await?; let relayer_addresses = - app.db.fetch_relayer_addresses(chain_id).await?; + app.db.get_relayer_addresses(chain_id).await?; update_relayer_nonces(relayer_addresses, &app, &rpc, chain_id) .await?; @@ -58,7 +58,7 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { } pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { - let rpc = app.fetch_http_provider(chain_id).await?; + let rpc = app.http_provider(chain_id).await?; loop { let latest_block_number = @@ -66,10 +66,9 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { tracing::info!(block_number = latest_block_number, "Estimating fees"); - let fee_estimates = - fetch_block_fee_estimates(&rpc, latest_block_number) - .await - .context("Failed to fetch fee estimates")?; + let fee_estimates = get_block_fee_estimates(&rpc, latest_block_number) + .await + .context("Failed to fetch fee estimates")?; let gas_price = rpc.get_gas_price().await?; @@ -129,7 +128,7 @@ async fn update_relayer_nonces( Ok(()) } -pub async fn fetch_block_fee_estimates( +pub async fn get_block_fee_estimates( rpc: &Provider, block_id: impl Into, ) -> eyre::Result { From 8a800ec2b650e78c576534d8f38b4776e974f784 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 00:56:32 +0100 Subject: [PATCH 013/135] Cleanup --- src/broadcast_utils.rs | 2 -- src/db.rs | 31 ------------------------------- 2 files changed, 33 deletions(-) diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index e719597..c4159f0 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -91,8 +91,6 @@ pub async fn should_send_transaction( .await? .context("Missing block")?; - tracing::info!(?chain_fees, gas_limit = ?gas_limit.value.0, "Checking gas price",); - if chain_fees.gas_price > gas_limit.value.0 { tracing::warn!( chain_id = relayer.chain_id, diff --git a/src/db.rs b/src/db.rs index 2d9a215..cb20193 100644 --- a/src/db.rs +++ b/src/db.rs @@ -391,14 +391,6 @@ impl Database { // TODO: Figure out how to do this without parsing let gas_price: BigDecimal = gas_price.to_string().parse()?; - tracing::info!( - block_number, - chain_id, - ?fee_estimates, - ?gas_price, - "Saving block fees" - ); - sqlx::query( r#" INSERT INTO block_fees (block_number, chain_id, fee_estimate, gas_price) @@ -876,10 +868,6 @@ mod tests { use chrono::NaiveDate; use eyre::ContextCompat; use postgres_docker_utils::DockerContainerGuard; - use tracing::level_filters::LevelFilter; - use tracing_subscriber::layer::SubscriberExt; - use tracing_subscriber::util::SubscriberInitExt; - use tracing_subscriber::EnvFilter; use super::*; use crate::db::data::U256Wrapper; @@ -956,16 +944,6 @@ mod tests { #[tokio::test] async fn relayer_methods() -> eyre::Result<()> { - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().pretty().compact()) - .with( - EnvFilter::builder() - .with_default_directive(LevelFilter::INFO.into()) - // Logging from fake_rpc can get very messy so we set it to warn only - .parse_lossy("info"), - ) - .init(); - let (db, _db_container) = setup_db().await?; let chain_id = 123; @@ -1249,13 +1227,4 @@ mod tests { fn uuid() -> String { uuid::Uuid::new_v4().to_string() } - - #[test] - fn bignum_conversion() { - let bd = BigDecimal::from(123u32); - let un = U256::from_dec_str("123").unwrap(); - - println!("bd = {bd}"); - println!("un = {un}"); - } } From 5218017519c46b9a16c77fc3252890121fdc72c9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 11:15:20 +0100 Subject: [PATCH 014/135] Add transaction priority --- TODO.md | 2 +- db/migrations/001_init.sql | 3 +++ src/db.rs | 19 +++++++++++++------ src/db/data.rs | 2 ++ src/server.rs | 1 + src/server/data.rs | 3 +++ src/tasks/broadcast.rs | 2 +- src/types.rs | 16 +++++++++++----- 8 files changed, 35 insertions(+), 13 deletions(-) diff --git a/TODO.md b/TODO.md index 6f8ed2c..e5d91c0 100644 --- a/TODO.md +++ b/TODO.md @@ -6,7 +6,7 @@ 4. [ ] Multiple RPCs support 5. IN PROGRESS [ ] RPC Access 6. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) -7. [ ] Transaction priority +7. [x] Transaction priority 8. [ ] Metrics 9. [ ] Tracing (add telemetry-batteries) 10. [ ] Automated testing diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 9144ffe..b0c75d1 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -2,6 +2,8 @@ CREATE TYPE tx_status AS ENUM ('pending', 'mined', 'finalized'); CREATE TYPE rpc_kind AS ENUM ('http', 'ws'); +CREATE TYPE transaction_priority AS ENUM ('slowest', 'slow', 'regular', 'fast', 'fastest'); + CREATE TABLE networks ( chain_id BIGINT PRIMARY KEY, name VARCHAR(255) NOT NULL @@ -43,6 +45,7 @@ CREATE TABLE transactions ( value BYTEA NOT NULL, gas_limit BYTEA NOT NULL, nonce BIGINT NOT NULL, + priority transaction_priority NOT NULL, relayer_id CHAR(36) NOT NULL REFERENCES relayers(id) ); diff --git a/src/db.rs b/src/db.rs index cb20193..74d0384 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,3 +1,5 @@ +#![allow(clippy::too_many_arguments)] + use std::time::Duration; use chrono::{DateTime, Utc}; @@ -8,7 +10,7 @@ use sqlx::{Pool, Postgres, Row}; use crate::broadcast_utils::gas_estimation::FeesEstimate; use crate::config::DatabaseConfig; -use crate::types::{RelayerInfo, RelayerUpdate}; +use crate::types::{RelayerInfo, RelayerUpdate, TransactionPriority}; pub mod data; @@ -143,6 +145,7 @@ impl Database { data: &[u8], value: U256, gas_limit: U256, + priority: TransactionPriority, relayer_id: &str, ) -> eyre::Result<()> { let mut tx = self.pool.begin().await?; @@ -154,8 +157,8 @@ impl Database { sqlx::query( r#" - INSERT INTO transactions (id, tx_to, data, value, gas_limit, relayer_id, nonce) - VALUES ($1, $2, $3, $4, $5, $6, (SELECT nonce FROM relayers WHERE id = $6)) + INSERT INTO transactions (id, tx_to, data, value, gas_limit, priority, relayer_id, nonce) + VALUES ($1, $2, $3, $4, $5, $6, $7, (SELECT nonce FROM relayers WHERE id = $7)) "#, ) .bind(tx_id) @@ -163,6 +166,7 @@ impl Database { .bind(data) .bind(value_bytes) .bind(gas_limit_bytes) + .bind(priority) .bind(relayer_id) .execute(tx.as_mut()) .await?; @@ -187,7 +191,7 @@ impl Database { pub async fn get_unsent_txs(&self) -> eyre::Result> { Ok(sqlx::query_as( r#" - SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.nonce, r.key_id, r.chain_id + SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.priority, t.nonce, r.key_id, r.chain_id FROM transactions t LEFT JOIN sent_transactions s ON (t.id = s.tx_id) INNER JOIN relayers r ON (t.relayer_id = r.id) @@ -1049,12 +1053,15 @@ mod tests { let data: &[u8] = &[]; let value = U256::from(0); let gas_limit = U256::from(0); + let priority = TransactionPriority::Regular; let tx = db.read_tx(tx_id).await?; assert!(tx.is_none(), "Tx has not been sent yet"); - db.create_transaction(tx_id, to, data, value, gas_limit, relayer_id) - .await?; + db.create_transaction( + tx_id, to, data, value, gas_limit, priority, relayer_id, + ) + .await?; let tx = db.read_tx(tx_id).await?.context("Missing tx")?; diff --git a/src/db/data.rs b/src/db/data.rs index a700a0d..5fccf40 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -6,6 +6,7 @@ use sqlx::prelude::FromRow; use sqlx::Database; use crate::broadcast_utils::gas_estimation::FeesEstimate; +use crate::types::TransactionPriority; #[derive(Debug, Clone, FromRow)] pub struct UnsentTx { @@ -15,6 +16,7 @@ pub struct UnsentTx { pub data: Vec, pub value: U256Wrapper, pub gas_limit: U256Wrapper, + pub priority: TransactionPriority, #[sqlx(try_from = "i64")] pub nonce: u64, pub key_id: String, diff --git a/src/server.rs b/src/server.rs index 30fb487..33af55b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -81,6 +81,7 @@ async fn send_tx( req.data.as_ref().map(|d| &d[..]).unwrap_or(&[]), req.value, req.gas_limit, + req.priority, &req.relayer_id, ) .await?; diff --git a/src/server/data.rs b/src/server/data.rs index 44fc65b..e141f33 100644 --- a/src/server/data.rs +++ b/src/server/data.rs @@ -2,6 +2,7 @@ use ethers::types::{Address, Bytes, H256, U256}; use serde::{Deserialize, Serialize}; use crate::db::TxStatus; +use crate::types::TransactionPriority; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -15,6 +16,8 @@ pub struct SendTxRequest { #[serde(with = "crate::serde_utils::decimal_u256")] pub gas_limit: U256, #[serde(default)] + pub priority: TransactionPriority, + #[serde(default)] pub tx_id: Option, } diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 9aaed68..5ec9e4f 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -42,7 +42,7 @@ pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { let (max_fee_per_gas, max_priority_fee_per_gas) = calculate_gas_fees_from_estimates( &fees.fee_estimates, - 2, // Priority - 50th percentile + tx.priority.to_percentile_index(), max_base_fee_per_gas, )?; diff --git a/src/types.rs b/src/types.rs index e86b6ea..47fd120 100644 --- a/src/types.rs +++ b/src/types.rs @@ -9,16 +9,22 @@ use crate::db::data::{AddressWrapper, U256Wrapper}; #[sqlx(type_name = "transaction_priority", rename_all = "camelCase")] pub enum TransactionPriority { // 5th percentile - Slowest, + Slowest = 0, // 25th percentile - Slow, + Slow = 1, // 50th percentile #[default] - Regular, + Regular = 2, // 75th percentile - Fast, + Fast = 3, // 95th percentile - Fastest, + Fastest = 4, +} + +impl TransactionPriority { + pub fn to_percentile_index(self) -> usize { + self as usize + } } #[derive(Deserialize, Serialize, Debug, Clone, FromRow)] From 7279840ae31e9d53540fafb46fa2e2739c09bf13 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 11:26:07 +0100 Subject: [PATCH 015/135] Refactor routes --- src/client.rs | 7 +- src/server.rs | 108 +----------------------------- src/server/data.rs | 62 ----------------- src/server/routes.rs | 2 + src/server/routes/relayer.rs | 65 ++++++++++++++++++ src/server/routes/transaction.rs | 110 +++++++++++++++++++++++++++++++ tests/common/mod.rs | 15 +++++ tests/create_relayer.rs | 8 +-- tests/send_many_txs.rs | 11 +--- tests/send_tx.rs | 11 +--- 10 files changed, 203 insertions(+), 196 deletions(-) delete mode 100644 src/server/data.rs create mode 100644 src/server/routes/relayer.rs create mode 100644 src/server/routes/transaction.rs diff --git a/src/client.rs b/src/client.rs index f3fd0c7..91bcd68 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,7 +1,8 @@ -use crate::server::data::{ - CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, SendTxResponse, -}; use crate::server::routes::network::NewNetworkInfo; +use crate::server::routes::relayer::{ + CreateRelayerRequest, CreateRelayerResponse, +}; +use crate::server::routes::transaction::{SendTxRequest, SendTxResponse}; pub struct TxSitterClient { client: reqwest::Client, diff --git a/src/server.rs b/src/server.rs index 33af55b..b04f2fc 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,24 +1,16 @@ use std::sync::Arc; -use axum::extract::{Json, Path, State}; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::{get, post, IntoMakeService}; -use axum::{Router, TypedHeader}; -use ethers::signers::Signer; -use eyre::Result; +use axum::Router; use hyper::server::conn::AddrIncoming; -use middleware::AuthorizedRelayer; use thiserror::Error; -use self::data::{ - CreateRelayerRequest, CreateRelayerResponse, GetTxResponse, SendTxRequest, - SendTxResponse, -}; +use self::routes::relayer::{create_relayer, get_relayer, update_relayer}; +use self::routes::transaction::{get_tx, send_tx}; use crate::app::App; -use crate::types::{RelayerInfo, RelayerUpdate}; -pub mod data; mod middleware; pub mod routes; @@ -59,100 +51,6 @@ impl IntoResponse for ApiError { } } -async fn send_tx( - State(app): State>, - TypedHeader(authorized_relayer): TypedHeader, - Json(req): Json, -) -> Result, ApiError> { - if !authorized_relayer.is_authorized(&req.relayer_id) { - return Err(ApiError::Unauthorized); - } - - let tx_id = if let Some(id) = req.tx_id { - id - } else { - uuid::Uuid::new_v4().to_string() - }; - - app.db - .create_transaction( - &tx_id, - req.to, - req.data.as_ref().map(|d| &d[..]).unwrap_or(&[]), - req.value, - req.gas_limit, - req.priority, - &req.relayer_id, - ) - .await?; - - Ok(Json(SendTxResponse { tx_id })) -} - -async fn get_tx( - State(app): State>, - Path(tx_id): Path, -) -> Result, ApiError> { - let tx = app.db.read_tx(&tx_id).await?.ok_or(ApiError::MissingTx)?; - - let get_tx_response = GetTxResponse { - tx_id: tx.tx_id, - to: tx.to.0, - data: if tx.data.is_empty() { - None - } else { - Some(tx.data.into()) - }, - value: tx.value.0, - gas_limit: tx.gas_limit.0, - nonce: tx.nonce, - tx_hash: tx.tx_hash.map(|h| h.0), - status: tx.status, - }; - - Ok(Json(get_tx_response)) -} - -async fn create_relayer( - State(app): State>, - Json(req): Json, -) -> Result, ApiError> { - let (key_id, signer) = app.keys_source.new_signer().await?; - - let address = signer.address(); - - let relayer_id = uuid::Uuid::new_v4(); - let relayer_id = relayer_id.to_string(); - - app.db - .create_relayer(&relayer_id, &req.name, req.chain_id, &key_id, address) - .await?; - - Ok(Json(CreateRelayerResponse { - relayer_id, - address, - })) -} - -async fn update_relayer( - State(app): State>, - Path(relayer_id): Path, - Json(req): Json, -) -> Result<(), ApiError> { - app.db.update_relayer(&relayer_id, &req).await?; - - Ok(()) -} - -async fn get_relayer( - State(app): State>, - Path(relayer_id): Path, -) -> Result, ApiError> { - let relayer_info = app.db.get_relayer(&relayer_id).await?; - - Ok(Json(relayer_info)) -} - pub async fn serve(app: Arc) -> eyre::Result<()> { let server = spawn_server(app).await?; diff --git a/src/server/data.rs b/src/server/data.rs deleted file mode 100644 index e141f33..0000000 --- a/src/server/data.rs +++ /dev/null @@ -1,62 +0,0 @@ -use ethers::types::{Address, Bytes, H256, U256}; -use serde::{Deserialize, Serialize}; - -use crate::db::TxStatus; -use crate::types::TransactionPriority; - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SendTxRequest { - pub relayer_id: String, - pub to: Address, - #[serde(with = "crate::serde_utils::decimal_u256")] - pub value: U256, - #[serde(default)] - pub data: Option, - #[serde(with = "crate::serde_utils::decimal_u256")] - pub gas_limit: U256, - #[serde(default)] - pub priority: TransactionPriority, - #[serde(default)] - pub tx_id: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SendTxResponse { - pub tx_id: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetTxResponse { - pub tx_id: String, - pub to: Address, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub data: Option, - #[serde(with = "crate::serde_utils::decimal_u256")] - pub value: U256, - #[serde(with = "crate::serde_utils::decimal_u256")] - pub gas_limit: U256, - pub nonce: u64, - - // Sent tx data - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tx_hash: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateRelayerRequest { - pub name: String, - pub chain_id: u64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateRelayerResponse { - pub relayer_id: String, - pub address: Address, -} diff --git a/src/server/routes.rs b/src/server/routes.rs index a61610b..4675703 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -1 +1,3 @@ pub mod network; +pub mod relayer; +pub mod transaction; diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs new file mode 100644 index 0000000..636c844 --- /dev/null +++ b/src/server/routes/relayer.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use axum::extract::{Json, Path, State}; +use ethers::signers::Signer; +use ethers::types::Address; +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use crate::app::App; +use crate::server::ApiError; +use crate::types::{RelayerInfo, RelayerUpdate}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateRelayerRequest { + pub name: String, + pub chain_id: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateRelayerResponse { + pub relayer_id: String, + pub address: Address, +} + +pub async fn create_relayer( + State(app): State>, + Json(req): Json, +) -> Result, ApiError> { + let (key_id, signer) = app.keys_source.new_signer().await?; + + let address = signer.address(); + + let relayer_id = uuid::Uuid::new_v4(); + let relayer_id = relayer_id.to_string(); + + app.db + .create_relayer(&relayer_id, &req.name, req.chain_id, &key_id, address) + .await?; + + Ok(Json(CreateRelayerResponse { + relayer_id, + address, + })) +} + +pub async fn update_relayer( + State(app): State>, + Path(relayer_id): Path, + Json(req): Json, +) -> Result<(), ApiError> { + app.db.update_relayer(&relayer_id, &req).await?; + + Ok(()) +} + +pub async fn get_relayer( + State(app): State>, + Path(relayer_id): Path, +) -> Result, ApiError> { + let relayer_info = app.db.get_relayer(&relayer_id).await?; + + Ok(Json(relayer_info)) +} diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs new file mode 100644 index 0000000..224f25b --- /dev/null +++ b/src/server/routes/transaction.rs @@ -0,0 +1,110 @@ +use std::sync::Arc; + +use axum::extract::{Json, Path, State}; +use axum::TypedHeader; +use ethers::types::{Address, Bytes, H256, U256}; +use eyre::Result; +use serde::{Deserialize, Serialize}; + +use crate::app::App; +use crate::db::TxStatus; +use crate::server::middleware::AuthorizedRelayer; +use crate::server::ApiError; +use crate::types::TransactionPriority; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SendTxRequest { + pub relayer_id: String, + pub to: Address, + #[serde(with = "crate::serde_utils::decimal_u256")] + pub value: U256, + #[serde(default)] + pub data: Option, + #[serde(with = "crate::serde_utils::decimal_u256")] + pub gas_limit: U256, + #[serde(default)] + pub priority: TransactionPriority, + #[serde(default)] + pub tx_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SendTxResponse { + pub tx_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetTxResponse { + pub tx_id: String, + pub to: Address, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(with = "crate::serde_utils::decimal_u256")] + pub value: U256, + #[serde(with = "crate::serde_utils::decimal_u256")] + pub gas_limit: U256, + pub nonce: u64, + + // Sent tx data + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tx_hash: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +pub async fn send_tx( + State(app): State>, + TypedHeader(authorized_relayer): TypedHeader, + Json(req): Json, +) -> Result, ApiError> { + if !authorized_relayer.is_authorized(&req.relayer_id) { + return Err(ApiError::Unauthorized); + } + + let tx_id = if let Some(id) = req.tx_id { + id + } else { + uuid::Uuid::new_v4().to_string() + }; + + app.db + .create_transaction( + &tx_id, + req.to, + req.data.as_ref().map(|d| &d[..]).unwrap_or(&[]), + req.value, + req.gas_limit, + req.priority, + &req.relayer_id, + ) + .await?; + + Ok(Json(SendTxResponse { tx_id })) +} + +pub async fn get_tx( + State(app): State>, + Path(tx_id): Path, +) -> Result, ApiError> { + let tx = app.db.read_tx(&tx_id).await?.ok_or(ApiError::MissingTx)?; + + let get_tx_response = GetTxResponse { + tx_id: tx.tx_id, + to: tx.to.0, + data: if tx.data.is_empty() { + None + } else { + Some(tx.data.into()) + }, + value: tx.value.0, + gas_limit: tx.gas_limit.0, + nonce: tx.nonce, + tx_hash: tx.tx_hash.map(|h| h.0), + status: tx.status, + }; + + Ok(Json(get_tx_response)) +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index e183663..21da184 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -26,6 +26,21 @@ use tracing_subscriber::EnvFilter; pub type AppMiddleware = SignerMiddleware>, LocalWallet>; +#[allow(unused_imports)] +pub mod prelude { + pub use std::time::Duration; + + pub use ethers::providers::Middleware; + pub use ethers::types::{Eip1559TransactionRequest, U256}; + pub use ethers::utils::parse_units; + pub use service::server::routes::relayer::{ + CreateRelayerRequest, CreateRelayerResponse, + }; + pub use service::server::routes::transaction::SendTxRequest; + + pub use super::*; +} + pub const DEFAULT_ANVIL_ACCOUNT: Address = H160(hex_literal::hex!( "f39Fd6e51aad88F6F4ce6aB8827279cffFb92266" )); diff --git a/tests/create_relayer.rs b/tests/create_relayer.rs index e8ab75e..e71732a 100644 --- a/tests/create_relayer.rs +++ b/tests/create_relayer.rs @@ -1,11 +1,7 @@ -use std::time::Duration; - -use service::server::data::{CreateRelayerRequest, CreateRelayerResponse}; - -use crate::common::*; - mod common; +use crate::common::prelude::*; + const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); #[tokio::test] diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 4ca70cd..cc5157c 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -1,15 +1,6 @@ -use std::time::Duration; - -use ethers::providers::Middleware; -use ethers::types::{Eip1559TransactionRequest, U256}; -use ethers::utils::parse_units; -use service::server::data::{ - CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, -}; - mod common; -use crate::common::*; +use crate::common::prelude::*; const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); diff --git a/tests/send_tx.rs b/tests/send_tx.rs index da4ece5..66112e6 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -1,15 +1,6 @@ -use std::time::Duration; - -use ethers::providers::Middleware; -use ethers::types::{Eip1559TransactionRequest, U256}; -use ethers::utils::parse_units; -use service::server::data::{ - CreateRelayerRequest, CreateRelayerResponse, SendTxRequest, -}; - mod common; -use crate::common::*; +use crate::common::prelude::*; const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); From d76b44aff5cb2a6b834b6a1c143350204c2b78dd Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 11:56:07 +0100 Subject: [PATCH 016/135] Fix manual test --- manual_test_gas_prices.nu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manual_test_gas_prices.nu b/manual_test_gas_prices.nu index ca5ec70..ca65b0e 100644 --- a/manual_test_gas_prices.nu +++ b/manual_test_gas_prices.nu @@ -1,6 +1,7 @@ # Setup dependencies in different terminals: # DB -psql postgres://postgres:postgres@127.0.0.1:5432/database +docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 postgres +# Can connect to using psql postgres://postgres:postgres@127.0.0.1:5432/database # Nodes anvil --chain-id 31337 -p 8545 --block-time 1 From bcdbae326a95648d56a9604714b163522d81815d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 11:59:15 +0100 Subject: [PATCH 017/135] RPC Access --- TODO.md | 2 +- crates/fake-rpc/src/lib.rs | 2 +- src/server.rs | 8 ++-- src/server/middleware/auth_middleware.rs | 1 + src/server/routes/network.rs | 3 ++ src/server/routes/relayer.rs | 56 ++++++++++++++++++++++++ src/server/routes/transaction.rs | 2 + tests/rpc_access.rs | 40 +++++++++++++++++ 8 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/rpc_access.rs diff --git a/TODO.md b/TODO.md index e5d91c0..344dd82 100644 --- a/TODO.md +++ b/TODO.md @@ -4,7 +4,7 @@ 2. [ ] Per network settings (max blocks age/number - for pruning) 3. [x] Per relayer settings (i.e. max inflight txs, max gas price) 4. [ ] Multiple RPCs support -5. IN PROGRESS [ ] RPC Access +5. [x] RPC Access 6. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) 7. [x] Transaction priority 8. [ ] Metrics diff --git a/crates/fake-rpc/src/lib.rs b/crates/fake-rpc/src/lib.rs index 1ec8738..1e252b6 100644 --- a/crates/fake-rpc/src/lib.rs +++ b/crates/fake-rpc/src/lib.rs @@ -63,7 +63,7 @@ struct JsonRpcReq { pub jsonrpc: String, pub method: String, #[serde(default)] - pub params: Vec, + pub params: Value, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/src/server.rs b/src/server.rs index b04f2fc..24d4e62 100644 --- a/src/server.rs +++ b/src/server.rs @@ -7,7 +7,9 @@ use axum::Router; use hyper::server::conn::AddrIncoming; use thiserror::Error; -use self::routes::relayer::{create_relayer, get_relayer, update_relayer}; +use self::routes::relayer::{ + create_relayer, get_relayer, relayer_rpc, update_relayer, +}; use self::routes::transaction::{get_tx, send_tx}; use crate::app::App; @@ -75,8 +77,8 @@ pub async fn spawn_server( let relayer_routes = Router::new() .route("/", post(create_relayer)) - .route("/:relayer_id", post(update_relayer)) - .route("/:relayer_id", get(get_relayer)) + .route("/:relayer_id", post(update_relayer).get(get_relayer)) + .route("/:relayer_id/rpc", post(relayer_rpc)) .with_state(app.clone()); let network_routes = Router::new() diff --git a/src/server/middleware/auth_middleware.rs b/src/server/middleware/auth_middleware.rs index 8935450..7c79c28 100644 --- a/src/server/middleware/auth_middleware.rs +++ b/src/server/middleware/auth_middleware.rs @@ -13,6 +13,7 @@ use crate::server::ApiError; pub const AUTHORIZED_RELAYER: &str = "x-authorized-relayer"; static HEADER_NAME: HeaderName = HeaderName::from_static(AUTHORIZED_RELAYER); +#[derive(Debug, Clone)] pub enum AuthorizedRelayer { Named(String), Any, diff --git a/src/server/routes/network.rs b/src/server/routes/network.rs index 3a6c8b7..2c0b349 100644 --- a/src/server/routes/network.rs +++ b/src/server/routes/network.rs @@ -27,6 +27,7 @@ pub struct NetworkInfo { pub ws_rpc: String, } +#[tracing::instrument(skip(app))] pub async fn create_network( State(app): State>, Path(chain_id): Path, @@ -57,6 +58,7 @@ pub async fn create_network( Ok(()) } +#[tracing::instrument(skip(_app))] pub async fn _get_network( State(_app): State>, Path(_chain_id): Path, @@ -64,6 +66,7 @@ pub async fn _get_network( "Hello, World!" } +#[tracing::instrument(skip(_app))] pub async fn _get_networks( State(_app): State>, Path(_chain_id): Path, diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 636c844..098de23 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -5,6 +5,7 @@ use ethers::signers::Signer; use ethers::types::Address; use eyre::Result; use serde::{Deserialize, Serialize}; +use serde_json::Value; use crate::app::App; use crate::server::ApiError; @@ -24,6 +25,31 @@ pub struct CreateRelayerResponse { pub address: Address, } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcRequest { + pub id: i32, + pub method: String, + #[serde(default)] + pub params: Value, + pub jsonrpc: JsonRpcVersion, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcResponse { + pub id: i32, + pub result: Value, + pub jsonrpc: JsonRpcVersion, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum JsonRpcVersion { + #[serde(rename = "2.0")] + V2, +} + +#[tracing::instrument(skip(app))] pub async fn create_relayer( State(app): State>, Json(req): Json, @@ -45,6 +71,7 @@ pub async fn create_relayer( })) } +#[tracing::instrument(skip(app))] pub async fn update_relayer( State(app): State>, Path(relayer_id): Path, @@ -55,6 +82,7 @@ pub async fn update_relayer( Ok(()) } +#[tracing::instrument(skip(app))] pub async fn get_relayer( State(app): State>, Path(relayer_id): Path, @@ -63,3 +91,31 @@ pub async fn get_relayer( Ok(Json(relayer_info)) } + +#[tracing::instrument(skip(app))] +pub async fn relayer_rpc( + State(app): State>, + Path(relayer_id): Path, + Json(req): Json, +) -> Result, ApiError> { + let relayer_info = app.db.get_relayer(&relayer_id).await?; + + // TODO: Cache? + let http_provider = app.http_provider(relayer_info.chain_id).await?; + let url = http_provider.url(); + + let response = reqwest::Client::new() + .post(url.clone()) + .json(&req) + .send() + .await + .map_err(|err| { + eyre::eyre!("Error sending request to {}: {}", url, err) + })?; + + let response: Value = response.json().await.map_err(|err| { + eyre::eyre!("Error parsing response from {}: {}", url, err) + })?; + + Ok(Json(response)) +} diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 224f25b..c051d46 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -55,6 +55,7 @@ pub struct GetTxResponse { pub status: Option, } +#[tracing::instrument(skip(app))] pub async fn send_tx( State(app): State>, TypedHeader(authorized_relayer): TypedHeader, @@ -85,6 +86,7 @@ pub async fn send_tx( Ok(Json(SendTxResponse { tx_id })) } +#[tracing::instrument(skip(app))] pub async fn get_tx( State(app): State>, Path(tx_id): Path, diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs new file mode 100644 index 0000000..a5b36a9 --- /dev/null +++ b/tests/rpc_access.rs @@ -0,0 +1,40 @@ +mod common; + +use ethers::prelude::*; +use url::Url; + +use crate::common::prelude::*; + +const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); + +#[tokio::test] +async fn rpc_access() -> eyre::Result<()> { + setup_tracing(); + + let (db_url, _db_container) = setup_db().await?; + let double_anvil = setup_double_anvil().await?; + + let (service, client) = + setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; + + let CreateRelayerResponse { relayer_id, .. } = client + .create_relayer(&CreateRelayerRequest { + name: "Test relayer".to_string(), + chain_id: DEFAULT_ANVIL_CHAIN_ID, + }) + .await?; + + let rpc_url = + format!("http://{}/1/relayer/{relayer_id}/rpc", service.local_addr()); + + let provider = Provider::new(Http::new(rpc_url.parse::()?)); + + let latest_block_number = provider.get_block_number().await?; + + let very_future_block = latest_block_number + 1000; + let very_future_block = provider.get_block(very_future_block).await?; + + assert!(very_future_block.is_none()); + + Ok(()) +} From 4b8e4f107330b0bbababefebe6c592b3c4e1decb Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 12:21:18 +0100 Subject: [PATCH 018/135] Minor fixes --- manual_test.nu | 2 +- src/tasks/escalate.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/manual_test.nu b/manual_test.nu index 0b045fa..0b178e4 100644 --- a/manual_test.nu +++ b/manual_test.nu @@ -8,7 +8,7 @@ http post -t application/json $"($txSitter)/1/network/31337" { } echo "Creating relayer" -let relayer = http post -t application/json $"($txSitter)/1/relayer/create" { "name": "My Relayer", "chainId": 31337 } +let relayer = http post -t application/json $"($txSitter)/1/relayer" { "name": "My Relayer", "chainId": 31337 } echo "Funding relayer" cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 05f65cd..5a2b7ea 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -47,7 +47,6 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { * increased_gas_price_percentage / factor; - // TODO: Add limits per network let max_priority_fee_per_gas = tx.initial_max_priority_fee_per_gas.0 + max_priority_fee_per_gas_increase; From 00356f7327c744af84b8347b3546e5b302ddd591 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 15:43:09 +0100 Subject: [PATCH 019/135] Parallelize broadcast per relayer + error handling --- Cargo.lock | 10 +++ Cargo.toml | 1 + src/db.rs | 3 +- src/tasks/broadcast.rs | 173 ++++++++++++++++++++++++++--------------- 4 files changed, 123 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5eb388e..16d6ab5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2376,6 +2376,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -3837,6 +3846,7 @@ dependencies = [ "humantime-serde", "hyper", "indoc", + "itertools 0.12.0", "num-bigint", "postgres-docker-utils", "rand", diff --git a/Cargo.toml b/Cargo.toml index 2a680e5..97cd4e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ num-bigint = "0.4.4" bigdecimal = "0.4.2" spki = "0.7.2" async-trait = "0.1.74" +itertools = "0.12.0" # Internal postgres-docker-utils = { path = "crates/postgres-docker-utils" } diff --git a/src/db.rs b/src/db.rs index 74d0384..49b66d8 100644 --- a/src/db.rs +++ b/src/db.rs @@ -196,7 +196,8 @@ impl Database { LEFT JOIN sent_transactions s ON (t.id = s.tx_id) INNER JOIN relayers r ON (t.relayer_id = r.id) WHERE s.tx_id IS NULL - AND (t.nonce - r.current_nonce < r.max_inflight_txs); + AND (t.nonce - r.current_nonce < r.max_inflight_txs) + ORDER BY r.id, t.nonce ASC "#, ) .fetch_all(&self.pool) diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 5ec9e4f..47c6944 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -6,83 +7,129 @@ use ethers::types::transaction::eip2718::TypedTransaction; use ethers::types::transaction::eip2930::AccessList; use ethers::types::{Address, Eip1559TransactionRequest, NameOrAddress}; use eyre::ContextCompat; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use itertools::Itertools; use crate::app::App; use crate::broadcast_utils::{ calculate_gas_fees_from_estimates, calculate_max_base_fee_per_gas, should_send_transaction, }; +use crate::db::UnsentTx; pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { loop { - let txs = app.db.get_unsent_txs().await?; + let mut txs = app.db.get_unsent_txs().await?; - // TODO: Parallelize per chain id? - for tx in txs { - tracing::info!(tx.id, "Sending tx"); + txs.sort_unstable_by_key(|tx| tx.relayer_id.clone()); - if !should_send_transaction(&app, &tx.relayer_id).await? { - tracing::warn!(id = tx.id, "Skipping transaction broadcast"); - continue; - } + let txs_by_relayer = + txs.into_iter().group_by(|tx| tx.relayer_id.clone()); + + let txs_by_relayer: HashMap<_, _> = txs_by_relayer + .into_iter() + .map(|(relayer_id, txs)| { + let mut txs = txs.collect_vec(); + + txs.sort_unstable_by_key(|tx| tx.nonce); + + (relayer_id, txs) + }) + .collect(); + + let mut futures = FuturesUnordered::new(); - let middleware = app - .signer_middleware(tx.chain_id, tx.key_id.clone()) - .await?; - - let fees = app - .db - .get_latest_block_fees_by_chain_id(tx.chain_id) - .await? - .context("Missing block")?; - - let max_base_fee_per_gas = - calculate_max_base_fee_per_gas(&fees.fee_estimates)?; - - let (max_fee_per_gas, max_priority_fee_per_gas) = - calculate_gas_fees_from_estimates( - &fees.fee_estimates, - tx.priority.to_percentile_index(), - max_base_fee_per_gas, - )?; - - let eip1559_tx = Eip1559TransactionRequest { - from: None, - to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), - gas: Some(tx.gas_limit.0), - value: Some(tx.value.0), - data: Some(tx.data.into()), - nonce: Some(tx.nonce.into()), - access_list: AccessList::default(), - max_priority_fee_per_gas: Some(max_priority_fee_per_gas), - max_fee_per_gas: Some(max_fee_per_gas), - chain_id: Some(tx.chain_id.into()), - }; - - tracing::debug!(?eip1559_tx, "Sending tx"); - - // TODO: Is it possible that we send a tx but don't store it in the DB? - // TODO: Be smarter about error handling - a tx can fail to be sent - // e.g. because the relayer is out of funds - // but we don't want to retry it forever - let pending_tx = middleware - .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await?; - - let tx_hash = pending_tx.tx_hash(); - - tracing::info!(?tx.id, ?tx_hash, "Tx sent successfully"); - - app.db - .insert_tx_broadcast( - &tx.id, - tx_hash, - max_fee_per_gas, - max_priority_fee_per_gas, - ) - .await?; + for (relayer_id, txs) in txs_by_relayer { + futures.push(broadcast_relayer_txs(&app, relayer_id, txs)); + } + + while let Some(result) = futures.next().await { + if let Err(err) = result { + tracing::error!(error = ?err, "Failed broadcasting txs"); + } } tokio::time::sleep(Duration::from_secs(1)).await; } } + +#[tracing::instrument(skip(app, txs))] +async fn broadcast_relayer_txs( + app: &App, + relayer_id: String, + txs: Vec, +) -> Result<(), eyre::Error> { + if txs.is_empty() { + return Ok(()); + } + + tracing::info!(relayer_id, num_txs = txs.len(), "Broadcasting relayer txs"); + + for tx in txs { + tracing::info!(tx.id, "Sending tx"); + + if !should_send_transaction(app, &tx.relayer_id).await? { + tracing::warn!(id = tx.id, "Skipping transaction broadcast"); + continue; + } + + let middleware = app + .signer_middleware(tx.chain_id, tx.key_id.clone()) + .await?; + + let fees = app + .db + .get_latest_block_fees_by_chain_id(tx.chain_id) + .await? + .context("Missing block")?; + + let max_base_fee_per_gas = + calculate_max_base_fee_per_gas(&fees.fee_estimates)?; + + let (max_fee_per_gas, max_priority_fee_per_gas) = + calculate_gas_fees_from_estimates( + &fees.fee_estimates, + tx.priority.to_percentile_index(), + max_base_fee_per_gas, + )?; + + let eip1559_tx = Eip1559TransactionRequest { + from: None, + to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), + gas: Some(tx.gas_limit.0), + value: Some(tx.value.0), + data: Some(tx.data.into()), + nonce: Some(tx.nonce.into()), + access_list: AccessList::default(), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + max_fee_per_gas: Some(max_fee_per_gas), + chain_id: Some(tx.chain_id.into()), + }; + + tracing::debug!(?eip1559_tx, "Sending tx"); + + // TODO: Is it possible that we send a tx but don't store it in the DB? + // TODO: Be smarter about error handling - a tx can fail to be sent + // e.g. because the relayer is out of funds + // but we don't want to retry it forever + let pending_tx = middleware + .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) + .await?; + + let tx_hash = pending_tx.tx_hash(); + + tracing::info!(?tx.id, ?tx_hash, "Tx sent successfully"); + + app.db + .insert_tx_broadcast( + &tx.id, + tx_hash, + max_fee_per_gas, + max_priority_fee_per_gas, + ) + .await?; + } + + Ok(()) +} From ffb3b67b99dd52ba0c7b75e08f2f0bb245c8cf42 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 16:09:08 +0100 Subject: [PATCH 020/135] Expose unsent status --- src/server/routes/transaction.rs | 42 +++++++++++++++++++++++++++++--- src/tasks/broadcast.rs | 13 +++++++++- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index c051d46..5b4b61b 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -51,8 +51,22 @@ pub struct GetTxResponse { // Sent tx data #[serde(default, skip_serializing_if = "Option::is_none")] pub tx_hash: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, + pub status: GetTxResponseStatus, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(rename_all = "camelCase")] +pub enum GetTxResponseStatus { + TxStatus(TxStatus), + Unsent(UnsentStatus), +} + +// We need this status as a separate enum to avoid manual serialization +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum UnsentStatus { + Unsent, } #[tracing::instrument(skip(app))] @@ -105,8 +119,30 @@ pub async fn get_tx( gas_limit: tx.gas_limit.0, nonce: tx.nonce, tx_hash: tx.tx_hash.map(|h| h.0), - status: tx.status, + status: tx + .status + .map(GetTxResponseStatus::TxStatus) + .unwrap_or(GetTxResponseStatus::Unsent(UnsentStatus::Unsent)), }; Ok(Json(get_tx_response)) } + +#[cfg(test)] +mod tests { + use test_case::test_case; + + use super::*; + + #[test_case(GetTxResponseStatus::TxStatus(TxStatus::Pending) => "pending")] + #[test_case(GetTxResponseStatus::Unsent(UnsentStatus::Unsent) => "unsent")] + fn get_tx_response_status_serialization( + status: GetTxResponseStatus, + ) -> &'static str { + let json = serde_json::to_string(&status).unwrap(); + + let s = json.trim_start_matches("\"").trim_end_matches("\""); + + Box::leak(s.to_owned().into_boxed_str()) + } +} diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 47c6944..f5d9fdb 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -115,7 +115,18 @@ async fn broadcast_relayer_txs( // but we don't want to retry it forever let pending_tx = middleware .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await?; + .await; + + let pending_tx = match pending_tx { + Ok(pending_tx) => { + tracing::info!(?pending_tx, "Tx sent successfully"); + pending_tx + } + Err(err) => { + tracing::error!(error = ?err, "Failed to send tx"); + continue; + } + }; let tx_hash = pending_tx.tx_hash(); From e87552cbb86a03bbc8c28493464036a1778d5b91 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 17:43:54 +0100 Subject: [PATCH 021/135] Update TODO --- TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index 344dd82..f8e2d59 100644 --- a/TODO.md +++ b/TODO.md @@ -17,7 +17,7 @@ 11. [x] Parallelization: 1. [x] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating - 3. [ ] Parallelize broadcast per chain id + 3. [x] Parallelize broadcast per ~chain id~ relayer id 12. [x] No need to insert all block txs into DB 13. [x] Prune block info 14. [ ] Authentication From a49cdcd025177bf7ab67af739e1847ffca1e345c Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 17:44:38 +0100 Subject: [PATCH 022/135] Add item to TODO --- TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index f8e2d59..8024ba7 100644 --- a/TODO.md +++ b/TODO.md @@ -21,4 +21,4 @@ 12. [x] No need to insert all block txs into DB 13. [x] Prune block info 14. [ ] Authentication - +15. [ ] Plug block holes - we can periodically fetch the list of known blocks for a given chain and find and fetch any missing ones from the RPC From b4004a804ba9ba34de5c1a905f5f3562ee4a38ce Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 29 Nov 2023 17:59:46 +0100 Subject: [PATCH 023/135] Consistent formatting of SQL --- db/migrations/001_init.sql | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index b0c75d1..d32ae0a 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -6,14 +6,14 @@ CREATE TYPE transaction_priority AS ENUM ('slowest', 'slow', 'regular', 'fast', CREATE TABLE networks ( chain_id BIGINT PRIMARY KEY, - name VARCHAR(255) NOT NULL + name VARCHAR(255) NOT NULL ); CREATE TABLE rpcs ( - id BIGSERIAL PRIMARY KEY, + id BIGSERIAL PRIMARY KEY, chain_id BIGINT NOT NULL REFERENCES networks(chain_id), - url VARCHAR(255) NOT NULL, - kind rpc_kind NOT NULL + url VARCHAR(255) NOT NULL, + kind rpc_kind NOT NULL ); CREATE TABLE relayers ( @@ -51,12 +51,12 @@ CREATE TABLE transactions ( -- Sent transaction attempts CREATE TABLE tx_hashes ( - tx_hash BYTEA PRIMARY KEY, - tx_id VARCHAR(255) NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - max_fee_per_gas BYTEA NOT NULL, + tx_hash BYTEA PRIMARY KEY, + tx_id VARCHAR(255) NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + max_fee_per_gas BYTEA NOT NULL, max_priority_fee_per_gas BYTEA NOT NULL, - escalated BOOL NOT NULL DEFAULT FALSE + escalated BOOL NOT NULL DEFAULT FALSE ); -- Dynamic tx data & data used for escalations @@ -78,15 +78,15 @@ CREATE TABLE sent_transactions ( CREATE TABLE blocks ( block_number BIGINT NOT NULL, - chain_id BIGINT NOT NULL, - timestamp TIMESTAMPTZ NOT NULL, + chain_id BIGINT NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, PRIMARY KEY (block_number, chain_id) ); CREATE TABLE block_txs ( block_number BIGINT NOT NULL, - chain_id BIGINT NOT NULL, - tx_hash BYTEA NOT NULL, + chain_id BIGINT NOT NULL, + tx_hash BYTEA NOT NULL, PRIMARY KEY (block_number, chain_id, tx_hash), FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE, FOREIGN KEY (tx_hash) REFERENCES tx_hashes (tx_hash) @@ -94,8 +94,8 @@ CREATE TABLE block_txs ( CREATE TABLE block_fees ( block_number BIGINT NOT NULL, - chain_id BIGINT NOT NULL, - gas_price NUMERIC(78, 0) NOT NULL, + chain_id BIGINT NOT NULL, + gas_price NUMERIC(78, 0) NOT NULL, fee_estimate JSON NOT NULL, PRIMARY KEY (block_number, chain_id), FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE From 785dbe6646981294bebc0ad9a425639f27b16017 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 10:28:15 +0100 Subject: [PATCH 024/135] Minor cleanup --- config.toml | 3 --- db/migrations/001_init.sql | 1 + src/db.rs | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config.toml b/config.toml index e4ca7dc..900f8a9 100644 --- a/config.toml +++ b/config.toml @@ -5,9 +5,6 @@ escalation_interval = "1m" host = "127.0.0.1:3000" disable_auth = true -[rpc] -rpcs = ["http://127.0.0.1:8545"] - [database] connection_string = "postgres://postgres:postgres@127.0.0.1:5432/database" diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index d32ae0a..f894e50 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -30,6 +30,7 @@ CREATE TABLE relayers ( -- Settings max_inflight_txs BIGINT NOT NULL DEFAULT 5, + -- e.g. [ { "chainId": 123, "value": "0x123"} ] gas_limits JSON NOT NULL DEFAULT '[]', -- Time keeping fields diff --git a/src/db.rs b/src/db.rs index 49b66d8..51b5271 100644 --- a/src/db.rs +++ b/src/db.rs @@ -459,7 +459,7 @@ impl Database { // but none of the associated tx hashes are present in block txs let items: Vec<(String,)> = sqlx::query_as( r#" - WITH fdsa AS ( + WITH reorg_candidates AS ( SELECT t.id, h.tx_hash, bt.chain_id FROM transactions t JOIN sent_transactions s ON t.id = s.tx_id @@ -468,7 +468,7 @@ impl Database { WHERE s.status = $1 ) SELECT t.id - FROM fdsa t + FROM reorg_candidates t GROUP BY t.id HAVING COUNT(t.chain_id) = 0 "#, From 4201210dba082546f03869b87120bfb87a5a2cc3 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 11:59:35 +0100 Subject: [PATCH 025/135] WIP --- src/server/routes/transaction.rs | 2 +- src/task_runner.rs | 40 +++----------------------------- src/tasks/broadcast.rs | 14 +++++++---- src/tasks/escalate.rs | 13 ++++++++++- 4 files changed, 25 insertions(+), 44 deletions(-) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 5b4b61b..364a082 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -141,7 +141,7 @@ mod tests { ) -> &'static str { let json = serde_json::to_string(&status).unwrap(); - let s = json.trim_start_matches("\"").trim_end_matches("\""); + let s = json.trim_start_matches('\"').trim_end_matches('\"'); Box::leak(s.to_owned().into_boxed_str()) } diff --git a/src/task_runner.rs b/src/task_runner.rs index e2d8e4d..2e4e096 100644 --- a/src/task_runner.rs +++ b/src/task_runner.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use futures::Future; +use tokio::task::JoinHandle; const FAILURE_MONITORING_PERIOD: Duration = Duration::from_secs(60); @@ -19,7 +20,7 @@ impl TaskRunner where T: Send + Sync + 'static, { - pub fn add_task(&self, label: S, task: C) + pub fn add_task(&self, label: S, task: C) -> JoinHandle<()> where S: ToString, C: Fn(Arc) -> F + Send + Sync + 'static, @@ -50,42 +51,7 @@ where break; } } - }); - } - - pub fn add_task_with_args(&self, label: S, task: C, args: A) - where - A: Clone + Send + 'static, - S: ToString, - C: Fn(Arc, A) -> F + Send + Sync + 'static, - F: Future> + Send + 'static, - { - let app = self.app.clone(); - let label = label.to_string(); - - tokio::spawn(async move { - let mut failures = vec![]; - - loop { - tracing::info!(label, "Running task"); - - let result = task(app.clone(), args.clone()).await; - - if let Err(err) = result { - tracing::error!(label, error = ?err, "Task failed"); - - failures.push(Instant::now()); - let backoff = determine_backoff(&failures); - - tokio::time::sleep(backoff).await; - - prune_failures(&mut failures); - } else { - tracing::info!(label, "Task finished"); - break; - } - } - }); + }) } } diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index f5d9fdb..2bca6d7 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -66,14 +66,18 @@ async fn broadcast_relayer_txs( tracing::info!(relayer_id, num_txs = txs.len(), "Broadcasting relayer txs"); + if !should_send_transaction(app, &relayer_id).await? { + tracing::warn!( + relayer_id = relayer_id, + "Skipping transaction broadcasts" + ); + + return Ok(()); + } + for tx in txs { tracing::info!(tx.id, "Sending tx"); - if !should_send_transaction(app, &tx.relayer_id).await? { - tracing::warn!(id = tx.id, "Skipping transaction broadcast"); - continue; - } - let middleware = app .signer_middleware(tx.chain_id, tx.key_id.clone()) .await?; diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 5a2b7ea..aea2178 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -69,7 +69,18 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { let pending_tx = middleware .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await?; + .await; + + let pending_tx = match pending_tx { + Ok(pending_tx) => { + tracing::info!(?pending_tx, "Tx sent successfully"); + pending_tx + } + Err(err) => { + tracing::error!(error = ?err, "Failed to send tx"); + continue; + } + }; let tx_hash = pending_tx.tx_hash(); From 73e1426f4389bd1df12c18290945fb89d2b4df48 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 12:13:27 +0100 Subject: [PATCH 026/135] Add TODO item --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index 8024ba7..002d7c8 100644 --- a/TODO.md +++ b/TODO.md @@ -22,3 +22,4 @@ 13. [x] Prune block info 14. [ ] Authentication 15. [ ] Plug block holes - we can periodically fetch the list of known blocks for a given chain and find and fetch any missing ones from the RPC +16. [ ] Find missing txs - sometimes a transaction can be sent but not saved in the DB. On every block we should fetch all the txs (not just hashes) and find txs coming from our relayer addresses. This way we can find missing transactions. From 2d124450d8b06005f3dc454ac7e55d9ba77d498f Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 12:15:08 +0100 Subject: [PATCH 027/135] Add TODO item --- TODO.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.md b/TODO.md index 002d7c8..9b64f49 100644 --- a/TODO.md +++ b/TODO.md @@ -23,3 +23,5 @@ 14. [ ] Authentication 15. [ ] Plug block holes - we can periodically fetch the list of known blocks for a given chain and find and fetch any missing ones from the RPC 16. [ ] Find missing txs - sometimes a transaction can be sent but not saved in the DB. On every block we should fetch all the txs (not just hashes) and find txs coming from our relayer addresses. This way we can find missing transactions. +17. [ ] Smarter broadcast error handling - we shouldn't constantly attempt to broadcast the same tx if it's failing (e.g. because relayer address is out of funds). + From 53c43dca2977bdb3fd0f2f45f569f87dea20ac80 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 12:28:31 +0100 Subject: [PATCH 028/135] misc --- README.md | 18 +++++++++++------- src/tasks/broadcast.rs | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f6198c8..960855d 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,6 @@ The Tx Sitter can be configured in 2 ways: host = "127.0.0.1:3000" disable_auth = true - [rpc] - rpcs = ["http://127.0.0.1:8545"] - [database] connection_string = "postgres://postgres:postgres@127.0.0.1:5432/database" @@ -30,7 +27,6 @@ The Tx Sitter can be configured in 2 ways: TX_SITTER__SERVICE__ESCALATION_INTERVAL="1m" TX_SITTER__SERVER__HOST="127.0.0.1:3000" TX_SITTER__SERVER__DISABLE_AUTH="true" - TX_SITTER_EXT__RPC__RPCS="http://127.0.0.1:8545" TX_SITTER__DATABASE__CONNECTION_STRING="postgres://postgres:postgres@127.0.0.1:5432/database" TX_SITTER__KEYS__KIND="local" ``` @@ -47,6 +43,14 @@ This will use the `config.toml` configuration. If you have [nushell](https://www.nushell.sh/) installed, `nu manual_test.nu` can be run to execute a basic test. ## Running tests -While you obviously can run tests with `cargo test --workspace` some tests take quite a long time (due to spinning up an anvil node, sending txs, etc.). - -Therefore I recommend [cargo-nextest](https://nexte.st/) as it runs all the tests in parallel. Once installed `cargo nextest run --workspace` can be used instead. +While you obviously can run tests with +``` +cargo test --workspace +``` +some tests take quite a long time (due to spinning up an anvil node, sending txs, etc.). + +Therefore I recommend [cargo-nextest](https://nexte.st/) as it runs all the tests in parallel. Once installed +``` +cargo nextest run --workspace +``` +can be used instead. diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 2bca6d7..c99e9cb 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -86,7 +86,7 @@ async fn broadcast_relayer_txs( .db .get_latest_block_fees_by_chain_id(tx.chain_id) .await? - .context("Missing block")?; + .context("Missing block fees")?; let max_base_fee_per_gas = calculate_max_base_fee_per_gas(&fees.fee_estimates)?; From 1027cf2637f801a8def9c697254d3772a9e396d6 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 30 Nov 2023 12:49:27 +0100 Subject: [PATCH 029/135] Section TODO --- TODO.md | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/TODO.md b/TODO.md index 9b64f49..87fb308 100644 --- a/TODO.md +++ b/TODO.md @@ -1,27 +1,30 @@ # TODO - -1. [x] Handling reorgs -2. [ ] Per network settings (max blocks age/number - for pruning) -3. [x] Per relayer settings (i.e. max inflight txs, max gas price) +1. [ ] Per network settings (max blocks age/number - for pruning) 4. [ ] Multiple RPCs support -5. [x] RPC Access -6. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) -7. [x] Transaction priority -8. [ ] Metrics -9. [ ] Tracing (add telemetry-batteries) -10. [ ] Automated testing +5. [ ] Telemtry (add telemetry-batteries) + 1. [ ] Metrics + 2. [ ] Tracing +6. [ ] Automated testing 1. [x] Basic 2. [ ] Basic with contracts 3. [ ] Escalation testing 4. [ ] Reorg testing (how?!?) -11. [x] Parallelization: - 1. [x] Parallelize block indexing - depends on per network settings - 2. [x] Parallelize nonce updating - 3. [x] Parallelize broadcast per ~chain id~ relayer id -12. [x] No need to insert all block txs into DB -13. [x] Prune block info -14. [ ] Authentication -15. [ ] Plug block holes - we can periodically fetch the list of known blocks for a given chain and find and fetch any missing ones from the RPC -16. [ ] Find missing txs - sometimes a transaction can be sent but not saved in the DB. On every block we should fetch all the txs (not just hashes) and find txs coming from our relayer addresses. This way we can find missing transactions. -17. [ ] Smarter broadcast error handling - we shouldn't constantly attempt to broadcast the same tx if it's failing (e.g. because relayer address is out of funds). +7. [ ] Plug block holes - we can periodically fetch the list of known blocks for a given chain and find and fetch any missing ones from the RPC +8. [ ] Find missing txs - sometimes a transaction can be sent but not saved in the DB. On every block we should fetch all the txs (not just hashes) and find txs coming from our relayer addresses. This way we can find missing transactions. +9. [ ] Smarter broadcast error handling - we shouldn't constantly attempt to broadcast the same tx if it's failing (e.g. because relayer address is out of funds). + +# IN PROGRESS +1. [ ] Design authentication +# DONE +1. [x] Parallelization: + 1. [x] Parallelize block indexing - depends on per network settings + 2. [x] Parallelize nonce updating + 3. [x] Parallelize broadcast per ~chain id~ relayer id +4. [x] No need to insert all block txs into DB +5. [x] Prune block info +6. [x] RPC Access +5. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) +6. [x] Transaction priority +7. [x] Handling reorgs +8. [x] Per relayer settings (i.e. max inflight txs, max gas price) From 7eecadb70b2b44a2321aec458d6cfdd529f810e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Fri, 1 Dec 2023 12:45:33 +0100 Subject: [PATCH 030/135] Dzejkop/api-keys (#3) --- Cargo.lock | 1 + Cargo.toml | 1 + TODO.md | 18 +-- config.toml | 2 +- db/migrations/001_init.sql | 5 + manual_test.nu | 25 +++- manual_test_gas_prices.nu | 69 ----------- src/api_key.rs | 118 ++++++++++++++++++ src/app.rs | 14 +++ src/client.rs | 73 ++++++++---- src/db.rs | 42 +++++++ src/lib.rs | 1 + src/server.rs | 18 ++- src/server/middleware.rs | 2 - src/server/middleware/auth_middleware.rs | 145 ----------------------- src/server/routes/relayer.rs | 29 ++++- src/server/routes/transaction.rs | 13 +- tests/rpc_access.rs | 7 +- tests/send_many_txs.rs | 22 ++-- tests/send_tx.rs | 22 ++-- 20 files changed, 345 insertions(+), 282 deletions(-) delete mode 100644 manual_test_gas_prices.nu create mode 100644 src/api_key.rs delete mode 100644 src/server/middleware/auth_middleware.rs diff --git a/Cargo.lock b/Cargo.lock index 16d6ab5..857248b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3830,6 +3830,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "axum", + "base64 0.21.5", "bigdecimal 0.4.2", "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 97cd4e0..1743dfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ bigdecimal = "0.4.2" spki = "0.7.2" async-trait = "0.1.74" itertools = "0.12.0" +base64 = "0.21.5" # Internal postgres-docker-utils = { path = "crates/postgres-docker-utils" } diff --git a/TODO.md b/TODO.md index 87fb308..16f0eeb 100644 --- a/TODO.md +++ b/TODO.md @@ -14,17 +14,17 @@ 9. [ ] Smarter broadcast error handling - we shouldn't constantly attempt to broadcast the same tx if it's failing (e.g. because relayer address is out of funds). # IN PROGRESS -1. [ ] Design authentication # DONE -1. [x] Parallelization: +2. [x] Parallelization: 1. [x] Parallelize block indexing - depends on per network settings 2. [x] Parallelize nonce updating 3. [x] Parallelize broadcast per ~chain id~ relayer id -4. [x] No need to insert all block txs into DB -5. [x] Prune block info -6. [x] RPC Access -5. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) -6. [x] Transaction priority -7. [x] Handling reorgs -8. [x] Per relayer settings (i.e. max inflight txs, max gas price) +3. [x] No need to insert all block txs into DB +4. [x] Prune block info +5. [x] RPC Access +6. [x] Cross-network dependencies (i.e. Optimism depends on L1 gas cost) +7. [x] Transaction priority +8. [x] Handling reorgs +9. [x] Per relayer settings (i.e. max inflight txs, max gas price) +10. [x] Authentication diff --git a/config.toml b/config.toml index 900f8a9..05b83a9 100644 --- a/config.toml +++ b/config.toml @@ -3,7 +3,7 @@ escalation_interval = "1m" [server] host = "127.0.0.1:3000" -disable_auth = true +disable_auth = false [database] connection_string = "postgres://postgres:postgres@127.0.0.1:5432/database" diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index f894e50..9889d47 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -101,3 +101,8 @@ CREATE TABLE block_fees ( PRIMARY KEY (block_number, chain_id), FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE ); + +CREATE TABLE api_keys ( + relayer_id CHAR(36) NOT NULL REFERENCES relayers(id) ON DELETE CASCADE, + key_hash BYTEA NOT NULL +); diff --git a/manual_test.nu b/manual_test.nu index 0b178e4..e22fda4 100644 --- a/manual_test.nu +++ b/manual_test.nu @@ -1,3 +1,19 @@ +## Setup dependencies in different terminals: +## DB +# docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 postgres +## Can connect to using psql postgres://postgres:postgres@127.0.0.1:5432/database + +## Nodes +# anvil --chain-id 31337 -p 8545 --block-time 1 +# anvil --chain-id 31338 -p 8546 --block-time 1 + +## TxSitter +# cargo watch -x run +## or just +# cargo run + +echo "Start" + let txSitter = "http://127.0.0.1:3000" let anvilSocket = "127.0.0.1:8545" @@ -10,11 +26,16 @@ http post -t application/json $"($txSitter)/1/network/31337" { echo "Creating relayer" let relayer = http post -t application/json $"($txSitter)/1/relayer" { "name": "My Relayer", "chainId": 31337 } +echo "Create api key" +let apiKey = http post $"($txSitter)/1/relayer/($relayer.relayerId)/key" "" + +$env.ETH_RPC_URL = $"($txSitter)/1/($apiKey.apiKey)/rpc" + echo "Funding relayer" cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' echo "Sending transaction" -let tx = http post -t application/json $"($txSitter)/1/tx/send" { +let tx = http post -t application/json $"($txSitter)/1/($apiKey.apiKey)/tx" { "relayerId": $relayer.relayerId, "to": $relayer.address, "value": "10", @@ -24,7 +45,7 @@ let tx = http post -t application/json $"($txSitter)/1/tx/send" { echo "Wait until tx is mined" for i in 0..100 { - let txResponse = http get $"($txSitter)/1/tx/($tx.txId)" + let txResponse = http get $"($txSitter)/1/($apiKey.apiKey)/tx/($tx.txId)" if ($txResponse | get -i status) == "mined" { echo $txResponse diff --git a/manual_test_gas_prices.nu b/manual_test_gas_prices.nu deleted file mode 100644 index ca65b0e..0000000 --- a/manual_test_gas_prices.nu +++ /dev/null @@ -1,69 +0,0 @@ -# Setup dependencies in different terminals: -# DB -docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 postgres -# Can connect to using psql postgres://postgres:postgres@127.0.0.1:5432/database - -# Nodes -anvil --chain-id 31337 -p 8545 --block-time 1 -anvil --chain-id 31338 -p 8546 --block-time 1 - -# TxSitter -cargo watch -x run -# or just -cargo run - -let txSitter = "http://127.0.0.1:3000" -let anvilSocket = "127.0.0.1:8545" -let anvilSocket2 = "127.0.0.1:8546" - -http post -t application/json $"($txSitter)/1/network/31337" { - name: "Anvil network", - httpRpc: $"http://($anvilSocket)", - wsRpc: $"ws://($anvilSocket)" -} - -http post -t application/json $"($txSitter)/1/network/31338" { - name: "Secondary Anvil network", - httpRpc: $"http://($anvilSocket2)", - wsRpc: $"ws://($anvilSocket2)" -} - -echo "Creating relayer" -let relayer = http post -t application/json $"($txSitter)/1/relayer" { "name": "My Relayer", "chainId": 31337 } - -echo "Update relayer - with secondary chain gas limit dependency" -http post -t application/json $"($txSitter)/1/relayer/($relayer.relayerId)" { - gasLimits: [ - { - chainId: 31338, - # Note that this value is hexadecimal - value: "3B9ACA00" - } - ] -} - -echo "Funding relayer" -cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' - -echo "Sending transaction" -let tx = http post -t application/json $"($txSitter)/1/tx/send" { - "relayerId": $relayer.relayerId, - "to": $relayer.address, - "value": "10", - "data": "" - "gasLimit": "150000" -} - -echo "Wait until tx is mined" -for i in 0..100 { - let txResponse = http get $"($txSitter)/1/tx/($tx.txId)" - - if ($txResponse | get -i status) == "mined" { - echo $txResponse - break - } else { - sleep 1sec - } -} - -echo "Success!" diff --git a/src/api_key.rs b/src/api_key.rs new file mode 100644 index 0000000..f180a83 --- /dev/null +++ b/src/api_key.rs @@ -0,0 +1,118 @@ +use std::str::FromStr; + +use base64::Engine; +use rand::rngs::OsRng; +use rand::RngCore; +use serde::Serialize; +use sha3::{Digest, Sha3_256}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ApiKey { + pub relayer_id: String, + pub api_key: [u8; 32], +} + +impl ApiKey { + pub fn new(relayer_id: impl ToString) -> Self { + let relayer_id = relayer_id.to_string(); + + let mut api_key = [0u8; 32]; + OsRng.fill_bytes(&mut api_key); + + Self { + relayer_id, + api_key, + } + } + + pub fn api_key_hash(&self) -> [u8; 32] { + Sha3_256::digest(self.api_key).into() + } +} + +impl Serialize for ApiKey { + fn serialize( + &self, + serializer: S, + ) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> serde::Deserialize<'de> for ApiKey { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + <&str>::deserialize(deserializer)? + .parse() + .map_err(serde::de::Error::custom) + } +} + +impl FromStr for ApiKey { + type Err = eyre::ErrReport; + + fn from_str(s: &str) -> Result { + let buffer = base64::prelude::BASE64_URL_SAFE.decode(s)?; + + if buffer.len() != 48 { + return Err(eyre::eyre!("invalid api key")); + } + + let relayer_id = uuid::Uuid::from_slice(&buffer[..16])?; + let relayer_id = relayer_id.to_string(); + + let mut api_key = [0u8; 32]; + api_key.copy_from_slice(&buffer[16..]); + + Ok(Self { + relayer_id, + api_key, + }) + } +} + +impl std::fmt::Display for ApiKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut buffer = [0u8; 48]; + + let relayer_id = uuid::Uuid::parse_str(&self.relayer_id) + .map_err(|_| std::fmt::Error)?; + + buffer[..16].copy_from_slice(relayer_id.as_bytes()); + buffer[16..].copy_from_slice(&self.api_key); + + let encoded = base64::prelude::BASE64_URL_SAFE.encode(buffer); + + write!(f, "{}", encoded) + } +} + +#[cfg(test)] +mod tests { + use rand::rngs::OsRng; + use rand::RngCore; + + use super::*; + + #[test] + fn from_to_str() { + let api_key = ApiKey { + relayer_id: uuid::Uuid::new_v4().to_string(), + api_key: { + let mut api_key = [0u8; 32]; + OsRng.fill_bytes(&mut api_key); + api_key + }, + }; + + let api_key_str = api_key.to_string(); + + println!("api_key_str = {api_key_str}"); + + let api_key_parsed = api_key_str.parse::().unwrap(); + + assert_eq!(api_key, api_key_parsed); + } +} diff --git a/src/app.rs b/src/app.rs index 2535001..4de5286 100644 --- a/src/app.rs +++ b/src/app.rs @@ -3,6 +3,7 @@ use ethers::providers::{Http, Provider, Ws}; use ethers::signers::Signer; use eyre::Context; +use crate::api_key::ApiKey; use crate::config::{Config, KeysConfig}; use crate::db::data::RpcKind; use crate::db::Database; @@ -74,6 +75,19 @@ impl App { Ok(middlware) } + + pub async fn is_authorized( + &self, + api_token: &ApiKey, + ) -> eyre::Result { + if self.config.server.disable_auth { + return Ok(true); + } + + self.db + .is_api_key_valid(&api_token.relayer_id, api_token.api_key_hash()) + .await + } } async fn init_keys_source( diff --git a/src/client.rs b/src/client.rs index 91bcd68..d60244f 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,9 @@ +use reqwest::Response; + +use crate::api_key::ApiKey; use crate::server::routes::network::NewNetworkInfo; use crate::server::routes::relayer::{ - CreateRelayerRequest, CreateRelayerResponse, + CreateApiKeyResponse, CreateRelayerRequest, CreateRelayerResponse, }; use crate::server::routes::transaction::{SendTxRequest, SendTxResponse}; @@ -17,34 +20,61 @@ impl TxSitterClient { } } + async fn post(&self, url: &str) -> eyre::Result + where + R: serde::de::DeserializeOwned, + { + let response = self.client.post(url).send().await?; + + let response = Self::validate_response(response).await?; + + Ok(response.json().await?) + } + + async fn json_post(&self, url: &str, body: T) -> eyre::Result + where + T: serde::Serialize, + R: serde::de::DeserializeOwned, + { + let response = self.client.post(url).json(&body).send().await?; + + let response = Self::validate_response(response).await?; + + Ok(response.json().await?) + } + + async fn validate_response(response: Response) -> eyre::Result { + if !response.status().is_success() { + let body = response.text().await?; + + return Err(eyre::eyre!("{body}")); + } + + Ok(response) + } pub async fn create_relayer( &self, req: &CreateRelayerRequest, ) -> eyre::Result { - let response = self - .client - .post(&format!("{}/1/relayer", self.url)) - .json(req) - .send() - .await?; - - let response: CreateRelayerResponse = response.json().await?; + self.json_post(&format!("{}/1/relayer", self.url), req) + .await + } - Ok(response) + pub async fn create_relayer_api_key( + &self, + relayer_id: &str, + ) -> eyre::Result { + self.post(&format!("{}/1/relayer/{relayer_id}/key", self.url,)) + .await } pub async fn send_tx( &self, + api_key: &ApiKey, req: &SendTxRequest, ) -> eyre::Result { - let response = self - .client - .post(&format!("{}/1/tx/send", self.url)) - .json(req) - .send() - .await?; - - Ok(response.json().await?) + self.json_post(&format!("{}/1/{api_key}/tx", self.url), req) + .await } pub async fn create_network( @@ -52,13 +82,14 @@ impl TxSitterClient { chain_id: u64, req: &NewNetworkInfo, ) -> eyre::Result<()> { - self.client + let response = self + .client .post(&format!("{}/1/network/{}", self.url, chain_id)) - .json(req) + .json(&req) .send() .await?; - // TODO: Handle status? + Self::validate_response(response).await?; Ok(()) } diff --git a/src/db.rs b/src/db.rs index 51b5271..6ce32f6 100644 --- a/src/db.rs +++ b/src/db.rs @@ -866,6 +866,48 @@ impl Database { Ok(items.into_iter().map(|(x,)| x as u64).collect()) } + + pub async fn save_api_key( + &self, + relayer_id: &str, + api_key_hash: [u8; 32], + ) -> eyre::Result<()> { + sqlx::query( + r#" + INSERT INTO api_keys (relayer_id, key_hash) + VALUES ($1, $2) + "#, + ) + .bind(relayer_id) + .bind(api_key_hash) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn is_api_key_valid( + &self, + relayer_id: &str, + api_key_hash: [u8; 32], + ) -> eyre::Result { + let (is_valid,): (bool,) = sqlx::query_as( + r#" + SELECT EXISTS ( + SELECT 1 + FROM api_keys + WHERE relayer_id = $1 + AND key_hash = $2 + ) + "#, + ) + .bind(relayer_id) + .bind(api_key_hash) + .fetch_one(&self.pool) + .await?; + + Ok(is_valid) + } } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index c897aa2..70b3af8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod api_key; pub mod app; pub mod aws; pub mod broadcast_utils; diff --git a/src/server.rs b/src/server.rs index 24d4e62..be201fa 100644 --- a/src/server.rs +++ b/src/server.rs @@ -8,7 +8,8 @@ use hyper::server::conn::AddrIncoming; use thiserror::Error; use self::routes::relayer::{ - create_relayer, get_relayer, relayer_rpc, update_relayer, + create_relayer, create_relayer_api_key, get_relayer, relayer_rpc, + update_relayer, }; use self::routes::transaction::{get_tx, send_tx}; use crate::app::App; @@ -66,19 +67,16 @@ pub async fn serve(app: Arc) -> eyre::Result<()> { pub async fn spawn_server( app: Arc, ) -> eyre::Result>> { - let tx_routes = Router::new() - .route("/send", post(send_tx)) - .route("/:tx_id", get(get_tx)) - .layer(axum::middleware::from_fn_with_state( - app.clone(), - middleware::auth, - )) + let permissioned_routes = Router::new() + .route("/:api_token/tx", post(send_tx)) + .route("/:api_token/tx/:tx_id", get(get_tx)) + .route("/:api_token/rpc", post(relayer_rpc)) .with_state(app.clone()); let relayer_routes = Router::new() .route("/", post(create_relayer)) .route("/:relayer_id", post(update_relayer).get(get_relayer)) - .route("/:relayer_id/rpc", post(relayer_rpc)) + .route("/:relayer_id/key", post(create_relayer_api_key)) .with_state(app.clone()); let network_routes = Router::new() @@ -88,7 +86,7 @@ pub async fn spawn_server( .with_state(app.clone()); let v1_routes = Router::new() - .nest("/tx", tx_routes) + .nest("/", permissioned_routes) .nest("/relayer", relayer_routes) .nest("/network", network_routes); diff --git a/src/server/middleware.rs b/src/server/middleware.rs index 50ba8c4..a56ef0c 100644 --- a/src/server/middleware.rs +++ b/src/server/middleware.rs @@ -1,5 +1,3 @@ -mod auth_middleware; mod log_response_middleware; -pub use self::auth_middleware::{auth, AuthorizedRelayer}; pub use self::log_response_middleware::log_response; diff --git a/src/server/middleware/auth_middleware.rs b/src/server/middleware/auth_middleware.rs deleted file mode 100644 index 7c79c28..0000000 --- a/src/server/middleware/auth_middleware.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::sync::Arc; - -use axum::extract::{Query, State}; -use axum::http::{HeaderName, HeaderValue, Request}; -use axum::middleware::Next; -use axum::response::{IntoResponse, Response}; -use headers::Header; -use serde::{Deserialize, Serialize}; - -use crate::app::App; -use crate::server::ApiError; - -pub const AUTHORIZED_RELAYER: &str = "x-authorized-relayer"; -static HEADER_NAME: HeaderName = HeaderName::from_static(AUTHORIZED_RELAYER); - -#[derive(Debug, Clone)] -pub enum AuthorizedRelayer { - Named(String), - Any, -} - -impl AuthorizedRelayer { - pub fn is_authorized(&self, relayer_id: &str) -> bool { - match self { - AuthorizedRelayer::Any => true, - AuthorizedRelayer::Named(name) => name == relayer_id, - } - } -} - -impl Header for AuthorizedRelayer { - fn name() -> &'static HeaderName { - &HEADER_NAME - } - - fn decode<'i, I>(values: &mut I) -> Result - where - Self: Sized, - I: Iterator, - { - let value = values.next().ok_or_else(headers::Error::invalid)?; - let value = value - .to_str() - .map_err(|_| headers::Error::invalid())? - .to_owned(); - - if value == "*" { - Ok(Self::Any) - } else { - Ok(Self::Named(value)) - } - } - - fn encode>(&self, values: &mut E) { - match self { - AuthorizedRelayer::Named(name) => values - .extend(std::iter::once(HeaderValue::from_str(name).unwrap())), - AuthorizedRelayer::Any => { - values.extend(std::iter::once(HeaderValue::from_static("*"))) - } - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct AuthParams { - #[serde(default)] - api_key: Option, -} - -pub async fn auth( - State(context): State>, - Query(query): Query, - request: Request, - next: Next, -) -> Response { - let (mut parts, body) = request.into_parts(); - - if context.config.server.disable_auth { - parts - .headers - .insert(AUTHORIZED_RELAYER, HeaderValue::from_str("*").unwrap()); - } else { - let authorized_relayer = match auth_inner(context.clone(), query).await - { - Ok(relayer_id) => relayer_id, - Err(error) => return error.into_response(), - }; - - parts.headers.insert( - AUTHORIZED_RELAYER, - HeaderValue::from_str(&authorized_relayer).unwrap(), - ); - } - - let request = Request::from_parts(parts, body); - - next.run(request).await -} - -async fn auth_inner( - _app: Arc, - _query: AuthParams, -) -> Result { - todo!("Add tables to DB and implement") - // let mut api_key = None; - - // TODO: Support Bearer in auth header - // let auth_header = parts.headers.get(AUTHORIZATION); - // if let Some(auth_header) = auth_header { - // todo!() - // } - - // if let Some(api_key_from_query) = query.api_key { - // api_key = Some(api_key_from_query); - // } - - // let Some(api_key) = api_key else { - // return Err(ApiError::Unauthorized); - // }; - - // let api_key = hex::decode(&api_key).map_err(|err| { - // tracing::warn!(?err, "Error decoding api key"); - - // ApiError::KeyEncoding - // })?; - - // let api_key: [u8; 32] = - // api_key.try_into().map_err(|_| ApiError::KeyLength)?; - - // let api_key_hash = Sha3_256::digest(&api_key); - - // let api_key_hash = hex::encode(api_key_hash); - - // // let relayer_id = context - // // .api_keys_db - // // .get_relayer_id_by_hash(api_key_hash) - // // .await? - // // .ok_or_else(|| ApiError::Unauthorized)?; - - // let relayer_id = todo!(); - - // Ok(relayer_id) -} diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 098de23..873facd 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -7,6 +7,7 @@ use eyre::Result; use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::api_key::ApiKey; use crate::app::App; use crate::server::ApiError; use crate::types::{RelayerInfo, RelayerUpdate}; @@ -49,6 +50,12 @@ pub enum JsonRpcVersion { V2, } +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateApiKeyResponse { + pub api_key: ApiKey, +} + #[tracing::instrument(skip(app))] pub async fn create_relayer( State(app): State>, @@ -95,10 +102,14 @@ pub async fn get_relayer( #[tracing::instrument(skip(app))] pub async fn relayer_rpc( State(app): State>, - Path(relayer_id): Path, + Path(api_token): Path, Json(req): Json, ) -> Result, ApiError> { - let relayer_info = app.db.get_relayer(&relayer_id).await?; + if !app.is_authorized(&api_token).await? { + return Err(ApiError::Unauthorized); + } + + let relayer_info = app.db.get_relayer(&api_token.relayer_id).await?; // TODO: Cache? let http_provider = app.http_provider(relayer_info.chain_id).await?; @@ -119,3 +130,17 @@ pub async fn relayer_rpc( Ok(Json(response)) } + +#[tracing::instrument(skip(app))] +pub async fn create_relayer_api_key( + State(app): State>, + Path(relayer_id): Path, +) -> Result, ApiError> { + let api_key = ApiKey::new(&relayer_id); + + app.db + .save_api_key(&relayer_id, api_key.api_key_hash()) + .await?; + + Ok(Json(CreateApiKeyResponse { api_key })) +} diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 364a082..a4f0064 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -1,14 +1,13 @@ use std::sync::Arc; use axum::extract::{Json, Path, State}; -use axum::TypedHeader; use ethers::types::{Address, Bytes, H256, U256}; use eyre::Result; use serde::{Deserialize, Serialize}; +use crate::api_key::ApiKey; use crate::app::App; use crate::db::TxStatus; -use crate::server::middleware::AuthorizedRelayer; use crate::server::ApiError; use crate::types::TransactionPriority; @@ -72,10 +71,10 @@ pub enum UnsentStatus { #[tracing::instrument(skip(app))] pub async fn send_tx( State(app): State>, - TypedHeader(authorized_relayer): TypedHeader, + Path(api_token): Path, Json(req): Json, ) -> Result, ApiError> { - if !authorized_relayer.is_authorized(&req.relayer_id) { + if !app.is_authorized(&api_token).await? { return Err(ApiError::Unauthorized); } @@ -103,8 +102,12 @@ pub async fn send_tx( #[tracing::instrument(skip(app))] pub async fn get_tx( State(app): State>, - Path(tx_id): Path, + Path((api_token, tx_id)): Path<(ApiKey, String)>, ) -> Result, ApiError> { + if !app.is_authorized(&api_token).await? { + return Err(ApiError::Unauthorized); + } + let tx = app.db.read_tx(&tx_id).await?.ok_or(ApiError::MissingTx)?; let get_tx_response = GetTxResponse { diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index a5b36a9..0ba1dc9 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -1,6 +1,7 @@ mod common; use ethers::prelude::*; +use service::server::routes::relayer::CreateApiKeyResponse; use url::Url; use crate::common::prelude::*; @@ -24,8 +25,10 @@ async fn rpc_access() -> eyre::Result<()> { }) .await?; - let rpc_url = - format!("http://{}/1/relayer/{relayer_id}/rpc", service.local_addr()); + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(&relayer_id).await?; + + let rpc_url = format!("http://{}/1/{api_key}/rpc", service.local_addr()); let provider = Provider::new(Http::new(rpc_url.parse::()?)); diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index cc5157c..995837e 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -1,5 +1,7 @@ mod common; +use service::server::routes::relayer::CreateApiKeyResponse; + use crate::common::prelude::*; const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); @@ -24,6 +26,9 @@ async fn send_many_txs() -> eyre::Result<()> { }) .await?; + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(&relayer_id).await?; + // Fund the relayer let middleware = setup_middleware( format!("http://{}", double_anvil.local_addr()), @@ -56,13 +61,16 @@ async fn send_many_txs() -> eyre::Result<()> { for _ in 0..num_transfers { client - .send_tx(&SendTxRequest { - relayer_id: relayer_id.clone(), - to: ARBITRARY_ADDRESS, - value, - gas_limit: U256::from(21_000), - ..Default::default() - }) + .send_tx( + &api_key, + &SendTxRequest { + relayer_id: relayer_id.clone(), + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) .await?; } diff --git a/tests/send_tx.rs b/tests/send_tx.rs index 66112e6..c6eb249 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -1,5 +1,7 @@ mod common; +use service::server::routes::relayer::CreateApiKeyResponse; + use crate::common::prelude::*; const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); @@ -24,6 +26,9 @@ async fn send_tx() -> eyre::Result<()> { }) .await?; + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(&relayer_id).await?; + // Fund the relayer let middleware = setup_middleware( format!("http://{}", double_anvil.local_addr()), @@ -53,13 +58,16 @@ async fn send_tx() -> eyre::Result<()> { // Send a transaction let value: U256 = parse_units("1", "ether")?.into(); client - .send_tx(&SendTxRequest { - relayer_id, - to: ARBITRARY_ADDRESS, - value, - gas_limit: U256::from(21_000), - ..Default::default() - }) + .send_tx( + &api_key, + &SendTxRequest { + relayer_id, + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) .await?; for _ in 0..10 { From a012706ad48246393e9565a3665ccfc57ebef06f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Fri, 1 Dec 2023 12:48:31 +0100 Subject: [PATCH 031/135] Dzejkop/dockerize (#4) --- .dockerignore | 6 +++ Cargo.lock | 100 ++++++++++++++++++++--------------------- Cargo.toml | 6 +-- Dockerfile | 40 +++++++++++++++++ src/main.rs | 22 +++++---- tests/common/mod.rs | 18 ++++---- tests/rpc_access.rs | 2 +- tests/send_many_txs.rs | 2 +- tests/send_tx.rs | 2 +- 9 files changed, 123 insertions(+), 75 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2957cfd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +target/ +.git +README.md +TODO.md +manual_test.nu +config.toml diff --git a/Cargo.lock b/Cargo.lock index 857248b..b549c1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3819,56 +3819,6 @@ dependencies = [ "serde", ] -[[package]] -name = "service" -version = "0.1.0" -dependencies = [ - "async-trait", - "aws-config", - "aws-credential-types", - "aws-sdk-kms", - "aws-smithy-types", - "aws-types", - "axum", - "base64 0.21.5", - "bigdecimal 0.4.2", - "chrono", - "clap", - "config", - "dotenv", - "ethers", - "eyre", - "fake-rpc", - "futures", - "headers", - "hex", - "hex-literal", - "humantime", - "humantime-serde", - "hyper", - "indoc", - "itertools 0.12.0", - "num-bigint", - "postgres-docker-utils", - "rand", - "reqwest", - "serde", - "serde_json", - "sha3", - "spki", - "sqlx", - "strum", - "test-case", - "thiserror", - "tokio", - "toml 0.8.8", - "tower-http", - "tracing", - "tracing-subscriber", - "url", - "uuid 0.8.2", -] - [[package]] name = "sha1" version = "0.10.6" @@ -4813,6 +4763,56 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tx-sitter" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-kms", + "aws-smithy-types", + "aws-types", + "axum", + "base64 0.21.5", + "bigdecimal 0.4.2", + "chrono", + "clap", + "config", + "dotenv", + "ethers", + "eyre", + "fake-rpc", + "futures", + "headers", + "hex", + "hex-literal", + "humantime", + "humantime-serde", + "hyper", + "indoc", + "itertools 0.12.0", + "num-bigint", + "postgres-docker-utils", + "rand", + "reqwest", + "serde", + "serde_json", + "sha3", + "spki", + "sqlx", + "strum", + "test-case", + "thiserror", + "tokio", + "toml 0.8.8", + "tower-http", + "tracing", + "tracing-subscriber", + "url", + "uuid 0.8.2", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index 1743dfb..8056d75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "service" +name = "tx-sitter" version = "0.1.0" edition = "2021" @@ -77,7 +77,3 @@ postgres-docker-utils = { path = "crates/postgres-docker-utils" } test-case = "3.1.0" indoc = "2.0.3" fake-rpc = { path = "crates/fake-rpc" } - -[[bin]] -name = "bootstrap" -path = "src/main.rs" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..61549a4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM debian:12 as build-env + +WORKDIR /src + +# Install dependencies +RUN apt-get update && \ + apt-get install -y curl build-essential libssl-dev texinfo libcap2-bin pkg-config + +# TODO: Use a specific version of rustup +# Install rustup +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +# Copy only rust-toolchain.toml for better caching +COPY ./rust-toolchain.toml ./rust-toolchain.toml + +# Set environment variables +ENV PATH="/root/.cargo/bin:${PATH}" +ENV RUSTUP_HOME="/root/.rustup" +ENV CARGO_HOME="/root/.cargo" + +# Install the toolchain +RUN rustup component add cargo + +# Copy all the source files +# .dockerignore ignores the target dir +COPY . . + +# Build the binary +RUN cargo build --release + +# Make sure it runs +RUN /src/target/release/tx-sitter --version + +# cc variant because we need libgcc and others +FROM gcr.io/distroless/cc-debian12:nonroot + +# Copy the tx-sitter binary +COPY --from=build-env --chown=0:10001 --chmod=010 /src/target/release/tx-sitter /bin/tx-sitter + +ENTRYPOINT [ "/bin/tx-sitter" ] diff --git a/src/main.rs b/src/main.rs index a0416ce..7842066 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,17 +2,18 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; -use service::config::Config; -use service::service::Service; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; +use tx_sitter::config::Config; +use tx_sitter::service::Service; #[derive(Parser)] +#[command(author, version, about)] #[clap(rename_all = "kebab-case")] struct Args { - #[clap(short, long, default_value = "./config.toml")] - config: PathBuf, + #[clap(short, long, default_value = "config.toml")] + config: Vec, #[clap(short, long)] env_file: Vec, @@ -33,10 +34,15 @@ async fn main() -> eyre::Result<()> { .with(EnvFilter::from_default_env()) .init(); - let settings = config::Config::builder() - .add_source( - config::File::from(args.config.as_ref()).format(FileFormat::Toml), - ) + let mut settings = config::Config::builder(); + + for arg in &args.config { + settings = settings.add_source( + config::File::from(arg.as_ref()).format(FileFormat::Toml), + ); + } + + let settings = settings .add_source( config::Environment::with_prefix("TX_SITTER").separator("__"), ) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 21da184..dff313b 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -11,18 +11,18 @@ use ethers::signers::{LocalWallet, Signer}; use ethers::types::{Address, Eip1559TransactionRequest, H160, U256}; use fake_rpc::DoubleAnvil; use postgres_docker_utils::DockerContainerGuard; -use service::client::TxSitterClient; -use service::config::{ - Config, DatabaseConfig, KeysConfig, LocalKeysConfig, ServerConfig, - TxSitterConfig, -}; -use service::server::routes::network::NewNetworkInfo; -use service::service::Service; use tokio::task::JoinHandle; use tracing::level_filters::LevelFilter; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; +use tx_sitter::client::TxSitterClient; +use tx_sitter::config::{ + Config, DatabaseConfig, KeysConfig, LocalKeysConfig, ServerConfig, + TxSitterConfig, +}; +use tx_sitter::server::routes::network::NewNetworkInfo; +use tx_sitter::service::Service; pub type AppMiddleware = SignerMiddleware>, LocalWallet>; @@ -33,10 +33,10 @@ pub mod prelude { pub use ethers::providers::Middleware; pub use ethers::types::{Eip1559TransactionRequest, U256}; pub use ethers::utils::parse_units; - pub use service::server::routes::relayer::{ + pub use tx_sitter::server::routes::relayer::{ CreateRelayerRequest, CreateRelayerResponse, }; - pub use service::server::routes::transaction::SendTxRequest; + pub use tx_sitter::server::routes::transaction::SendTxRequest; pub use super::*; } diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index 0ba1dc9..a8ace6d 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -1,7 +1,7 @@ mod common; use ethers::prelude::*; -use service::server::routes::relayer::CreateApiKeyResponse; +use tx_sitter::server::routes::relayer::CreateApiKeyResponse; use url::Url; use crate::common::prelude::*; diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 995837e..44106ee 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -1,6 +1,6 @@ mod common; -use service::server::routes::relayer::CreateApiKeyResponse; +use tx_sitter::server::routes::relayer::CreateApiKeyResponse; use crate::common::prelude::*; diff --git a/tests/send_tx.rs b/tests/send_tx.rs index c6eb249..8552c31 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -1,6 +1,6 @@ mod common; -use service::server::routes::relayer::CreateApiKeyResponse; +use tx_sitter::server::routes::relayer::CreateApiKeyResponse; use crate::common::prelude::*; From cb21f8ccc0c2dbfef2675bdf04b3f8b3a8efe3cb Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 1 Dec 2023 13:08:46 +0100 Subject: [PATCH 032/135] Update API routes --- manual_test.nu | 12 ++++++------ src/client.rs | 8 ++++---- src/server.rs | 25 +++++++++++-------------- tests/rpc_access.rs | 3 ++- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/manual_test.nu b/manual_test.nu index e22fda4..95900d0 100644 --- a/manual_test.nu +++ b/manual_test.nu @@ -17,25 +17,25 @@ echo "Start" let txSitter = "http://127.0.0.1:3000" let anvilSocket = "127.0.0.1:8545" -http post -t application/json $"($txSitter)/1/network/31337" { +http post -t application/json $"($txSitter)/1/admin/network/31337" { name: "Anvil network", httpRpc: $"http://($anvilSocket)", wsRpc: $"ws://($anvilSocket)" } echo "Creating relayer" -let relayer = http post -t application/json $"($txSitter)/1/relayer" { "name": "My Relayer", "chainId": 31337 } +let relayer = http post -t application/json $"($txSitter)/1/admin/relayer" { "name": "My Relayer", "chainId": 31337 } echo "Create api key" -let apiKey = http post $"($txSitter)/1/relayer/($relayer.relayerId)/key" "" +let apiKey = http post $"($txSitter)/1/admin/relayer/($relayer.relayerId)/key" "" -$env.ETH_RPC_URL = $"($txSitter)/1/($apiKey.apiKey)/rpc" +$env.ETH_RPC_URL = $"($txSitter)/1/api/($apiKey.apiKey)/rpc" echo "Funding relayer" cast send --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --value 100ether $relayer.address '' echo "Sending transaction" -let tx = http post -t application/json $"($txSitter)/1/($apiKey.apiKey)/tx" { +let tx = http post -t application/json $"($txSitter)/1/api/($apiKey.apiKey)/tx" { "relayerId": $relayer.relayerId, "to": $relayer.address, "value": "10", @@ -45,7 +45,7 @@ let tx = http post -t application/json $"($txSitter)/1/($apiKey.apiKey)/tx" { echo "Wait until tx is mined" for i in 0..100 { - let txResponse = http get $"($txSitter)/1/($apiKey.apiKey)/tx/($tx.txId)" + let txResponse = http get $"($txSitter)/1/api/($apiKey.apiKey)/tx/($tx.txId)" if ($txResponse | get -i status) == "mined" { echo $txResponse diff --git a/src/client.rs b/src/client.rs index d60244f..b479e5d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -56,7 +56,7 @@ impl TxSitterClient { &self, req: &CreateRelayerRequest, ) -> eyre::Result { - self.json_post(&format!("{}/1/relayer", self.url), req) + self.json_post(&format!("{}/1/admin/relayer", self.url), req) .await } @@ -64,7 +64,7 @@ impl TxSitterClient { &self, relayer_id: &str, ) -> eyre::Result { - self.post(&format!("{}/1/relayer/{relayer_id}/key", self.url,)) + self.post(&format!("{}/1/admin/relayer/{relayer_id}/key", self.url,)) .await } @@ -73,7 +73,7 @@ impl TxSitterClient { api_key: &ApiKey, req: &SendTxRequest, ) -> eyre::Result { - self.json_post(&format!("{}/1/{api_key}/tx", self.url), req) + self.json_post(&format!("{}/1/api/{api_key}/tx", self.url), req) .await } @@ -84,7 +84,7 @@ impl TxSitterClient { ) -> eyre::Result<()> { let response = self .client - .post(&format!("{}/1/network/{}", self.url, chain_id)) + .post(&format!("{}/1/admin/network/{}", self.url, chain_id)) .json(&req) .send() .await?; diff --git a/src/server.rs b/src/server.rs index be201fa..9c7dc98 100644 --- a/src/server.rs +++ b/src/server.rs @@ -67,28 +67,25 @@ pub async fn serve(app: Arc) -> eyre::Result<()> { pub async fn spawn_server( app: Arc, ) -> eyre::Result>> { - let permissioned_routes = Router::new() + let api_routes = Router::new() .route("/:api_token/tx", post(send_tx)) .route("/:api_token/tx/:tx_id", get(get_tx)) .route("/:api_token/rpc", post(relayer_rpc)) .with_state(app.clone()); - let relayer_routes = Router::new() - .route("/", post(create_relayer)) - .route("/:relayer_id", post(update_relayer).get(get_relayer)) - .route("/:relayer_id/key", post(create_relayer_api_key)) - .with_state(app.clone()); - - let network_routes = Router::new() - // .route("/", get(routes::network::get_networks)) - // .route("/:chain_id", get(routes::network::get_network)) - .route("/:chain_id", post(routes::network::create_network)) + let admin_routes = Router::new() + .route("/relayer", post(create_relayer)) + .route( + "/relayer/:relayer_id", + post(update_relayer).get(get_relayer), + ) + .route("/relayer/:relayer_id/key", post(create_relayer_api_key)) + .route("/network/:chain_id", post(routes::network::create_network)) .with_state(app.clone()); let v1_routes = Router::new() - .nest("/", permissioned_routes) - .nest("/relayer", relayer_routes) - .nest("/network", network_routes); + .nest("/api", api_routes) + .nest("/admin", admin_routes); let router = Router::new() .nest("/1", v1_routes) diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index a8ace6d..81509cc 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -28,7 +28,8 @@ async fn rpc_access() -> eyre::Result<()> { let CreateApiKeyResponse { api_key } = client.create_relayer_api_key(&relayer_id).await?; - let rpc_url = format!("http://{}/1/{api_key}/rpc", service.local_addr()); + let rpc_url = + format!("http://{}/1/api/{api_key}/rpc", service.local_addr()); let provider = Provider::new(Http::new(rpc_url.parse::()?)); From 06147123bc2465bde0885359fe100be88792cbea Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 1 Dec 2023 13:11:39 +0100 Subject: [PATCH 033/135] Log more --- src/tasks/escalate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index aea2178..9725842 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -17,7 +17,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { .await?; for tx in txs_for_escalation { - tracing::info!(tx.id, "Escalating tx"); + tracing::info!(tx.id, tx.escalation_count, "Escalating tx"); if !should_send_transaction(&app, &tx.relayer_id).await? { tracing::warn!(id = tx.id, "Skipping transaction broadcast"); From 7444dddb0dadfcb308fba98abe9dfcb5875a82e7 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 1 Dec 2023 15:55:50 +0100 Subject: [PATCH 034/135] Save all txs --- db/migrations/001_init.sql | 4 +--- src/db.rs | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 9889d47..62a7209 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -88,9 +88,7 @@ CREATE TABLE block_txs ( block_number BIGINT NOT NULL, chain_id BIGINT NOT NULL, tx_hash BYTEA NOT NULL, - PRIMARY KEY (block_number, chain_id, tx_hash), - FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE, - FOREIGN KEY (tx_hash) REFERENCES tx_hashes (tx_hash) + FOREIGN KEY (block_number, chain_id) REFERENCES blocks (block_number, chain_id) ON DELETE CASCADE ); CREATE TABLE block_fees ( diff --git a/src/db.rs b/src/db.rs index 6ce32f6..62fadef 100644 --- a/src/db.rs +++ b/src/db.rs @@ -368,11 +368,6 @@ impl Database { INSERT INTO block_txs (block_number, chain_id, tx_hash) SELECT $1, $2, unnested.tx_hash FROM UNNEST($3::BYTEA[]) AS unnested(tx_hash) - WHERE EXISTS ( - SELECT 1 - FROM tx_hashes - WHERE tx_hashes.tx_hash = unnested.tx_hash - ); "#, ) .bind(block_number as i64) From 59e429240238d8821c1bfff3b079532ffa2052d0 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 1 Dec 2023 15:55:59 +0100 Subject: [PATCH 035/135] Fetch block hashes --- src/tasks/index.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 9db8a58..cd43323 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -37,6 +37,11 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { ) .context("Invalid timestamp")?; + let block = rpc + .get_block(block_number) + .await? + .context("Missing block")?; + app.db .save_block( block.number.unwrap().as_u64(), From 218a8ec16f82286e1d97aaeb4e0370c918e3cce9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 1 Dec 2023 15:56:35 +0100 Subject: [PATCH 036/135] Update AWS libs --- Cargo.lock | 76 +++++++++++++++++++++------------------- Cargo.toml | 11 +++--- src/aws/ethers_signer.rs | 12 +++---- src/config.rs | 4 +-- src/keys.rs | 10 +++--- 5 files changed, 56 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b549c1e..b560d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -206,9 +206,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.57.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2bf00cb9416daab4ce4927c54ebe63c08b9caf4d7b9314b6d7a4a2c5a1afb09" +checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" dependencies = [ "aws-credential-types", "aws-http", @@ -237,9 +237,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "0.57.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9073c88dbf12f68ce7d0e149f989627a1d1ae3d2b680459f04ccc29d1cbd0f" +checksum = "8c1317e1a3514b103cf7d5828bbab3b4d30f56bd22d684f8568bc51b6cfbbb1c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -249,11 +249,10 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24067106d09620cf02d088166cdaedeaca7146d4d499c41b37accecbea11b246" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" dependencies = [ - "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -266,9 +265,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "0.57.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6ee0152c06d073602236a4e94a8c52a327d310c1ecd596570ce795af8777ff" +checksum = "1ed7ef604a15fd0d4d9e43701295161ea6b504b63c44990ead352afea2bc15e9" dependencies = [ "aws-credential-types", "aws-http", @@ -287,9 +286,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "0.36.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "674c06944cbef8df0c5ab43226f85ac28a15e6f4498d74aa200e00588b9b75e4" +checksum = "e5a1b6e2e95bc32f3b88d00de3b48156fbece5d8112dc76d975b9c2d2837dc8d" dependencies = [ "aws-credential-types", "aws-http", @@ -309,9 +308,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "0.36.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb8158015232b4596ccef74a205600398e152d704b40b7ec9f486092474d7fa" +checksum = "0619ab97a5ca8982e7de073cdc66f93e5f6a1b05afc09e696bec1cb3607cd4df" dependencies = [ "aws-credential-types", "aws-http", @@ -331,9 +330,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "0.36.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36a1493e1c57f173e53621935bfb5b6217376168dbdb4cd459aebcf645924a48" +checksum = "f04b9f5474cc0f35d829510b2ec8c21e352309b46bf9633c5a81fb9321e9b1c7" dependencies = [ "aws-credential-types", "aws-http", @@ -353,9 +352,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.36.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e032b77f5cd1dd3669d777a38ac08cbf8ec68e29460d4ef5d3e50cffa74ec75a" +checksum = "798c8d82203af9e15a8b406574e0b36da91dd6db533028b74676489a1bc8bc7d" dependencies = [ "aws-credential-types", "aws-http", @@ -376,13 +375,14 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.57.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f81a6abc4daab06b53cabf27c54189928893283093e37164ca53aa47488a5b" +checksum = "380adcc8134ad8bbdfeb2ace7626a869914ee266322965276cbc54066186d236" dependencies = [ "aws-credential-types", "aws-smithy-http", "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", "form_urlencoded", "hex", @@ -398,9 +398,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.57.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe53fccd3b10414b9cae63767a15a2789b34e6c6727b6e32b33e8c7998a3e80" +checksum = "3e37ca17d25fe1e210b6d4bdf59b81caebfe99f986201a1228cb5061233b4b13" dependencies = [ "futures-util", "pin-project-lite", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7972373213d1d6e619c0edc9dda2d6634154e4ed75c5e0b2bf065cd5ec9f0d1" +checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -429,18 +429,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d64d5af16dd585de9ff6c606423c1aaad47c6baa38de41c2beb32ef21c6645" +checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7527bf5335154ba1b285479c50b630e44e93d1b4a759eaceb8d0bf9fbc82caa5" +checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" dependencies = [ "aws-smithy-types", "urlencoding", @@ -448,9 +448,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "0.57.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839b363adf3b2bdab2742a1f540fec23039ea8bc9ec0f9f61df48470cfe5527b" +checksum = "273479291efc55e7b0bce985b139d86b6031adb8e50f65c1f712f20ba38f6388" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -458,6 +458,7 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand", + "h2", "http", "http-body", "hyper", @@ -472,9 +473,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "0.57.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24ecc446e62c3924539e7c18dec8038dba4fdf8718d5c2de62f9d2fecca8ba9" +checksum = "c6cebff0d977b6b6feed2fd07db52aac58ba3ccaf26cdd49f1af4add5061bef9" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -488,9 +489,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.57.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051de910296522a21178a2ea402ea59027eef4b63f1cef04a0be2bb5e25dea03" +checksum = "d7f48b3f27ddb40ab19892a5abda331f403e3cb877965e4e51171447807104af" dependencies = [ "base64-simd", "bytes", @@ -511,18 +512,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.57.2" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb1e3ac22c652662096c8e37a6f9af80c6f3520cab5610b2fe76c725bce18eac" +checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.57.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "048bbf1c24cdf4eb1efcdc243388a93a90ebf63979e25fc1c7b8cbd9cb6beb38" +checksum = "8403fc56b1f3761e8efe45771ddc1165e47ec3417c68e68a4519b5cb030159ca" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -4771,6 +4772,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-kms", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "axum", diff --git a/Cargo.toml b/Cargo.toml index 8056d75..3878858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,11 +9,12 @@ members = ["crates/*"] [dependencies] # Third Party ## AWS -aws-config = { version = "0.57.2" } -aws-sdk-kms = "0.36.0" -aws-smithy-types = "0.57.2" -aws-types = "0.57.2" -aws-credential-types = { version = "0.57.2", features = [ +aws-config = { version = "1.0.1" } +aws-sdk-kms = "1.3.0" +aws-smithy-types = "1.0.2" +aws-smithy-runtime-api = "1.0.2" +aws-types = "1.0.1" +aws-credential-types = { version = "1.0.1", features = [ "hardcoded-credentials", ] } diff --git a/src/aws/ethers_signer.rs b/src/aws/ethers_signer.rs index b4530d2..f8129ab 100644 --- a/src/aws/ethers_signer.rs +++ b/src/aws/ethers_signer.rs @@ -6,7 +6,6 @@ use aws_sdk_kms::operation::get_public_key::{ }; use aws_sdk_kms::operation::sign::{SignError, SignOutput}; use aws_sdk_kms::types::{MessageType, SigningAlgorithmSpec}; -use aws_smithy_types::body::SdkBody; use aws_smithy_types::Blob; use ethers::core::k256::ecdsa::{ Error as K256Error, Signature as KSig, VerifyingKey, @@ -15,7 +14,6 @@ use ethers::core::types::transaction::eip2718::TypedTransaction; use ethers::core::types::transaction::eip712::Eip712; use ethers::core::types::{Address, Signature as EthSig, H256}; use ethers::core::utils::hash_message; -use hyper::Response; use tracing::{debug, instrument, trace}; mod utils; @@ -81,9 +79,9 @@ impl std::fmt::Display for AwsSigner { #[derive(thiserror::Error, Debug)] pub enum AwsSignerError { #[error("{0}")] - SignError(#[from] SdkError>), + SignError(#[from] SdkError), #[error("{0}")] - GetPublicKeyError(#[from] SdkError>), + GetPublicKeyError(#[from] SdkError), #[error("{0}")] K256(#[from] K256Error), #[error("{0}")] @@ -114,7 +112,7 @@ impl From for AwsSignerError { async fn request_get_pubkey( kms: &aws_sdk_kms::Client, key_id: T, -) -> Result>> +) -> Result> where T: AsRef, { @@ -130,7 +128,7 @@ async fn request_sign_digest( kms: &aws_sdk_kms::Client, key_id: T, digest: [u8; 32], -) -> Result>> +) -> Result> where T: AsRef, { @@ -143,7 +141,9 @@ where .signing_algorithm(SigningAlgorithmSpec::EcdsaSha256) .send() .await; + trace!("{:?}", &resp); + resp } diff --git a/src/config.rs b/src/config.rs index 1c5e218..069923b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -43,9 +43,7 @@ pub enum KeysConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub struct KmsKeysConfig { - pub region: String, -} +pub struct KmsKeysConfig {} #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] diff --git a/src/keys.rs b/src/keys.rs index 9814aa8..01c4647 100644 --- a/src/keys.rs +++ b/src/keys.rs @@ -1,5 +1,5 @@ +use aws_config::BehaviorVersion; use aws_sdk_kms::types::{KeySpec, KeyUsageType}; -use aws_types::region::Region; use ethers::core::k256::ecdsa::SigningKey; use ethers::signers::Wallet; use eyre::{Context, ContextCompat}; @@ -24,11 +24,9 @@ pub struct KmsKeys { } impl KmsKeys { - pub async fn new(config: &KmsKeysConfig) -> eyre::Result { - let aws_config = aws_config::from_env() - .region(Region::new(config.region.clone())) - .load() - .await; + pub async fn new(_config: &KmsKeysConfig) -> eyre::Result { + let aws_config = + aws_config::load_defaults(BehaviorVersion::latest()).await; let kms_client = aws_sdk_kms::Client::new(&aws_config); From bbaabf235b08dad6ef8c18fb7b5073edd2cac3f9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 4 Dec 2023 17:49:23 +0100 Subject: [PATCH 037/135] Minor fixes --- Cargo.toml | 4 +++ Dockerfile | 2 +- config.toml | 1 - manual_test.nu | 1 + manual_test_kms.nu | 59 ++++++++++++++++++++++++++++++++++++++++ src/aws/ethers_signer.rs | 5 ++-- src/db.rs | 20 ++++++++++---- src/main.rs | 6 +++- src/tasks/index.rs | 13 ++++++++- tests/send_tx.rs | 2 +- 10 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 manual_test_kms.nu diff --git a/Cargo.toml b/Cargo.toml index 3878858..04bf3c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,3 +78,7 @@ postgres-docker-utils = { path = "crates/postgres-docker-utils" } test-case = "3.1.0" indoc = "2.0.3" fake-rpc = { path = "crates/fake-rpc" } + +[features] +default = [ "default-config" ] +default-config = [] diff --git a/Dockerfile b/Dockerfile index 61549a4..0c28393 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN rustup component add cargo COPY . . # Build the binary -RUN cargo build --release +RUN cargo build --release --no-default-features # Make sure it runs RUN /src/target/release/tx-sitter --version diff --git a/config.toml b/config.toml index 05b83a9..fa4ed3e 100644 --- a/config.toml +++ b/config.toml @@ -14,4 +14,3 @@ kind = "local" # Example KMS configuration # [keys] # kind = "kms" -# region = "us-east-1" diff --git a/manual_test.nu b/manual_test.nu index 95900d0..2943744 100644 --- a/manual_test.nu +++ b/manual_test.nu @@ -51,6 +51,7 @@ for i in 0..100 { echo $txResponse break } else { + echo $txResponse sleep 1sec } } diff --git a/manual_test_kms.nu b/manual_test_kms.nu new file mode 100644 index 0000000..e426991 --- /dev/null +++ b/manual_test_kms.nu @@ -0,0 +1,59 @@ +## Setup dependencies in different terminals: +## DB +# docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 postgres +## Can connect to using psql postgres://postgres:postgres@127.0.0.1:5432/database +## TxSitter +# cargo watch -x run +## or just +# cargo run + +echo "Start" + +let txSitter = "http://127.0.0.1:3000" + +http post -t application/json $"($txSitter)/1/admin/network/11155111" { + name: "Ethereum Sepolia", + httpRpc: $env.SEPOLIA_HTTP_RPC, + wsRpc: $env.SEPOLIA_WS_RPC, +} + +echo "Creating relayer" +let relayer = http post -t application/json $"($txSitter)/1/admin/relayer" { "name": "My Relayer", "chainId": 11155111 } + +http post -t application/json $"($txSitter)/1/admin/relayer/($relayer.relayerId)" { + gasLimits: [ + { chainId: 11155111, value: "0x123" } + ] +} + +echo "Create api key" +let apiKey = http post $"($txSitter)/1/admin/relayer/($relayer.relayerId)/key" "" + +$env.ETH_RPC_URL = $"($txSitter)/1/api/($apiKey.apiKey)/rpc" + +echo "Funding relayer" +cast send --private-key $env.PRIVATE_KEY --value 1ether $relayer.address '' + +echo "Sending transaction" +let tx = http post -t application/json $"($txSitter)/1/api/($apiKey.apiKey)/tx" { + "relayerId": $relayer.relayerId, + "to": $relayer.address, + "value": "10", + "data": "" + "gasLimit": "150000" +} + +echo "Wait until tx is mined" +for i in 0..100 { + let txResponse = http get $"($txSitter)/1/api/($apiKey.apiKey)/tx/($tx.txId)" + + if ($txResponse | get -i status) == "mined" { + echo $txResponse + break + } else { + echo $txResponse + sleep 1sec + } +} + +echo "Success!" diff --git a/src/aws/ethers_signer.rs b/src/aws/ethers_signer.rs index f8129ab..7c8eb89 100644 --- a/src/aws/ethers_signer.rs +++ b/src/aws/ethers_signer.rs @@ -310,6 +310,7 @@ impl ethers::signers::Signer for AwsSigner { #[cfg(test)] mod tests { + use aws_config::BehaviorVersion; use aws_credential_types::Credentials; use aws_sdk_kms::Client as KmsClient; use aws_types::region::Region; @@ -324,7 +325,7 @@ mod tests { let credentials = Credentials::from_keys(access_key, secret_access_key, None); - let config = aws_config::from_env() + let config = aws_config::defaults(BehaviorVersion::latest()) .credentials_provider(credentials) .region(Region::new("us-west-1")) .load() @@ -335,7 +336,7 @@ mod tests { #[allow(dead_code)] async fn env_client() -> KmsClient { - let config = aws_config::from_env() + let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new("us-west-1")) .load() .await; diff --git a/src/db.rs b/src/db.rs index 62fadef..c7f1be3 100644 --- a/src/db.rs +++ b/src/db.rs @@ -251,24 +251,29 @@ impl Database { Ok(()) } - pub async fn get_latest_block_number( + pub async fn get_latest_block_number_without_fee_estimates( &self, chain_id: u64, - ) -> eyre::Result { - let (block_number,): (i64,) = sqlx::query_as( + ) -> eyre::Result> { + let block_number: Option<(i64,)> = sqlx::query_as( r#" SELECT block_number FROM blocks WHERE chain_id = $1 + AND block_number NOT IN ( + SELECT block_number + FROM block_fees + WHERE chain_id = $1 + ) ORDER BY block_number DESC LIMIT 1 "#, ) .bind(chain_id as i64) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; - Ok(block_number as u64) + Ok(block_number.map(|(n,)| n as u64)) } pub async fn get_latest_block_fees_by_chain_id( @@ -368,6 +373,11 @@ impl Database { INSERT INTO block_txs (block_number, chain_id, tx_hash) SELECT $1, $2, unnested.tx_hash FROM UNNEST($3::BYTEA[]) AS unnested(tx_hash) + WHERE EXISTS ( + SELECT 1 + FROM tx_hashes + WHERE tx_hashes.tx_hash = unnested.tx_hash + ) "#, ) .bind(block_number as i64) diff --git a/src/main.rs b/src/main.rs index 7842066..faf405d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,11 @@ use tx_sitter::service::Service; #[command(author, version, about)] #[clap(rename_all = "kebab-case")] struct Args { - #[clap(short, long, default_value = "config.toml")] + #[clap(short, long)] + #[cfg_attr( + feature = "default-config", + clap(default_value = "config.toml") + )] config: Vec, #[clap(short, long)] diff --git a/src/tasks/index.rs b/src/tasks/index.rs index cd43323..c8d1db1 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -67,7 +67,18 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { loop { let latest_block_number = - app.db.get_latest_block_number(chain_id).await?; + app.db.get_latest_block_number_without_fee_estimates(chain_id).await?; + + let Some(latest_block_number) = latest_block_number else { + tracing::info!("No blocks to estimate fees for"); + + tokio::time::sleep(Duration::from_secs( + TIME_BETWEEN_FEE_ESTIMATION_SECONDS, + )) + .await; + + continue; + }; tracing::info!(block_number = latest_block_number, "Estimating fees"); diff --git a/tests/send_tx.rs b/tests/send_tx.rs index 8552c31..625356d 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -76,7 +76,7 @@ async fn send_tx() -> eyre::Result<()> { if balance == value { return Ok(()); } else { - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(5)).await; } } From 4f77401098b67bf83931968938db98798645d934 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 4 Dec 2023 22:13:36 +0100 Subject: [PATCH 038/135] Build and push image --- .github/workflows/build-image-and-publish.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/build-image-and-publish.yml diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml new file mode 100644 index 0000000..60e9483 --- /dev/null +++ b/.github/workflows/build-image-and-publish.yml @@ -0,0 +1,26 @@ +name: Build and Publish Docker Image + +on: + push: + branches: [ main, dev ] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: true + tags: dzejkop/tx-sitter:latest From 16de8e3b5316dec269e9b4373ab204ecef8e8c9f Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 00:26:33 +0100 Subject: [PATCH 039/135] Allow parts for db settings --- config.toml | 9 ++++ src/config.rs | 102 ++++++++++++++++++++++++++++++++++++++++---- src/db.rs | 14 +++--- src/tasks/index.rs | 11 +++-- tests/common/mod.rs | 4 +- 5 files changed, 115 insertions(+), 25 deletions(-) diff --git a/config.toml b/config.toml index fa4ed3e..fcb1ac3 100644 --- a/config.toml +++ b/config.toml @@ -6,8 +6,17 @@ host = "127.0.0.1:3000" disable_auth = false [database] +kind = "connection_string" connection_string = "postgres://postgres:postgres@127.0.0.1:5432/database" +# [database] +# kind = "parts" +# host = "127.0.0.1" +# port = "5432" +# username = "postgres" +# password = "postgres" +# database = "database" + [keys] kind = "local" diff --git a/src/config.rs b/src/config.rs index 069923b..e371edb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -28,12 +28,53 @@ pub struct ServerConfig { pub disable_auth: bool, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum DatabaseConfig { + ConnectionString(DbConnectionString), + Parts(DbParts), +} + +impl DatabaseConfig { + pub fn connection_string(s: impl ToString) -> Self { + Self::ConnectionString(DbConnectionString { + connection_string: s.to_string(), + }) + } + + pub fn to_connection_string(&self) -> String { + match self { + Self::ConnectionString(s) => s.connection_string.clone(), + Self::Parts(parts) => { + format!( + "postgres://{}:{}@{}:{}/{}", + parts.username, + parts.password, + parts.host, + parts.port, + parts.database + ) + } + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] -pub struct DatabaseConfig { +pub struct DbConnectionString { pub connection_string: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct DbParts { + pub host: String, + pub port: String, + pub username: String, + pub password: String, + pub database: String, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "kind")] pub enum KeysConfig { @@ -55,7 +96,7 @@ mod tests { use super::*; - const SAMPLE: &str = indoc! {r#" + const WITH_DB_CONNECTION_STRING: &str = indoc! {r#" [service] escalation_interval = "1h" @@ -64,14 +105,35 @@ mod tests { disable_auth = false [database] + kind = "connection_string" connection_string = "postgres://postgres:postgres@127.0.0.1:52804/database" [keys] kind = "local" "#}; + const WITH_DB_PARTS: &str = indoc! {r#" + [service] + escalation_interval = "1h" + + [server] + host = "127.0.0.1:3000" + disable_auth = false + + [database] + kind = "parts" + host = "host" + port = "5432" + username = "user" + password = "pass" + database = "db" + + [keys] + kind = "local" + "#}; + #[test] - fn sample() { + fn with_db_connection_string() { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), @@ -80,16 +142,40 @@ mod tests { host: SocketAddr::from(([127, 0, 0, 1], 3000)), disable_auth: false, }, - database: DatabaseConfig { - connection_string: - "postgres://postgres:postgres@127.0.0.1:52804/database" - .to_string(), + database: DatabaseConfig::connection_string( + "postgres://postgres:postgres@127.0.0.1:52804/database" + .to_string(), + ), + keys: KeysConfig::Local(LocalKeysConfig {}), + }; + + let toml = toml::to_string_pretty(&config).unwrap(); + + assert_eq!(toml, WITH_DB_CONNECTION_STRING); + } + + #[test] + fn with_db_parts() { + let config = Config { + service: TxSitterConfig { + escalation_interval: Duration::from_secs(60 * 60), + }, + server: ServerConfig { + host: SocketAddr::from(([127, 0, 0, 1], 3000)), + disable_auth: false, }, + database: DatabaseConfig::Parts(DbParts { + host: "host".to_string(), + port: "5432".to_string(), + username: "user".to_string(), + password: "pass".to_string(), + database: "db".to_string(), + }), keys: KeysConfig::Local(LocalKeysConfig {}), }; let toml = toml::to_string_pretty(&config).unwrap(); - assert_eq!(toml, SAMPLE); + assert_eq!(toml, WITH_DB_PARTS); } } diff --git a/src/db.rs b/src/db.rs index c7f1be3..4b13993 100644 --- a/src/db.rs +++ b/src/db.rs @@ -26,19 +26,20 @@ pub struct Database { impl Database { pub async fn new(config: &DatabaseConfig) -> eyre::Result { + let connection_string = config.to_connection_string(); let pool = loop { - if !Postgres::database_exists(&config.connection_string).await? { - Postgres::create_database(&config.connection_string).await?; + if !Postgres::database_exists(&connection_string).await? { + Postgres::create_database(&connection_string).await?; } - let pool = Pool::connect(&config.connection_string).await?; + let pool = Pool::connect(&connection_string).await?; if let Err(err) = MIGRATOR.run(&pool).await { tracing::error!("{err:?}"); tracing::warn!("Migration mismatch dropping previosu db"); drop(pool); // Drop the DB if it's out of date - ONLY FOR TESTING - Postgres::drop_database(&config.connection_string).await?; + Postgres::drop_database(&connection_string).await?; } else { break pool; } @@ -931,10 +932,7 @@ mod tests { let url = format!("postgres://postgres:postgres@{db_socket_addr}/database"); - let db = Database::new(&DatabaseConfig { - connection_string: url, - }) - .await?; + let db = Database::new(&DatabaseConfig::connection_string(url)).await?; Ok((db, db_container)) } diff --git a/src/tasks/index.rs b/src/tasks/index.rs index c8d1db1..903f99a 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -66,16 +66,15 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { let rpc = app.http_provider(chain_id).await?; loop { - let latest_block_number = - app.db.get_latest_block_number_without_fee_estimates(chain_id).await?; + let latest_block_number = app + .db + .get_latest_block_number_without_fee_estimates(chain_id) + .await?; let Some(latest_block_number) = latest_block_number else { tracing::info!("No blocks to estimate fees for"); - tokio::time::sleep(Duration::from_secs( - TIME_BETWEEN_FEE_ESTIMATION_SECONDS, - )) - .await; + tokio::time::sleep(Duration::from_secs(2)).await; continue; }; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index dff313b..d233f81 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -149,9 +149,7 @@ pub async fn setup_service( )), disable_auth: true, }, - database: DatabaseConfig { - connection_string: db_connection_url.to_string(), - }, + database: DatabaseConfig::connection_string(db_connection_url), keys: KeysConfig::Local(LocalKeysConfig {}), }; From c14832553719659d00be3f2310c0d84d4ff3080a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 01:01:40 +0100 Subject: [PATCH 040/135] Prefetch deps + use sparse registries + log connection string --- .cargo/config.toml | 2 ++ Dockerfile | 1 + src/db.rs | 4 ++++ 3 files changed, 7 insertions(+) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..70f9eae --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[registries.crates-io] +protocol = "sparse" diff --git a/Dockerfile b/Dockerfile index 0c28393..abc018b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN rustup component add cargo COPY . . # Build the binary +RUN cargo fetch RUN cargo build --release --no-default-features # Make sure it runs diff --git a/src/db.rs b/src/db.rs index 4b13993..f30f8f1 100644 --- a/src/db.rs +++ b/src/db.rs @@ -27,6 +27,10 @@ pub struct Database { impl Database { pub async fn new(config: &DatabaseConfig) -> eyre::Result { let connection_string = config.to_connection_string(); + + // TODO: Remove this! + tracing::info!("Connecting to database: {}", connection_string); + let pool = loop { if !Postgres::database_exists(&connection_string).await? { Postgres::create_database(&connection_string).await?; From 413f38f33d79d2ed19ca66aebb989eb4d4c78ae3 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 01:53:52 +0100 Subject: [PATCH 041/135] Add GH actions --- .github/workflows/tests.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..da8b78a --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,25 @@ +name: "Run tests" +on: + push: + pull_request: + +jobs: + test: + name: cargo test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - run: cargo test --workspace + - run: cargo clippy --workspace --tests + + formatting: + name: cargo fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + - name: Rustfmt Check + uses: actions-rust-lang/rustfmt@v1 From a8b66dbfaacb71b6f2a51c382859466ab1b6bae0 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 12:19:45 +0100 Subject: [PATCH 042/135] Enable auth for admin routes --- Cargo.lock | 2 ++ Cargo.toml | 2 +- src/app.rs | 4 ---- src/config.rs | 21 +++++++++++++++------ src/server.rs | 8 +++++++- tests/common/mod.rs | 3 ++- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b560d61..b9e9345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4641,6 +4641,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ + "base64 0.21.5", "bitflags 2.4.1", "bytes", "futures-core", @@ -4648,6 +4649,7 @@ dependencies = [ "http", "http-body", "http-range-header", + "mime", "pin-project-lite", "tower-layer", "tower-service", diff --git a/Cargo.toml b/Cargo.toml index 04bf3c5..be920b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [ "json", "ansi", ] } -tower-http = { version = "0.4.4", features = ["trace"] } +tower-http = { version = "0.4.4", features = [ "trace", "auth" ] } uuid = { version = "0.8", features = ["v4"] } futures = "0.3" chrono = "0.4" diff --git a/src/app.rs b/src/app.rs index 4de5286..712423a 100644 --- a/src/app.rs +++ b/src/app.rs @@ -80,10 +80,6 @@ impl App { &self, api_token: &ApiKey, ) -> eyre::Result { - if self.config.server.disable_auth { - return Ok(true); - } - self.db .is_api_key_valid(&api_token.relayer_id, api_token.api_key_hash()) .await diff --git a/src/config.rs b/src/config.rs index e371edb..8836e9c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -24,8 +24,17 @@ pub struct TxSitterConfig { pub struct ServerConfig { pub host: SocketAddr, - #[serde(default)] - pub disable_auth: bool, + pub username: Option, + pub password: Option, +} + +impl ServerConfig { + pub fn credentials(&self) -> Option<(&str, &str)> { + let username = self.username.as_deref()?; + let password = self.password.as_deref()?; + + Some((username, password)) + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -102,7 +111,6 @@ mod tests { [server] host = "127.0.0.1:3000" - disable_auth = false [database] kind = "connection_string" @@ -118,7 +126,6 @@ mod tests { [server] host = "127.0.0.1:3000" - disable_auth = false [database] kind = "parts" @@ -140,7 +147,8 @@ mod tests { }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), - disable_auth: false, + username: None, + password: None, }, database: DatabaseConfig::connection_string( "postgres://postgres:postgres@127.0.0.1:52804/database" @@ -162,7 +170,8 @@ mod tests { }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), - disable_auth: false, + username: None, + password: None, }, database: DatabaseConfig::Parts(DbParts { host: "host".to_string(), diff --git a/src/server.rs b/src/server.rs index 9c7dc98..0bf6b5a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -6,6 +6,7 @@ use axum::routing::{get, post, IntoMakeService}; use axum::Router; use hyper::server::conn::AddrIncoming; use thiserror::Error; +use tower_http::validate_request::ValidateRequestHeaderLayer; use self::routes::relayer::{ create_relayer, create_relayer_api_key, get_relayer, relayer_rpc, @@ -73,7 +74,7 @@ pub async fn spawn_server( .route("/:api_token/rpc", post(relayer_rpc)) .with_state(app.clone()); - let admin_routes = Router::new() + let mut admin_routes = Router::new() .route("/relayer", post(create_relayer)) .route( "/relayer/:relayer_id", @@ -83,6 +84,11 @@ pub async fn spawn_server( .route("/network/:chain_id", post(routes::network::create_network)) .with_state(app.clone()); + if let Some((username, password)) = app.config.server.credentials() { + admin_routes = admin_routes + .layer(ValidateRequestHeaderLayer::basic(username, password)); + } + let v1_routes = Router::new() .nest("/api", api_routes) .nest("/admin", admin_routes); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index d233f81..50c0460 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -147,7 +147,8 @@ pub async fn setup_service( Ipv4Addr::new(127, 0, 0, 1), 0, )), - disable_auth: true, + username: None, + password: None, }, database: DatabaseConfig::connection_string(db_connection_url), keys: KeysConfig::Local(LocalKeysConfig {}), From f657f5cc654a6f7c0b0bd8f286efe01dc367bd37 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 12:29:25 +0100 Subject: [PATCH 043/135] Speed up tests --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da8b78a..493f5dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions-rust-lang/setup-rust-toolchain@v1 - - run: cargo test --workspace + - name: Install latest nextest release + uses: taiki-e/install-action@nextest + - run: cargo nextest run --workspace + env: + RUST_LOG: info,tx-sitter=debug - run: cargo clippy --workspace --tests formatting: From 4a0d9e1770b251cf2a95093920cc4c1583120e6a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 14:14:47 +0100 Subject: [PATCH 044/135] Install foundry for tests --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 493f5dc..65c0030 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,6 +12,10 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install latest nextest release uses: taiki-e/install-action@nextest + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + with: + version: nightly - run: cargo nextest run --workspace env: RUST_LOG: info,tx-sitter=debug From 7766fa949280e77733854effc6cf9f3956a8e81d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 15:13:46 +0100 Subject: [PATCH 045/135] Add health endpoint --- crates/fake-rpc/src/lib.rs | 4 ++-- src/server.rs | 1 + src/server/routes.rs | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/fake-rpc/src/lib.rs b/crates/fake-rpc/src/lib.rs index 1e252b6..48f88f7 100644 --- a/crates/fake-rpc/src/lib.rs +++ b/crates/fake-rpc/src/lib.rs @@ -90,10 +90,10 @@ async fn rpc( anvil.advance().await.unwrap(); } - anvil.reference_anvil.lock().await + anvil.main_anvil.lock().await } "eth_getTransactionReceipt" => anvil.main_anvil.lock().await, - "eth_getTransactionByHash" => anvil.reference_anvil.lock().await, + "eth_getTransactionByHash" => anvil.main_anvil.lock().await, _ => anvil.main_anvil.lock().await, }; diff --git a/src/server.rs b/src/server.rs index 0bf6b5a..41295c6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -95,6 +95,7 @@ pub async fn spawn_server( let router = Router::new() .nest("/1", v1_routes) + .route("/health", get(routes::health)) .layer(tower_http::trace::TraceLayer::new_for_http()) .layer(axum::middleware::from_fn(middleware::log_response)); diff --git a/src/server/routes.rs b/src/server/routes.rs index 4675703..de17332 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -1,3 +1,9 @@ +use hyper::StatusCode; + pub mod network; pub mod relayer; pub mod transaction; + +pub async fn health() -> StatusCode { + StatusCode::OK +} From 3e55932a3abaf70e675713aaf123fd457377559d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 15:22:32 +0100 Subject: [PATCH 046/135] Fix anvil issue --- crates/fake-rpc/src/lib.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/fake-rpc/src/lib.rs b/crates/fake-rpc/src/lib.rs index 48f88f7..81c334e 100644 --- a/crates/fake-rpc/src/lib.rs +++ b/crates/fake-rpc/src/lib.rs @@ -1,6 +1,7 @@ use std::net::{Ipv4Addr, SocketAddr}; use std::sync::atomic::AtomicBool; use std::sync::Arc; +use std::time::Duration; use axum::extract::State; use axum::routing::{post, IntoMakeService}; @@ -11,6 +12,8 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio::sync::Mutex; +pub const BLOCK_TIME_SECONDS: u64 = 2; + pub struct DoubleAnvil { main_anvil: Mutex, reference_anvil: Mutex, @@ -90,10 +93,10 @@ async fn rpc( anvil.advance().await.unwrap(); } - anvil.main_anvil.lock().await + anvil.reference_anvil.lock().await } "eth_getTransactionReceipt" => anvil.main_anvil.lock().await, - "eth_getTransactionByHash" => anvil.main_anvil.lock().await, + "eth_getTransactionByHash" => anvil.reference_anvil.lock().await, _ => anvil.main_anvil.lock().await, }; @@ -119,8 +122,10 @@ pub async fn serve( Arc, axum::Server>, ) { - let main_anvil = Anvil::new().spawn(); - let reference_anvil = Anvil::new().spawn(); + let main_anvil = Anvil::new().block_time(BLOCK_TIME_SECONDS).spawn(); + let reference_anvil = Anvil::new().block_time(BLOCK_TIME_SECONDS).spawn(); + + tokio::time::sleep(Duration::from_secs(BLOCK_TIME_SECONDS)).await; tracing::info!("Main anvil instance: {}", main_anvil.endpoint()); tracing::info!("Reference anvil instance: {}", reference_anvil.endpoint()); From 185285516124f38379ab4402e3eb25290b3d1bb2 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 15:29:38 +0100 Subject: [PATCH 047/135] Cache dependencies in docker builds --- Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Dockerfile b/Dockerfile index abc018b..ae9dd47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,25 @@ ENV CARGO_HOME="/root/.cargo" # Install the toolchain RUN rustup component add cargo +# TODO: Hacky but it works +RUN mkdir -p ./src +RUN mkdir -p ./crates/postgres-docker-utils/src +RUN mkdir -p ./crates/fake-rpc/src + +# Copy only Cargo.toml for better caching +COPY ./Cargo.toml ./Cargo.toml +COPY ./Cargo.lock ./Cargo.lock +COPY ./crates/postgres-docker-utils/Cargo.toml ./crates/postgres-docker-utils/Cargo.toml +COPY ./crates/fake-rpc/Cargo.toml ./crates/fake-rpc/Cargo.toml + +RUN echo "fn main() {}" > ./src/main.rs +RUN echo "fn main() {}" > ./crates/postgres-docker-utils/src/main.rs +RUN echo "fn main() {}" > ./crates/fake-rpc/src/main.rs + +# Prebuild dependencies +RUN cargo fetch +RUN cargo build --release --no-default-features + # Copy all the source files # .dockerignore ignores the target dir COPY . . From aec0893de6589147724bae775e42229ccbdb379f Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 17:46:17 +0100 Subject: [PATCH 048/135] Remove secrets leak --- src/db.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/db.rs b/src/db.rs index f30f8f1..af4e716 100644 --- a/src/db.rs +++ b/src/db.rs @@ -28,9 +28,6 @@ impl Database { pub async fn new(config: &DatabaseConfig) -> eyre::Result { let connection_string = config.to_connection_string(); - // TODO: Remove this! - tracing::info!("Connecting to database: {}", connection_string); - let pool = loop { if !Postgres::database_exists(&connection_string).await? { Postgres::create_database(&connection_string).await?; From 5c09c1f6576ac9040f1d9c84686bf5f6f28ffa28 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 17:48:54 +0100 Subject: [PATCH 049/135] Use cache --- .github/workflows/build-image-and-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 60e9483..9166a51 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -24,3 +24,5 @@ jobs: file: Dockerfile push: true tags: dzejkop/tx-sitter:latest + cache-from: type=gha + cache-to: type=gha,mode=max From 157c12b4aee33e4df31240594e8d0a0a657d6bfd Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 5 Dec 2023 18:57:31 +0100 Subject: [PATCH 050/135] Use buildx --- .github/workflows/build-image-and-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 9166a51..e3191c3 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -17,6 +17,9 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push Docker image uses: docker/build-push-action@v2 with: From d32f749efe2146d482db33d23b7e08c6d0f9472a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 01:19:22 +0100 Subject: [PATCH 051/135] Add clippy --- rust-toolchain.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 381275b..b6e0a4d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,3 @@ [toolchain] channel = "nightly-2023-11-15" +components = [ "clippy" ] From 5642de939205a63d6be56c8334580d7f6d36935d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 09:29:52 +0100 Subject: [PATCH 052/135] Add getTxs endpoint --- src/db.rs | 19 +++++++++++++++++ src/server.rs | 3 ++- src/server/routes/transaction.rs | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/db.rs b/src/db.rs index af4e716..2d89122 100644 --- a/src/db.rs +++ b/src/db.rs @@ -718,6 +718,25 @@ impl Database { .await?) } + pub async fn read_txs( + &self, + relayer_id: &str, + ) -> eyre::Result> { + Ok(sqlx::query_as( + r#" + SELECT t.id as tx_id, t.tx_to as to, t.data, t.value, t.gas_limit, t.nonce, + h.tx_hash, s.status + FROM transactions t + LEFT JOIN sent_transactions s ON t.id = s.tx_id + LEFT JOIN tx_hashes h ON s.valid_tx_hash = h.tx_hash + WHERE t.relayer_id = $1 + "#, + ) + .bind(relayer_id) + .fetch_all(&self.pool) + .await?) + } + pub async fn get_relayer_addresses( &self, chain_id: u64, diff --git a/src/server.rs b/src/server.rs index 41295c6..fb9a3dd 100644 --- a/src/server.rs +++ b/src/server.rs @@ -12,7 +12,7 @@ use self::routes::relayer::{ create_relayer, create_relayer_api_key, get_relayer, relayer_rpc, update_relayer, }; -use self::routes::transaction::{get_tx, send_tx}; +use self::routes::transaction::{get_tx, get_txs, send_tx}; use crate::app::App; mod middleware; @@ -71,6 +71,7 @@ pub async fn spawn_server( let api_routes = Router::new() .route("/:api_token/tx", post(send_tx)) .route("/:api_token/tx/:tx_id", get(get_tx)) + .route("/:api_token/txs", get(get_txs)) .route("/:api_token/rpc", post(relayer_rpc)) .with_state(app.clone()); diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index a4f0064..25a5f6f 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -99,6 +99,41 @@ pub async fn send_tx( Ok(Json(SendTxResponse { tx_id })) } +#[tracing::instrument(skip(app, api_token))] +pub async fn get_txs( + State(app): State>, + Path((api_token, tx_id)): Path<(ApiKey, String)>, +) -> Result>, ApiError> { + if !app.is_authorized(&api_token).await? { + return Err(ApiError::Unauthorized); + } + + let txs = app.db.read_txs(&tx_id).await?; + + let txs = txs + .into_iter() + .map(|tx| GetTxResponse { + tx_id: tx.tx_id, + to: tx.to.0, + data: if tx.data.is_empty() { + None + } else { + Some(tx.data.into()) + }, + value: tx.value.0, + gas_limit: tx.gas_limit.0, + nonce: tx.nonce, + tx_hash: tx.tx_hash.map(|h| h.0), + status: tx + .status + .map(GetTxResponseStatus::TxStatus) + .unwrap_or(GetTxResponseStatus::Unsent(UnsentStatus::Unsent)), + }) + .collect(); + + Ok(Json(txs)) +} + #[tracing::instrument(skip(app))] pub async fn get_tx( State(app): State>, From b7201818290c68665213455cc7995abfb2afd997 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 09:32:08 +0100 Subject: [PATCH 053/135] WIP: Integrate telemetry-batteries --- Cargo.lock | 267 ++++++++++++++++++++++++++++++- Cargo.toml | 4 + config.toml | 1 + src/config.rs | 7 + src/main.rs | 23 ++- src/server/routes/transaction.rs | 39 +++-- tests/common/mod.rs | 1 + 7 files changed, 311 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9e9345..a51d2d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,6 +782,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cadence" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f39286bc075b023101dccdb79456a1334221c768b8faede0c2aff7ed29a9482d" +dependencies = [ + "crossbeam-channel", +] + [[package]] name = "camino" version = "1.1.6" @@ -1045,6 +1054,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -2013,7 +2032,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -2308,6 +2327,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.1.0" @@ -2597,6 +2626,39 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash 0.8.6", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-statsd" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34a620eecf9e4321ebbef8f2f8e7cd22e098f11b65f2d987ce66faaa8918418" +dependencies = [ + "cadence", + "metrics", + "thiserror", +] + +[[package]] +name = "metrics-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "mime" version = "0.3.17" @@ -2791,12 +2853,111 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-datadog" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f4ecf595095d3b641dd2761a0c3d1f175d3d6c28f38e65418d8004ea3255dd" +dependencies = [ + "futures-core", + "http", + "indexmap 1.9.3", + "itertools 0.10.5", + "once_cell", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-semantic-conventions", + "reqwest", + "rmp", + "thiserror", + "url", +] + +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float", + "percent-encoding", + "rand", + "regex", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2995,7 +3156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.1.0", ] [[package]] @@ -3118,6 +3279,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + [[package]] name = "postgres-docker-utils" version = "0.1.0" @@ -3491,6 +3658,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmp" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + [[package]] name = "ron" version = "0.7.1" @@ -4020,7 +4198,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap", + "indexmap 2.1.0", "log", "memchr", "once_cell", @@ -4332,6 +4510,30 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "telemetry-batteries" +version = "0.1.0" +source = "git+https://github.com/worldcoin/telemetry-batteries#42d074d9cd87889b9244e4b2823c73dc03596007" +dependencies = [ + "chrono", + "dirs", + "http", + "metrics", + "metrics-exporter-statsd", + "opentelemetry", + "opentelemetry-datadog", + "opentelemetry-http", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-appender", + "tracing-opentelemetry", + "tracing-serde", + "tracing-subscriber", +] + [[package]] name = "tempfile" version = "3.8.1" @@ -4590,7 +4792,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap", + "indexmap 2.1.0", "toml_datetime", "winnow", ] @@ -4601,7 +4803,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap", + "indexmap 2.1.0", "toml_datetime", "winnow", ] @@ -4612,7 +4814,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap", + "indexmap 2.1.0", "serde", "serde_spanned", "toml_datetime", @@ -4680,6 +4882,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.27" @@ -4711,6 +4925,44 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +dependencies = [ + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -4734,9 +4986,11 @@ dependencies = [ "serde", "serde_json", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log 0.2.0", "tracing-serde", ] @@ -4806,6 +5060,7 @@ dependencies = [ "spki", "sqlx", "strum", + "telemetry-batteries", "test-case", "thiserror", "tokio", diff --git a/Cargo.toml b/Cargo.toml index be920b0..d5bfd8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "tx-sitter" version = "0.1.0" edition = "2021" +default-run = "tx-sitter" [workspace] members = ["crates/*"] @@ -71,6 +72,9 @@ async-trait = "0.1.74" itertools = "0.12.0" base64 = "0.21.5" +# Company +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries" } + # Internal postgres-docker-utils = { path = "crates/postgres-docker-utils" } diff --git a/config.toml b/config.toml index fcb1ac3..d34c83f 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,6 @@ [service] escalation_interval = "1m" +datadog_enabled = true [server] host = "127.0.0.1:3000" diff --git a/src/config.rs b/src/config.rs index 8836e9c..5813ba4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,9 @@ pub struct Config { pub struct TxSitterConfig { #[serde(with = "humantime_serde")] pub escalation_interval: Duration, + + #[serde(default)] + pub datadog_enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -108,6 +111,7 @@ mod tests { const WITH_DB_CONNECTION_STRING: &str = indoc! {r#" [service] escalation_interval = "1h" + datadog_enabled = false [server] host = "127.0.0.1:3000" @@ -123,6 +127,7 @@ mod tests { const WITH_DB_PARTS: &str = indoc! {r#" [service] escalation_interval = "1h" + datadog_enabled = false [server] host = "127.0.0.1:3000" @@ -144,6 +149,7 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), @@ -167,6 +173,7 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), diff --git a/src/main.rs b/src/main.rs index faf405d..4913e51 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; +use telemetry_batteries::tracing::batteries::datadog::DatadogBattery; +use tracing::Level; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; @@ -33,11 +35,6 @@ async fn main() -> eyre::Result<()> { dotenv::from_path(path)?; } - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().pretty().compact()) - .with(EnvFilter::from_default_env()) - .init(); - let mut settings = config::Config::builder(); for arg in &args.config { @@ -60,6 +57,22 @@ async fn main() -> eyre::Result<()> { let config = settings.try_deserialize::()?; + if config.service.datadog_enabled { + let datadog_battery = DatadogBattery::new( + Some("http://localhost:8126"), + Level::DEBUG, + "tx-sitter", + None, + ); + + datadog_battery.init()?; + } else { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().pretty().compact()) + .with(EnvFilter::from_default_env()) + .init(); + } + let service = Service::new(config).await?; service.wait().await?; diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 25a5f6f..b51c151 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -110,26 +110,25 @@ pub async fn get_txs( let txs = app.db.read_txs(&tx_id).await?; - let txs = txs - .into_iter() - .map(|tx| GetTxResponse { - tx_id: tx.tx_id, - to: tx.to.0, - data: if tx.data.is_empty() { - None - } else { - Some(tx.data.into()) - }, - value: tx.value.0, - gas_limit: tx.gas_limit.0, - nonce: tx.nonce, - tx_hash: tx.tx_hash.map(|h| h.0), - status: tx - .status - .map(GetTxResponseStatus::TxStatus) - .unwrap_or(GetTxResponseStatus::Unsent(UnsentStatus::Unsent)), - }) - .collect(); + let txs = + txs.into_iter() + .map(|tx| GetTxResponse { + tx_id: tx.tx_id, + to: tx.to.0, + data: if tx.data.is_empty() { + None + } else { + Some(tx.data.into()) + }, + value: tx.value.0, + gas_limit: tx.gas_limit.0, + nonce: tx.nonce, + tx_hash: tx.tx_hash.map(|h| h.0), + status: tx.status.map(GetTxResponseStatus::TxStatus).unwrap_or( + GetTxResponseStatus::Unsent(UnsentStatus::Unsent), + ), + }) + .collect(); Ok(Json(txs)) } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 50c0460..6620594 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -141,6 +141,7 @@ pub async fn setup_service( let config = Config { service: TxSitterConfig { escalation_interval, + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::V4(SocketAddrV4::new( From 0ab9ab51e717b6fb2dfa69cfa1dc1ca80062b42b Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 12:43:11 +0100 Subject: [PATCH 054/135] Change service tag --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 4913e51..fe77f22 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,7 +61,7 @@ async fn main() -> eyre::Result<()> { let datadog_battery = DatadogBattery::new( Some("http://localhost:8126"), Level::DEBUG, - "tx-sitter", + "tx-sitter-monolith", None, ); From 384a46ab8af1d1c6a000dd94b2a558ac5485ca51 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 13:07:08 +0100 Subject: [PATCH 055/135] Update telemetry-batteries --- Cargo.lock | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a51d2d7..79627df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4513,7 +4513,7 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "telemetry-batteries" version = "0.1.0" -source = "git+https://github.com/worldcoin/telemetry-batteries#42d074d9cd87889b9244e4b2823c73dc03596007" +source = "git+https://github.com/worldcoin/telemetry-batteries#53c0934bf00e10c5326391e075be9cc625a6013a" dependencies = [ "chrono", "dirs", diff --git a/src/main.rs b/src/main.rs index fe77f22..867a23d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,7 +59,7 @@ async fn main() -> eyre::Result<()> { if config.service.datadog_enabled { let datadog_battery = DatadogBattery::new( - Some("http://localhost:8126"), + None, Level::DEBUG, "tx-sitter-monolith", None, From 6f39962e1bf1b499f6481ae7aa4cae3ef39bfd0d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 13:10:08 +0100 Subject: [PATCH 056/135] fmt --- src/main.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 867a23d..2047a59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,12 +58,8 @@ async fn main() -> eyre::Result<()> { let config = settings.try_deserialize::()?; if config.service.datadog_enabled { - let datadog_battery = DatadogBattery::new( - None, - Level::DEBUG, - "tx-sitter-monolith", - None, - ); + let datadog_battery = + DatadogBattery::new(None, Level::DEBUG, "tx-sitter-monolith", None); datadog_battery.init()?; } else { From c2315306444380f6458fd774c258e515bc942e46 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 13:10:13 +0100 Subject: [PATCH 057/135] Don't destroy the db --- src/db.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/db.rs b/src/db.rs index 2d89122..550844c 100644 --- a/src/db.rs +++ b/src/db.rs @@ -28,23 +28,13 @@ impl Database { pub async fn new(config: &DatabaseConfig) -> eyre::Result { let connection_string = config.to_connection_string(); - let pool = loop { - if !Postgres::database_exists(&connection_string).await? { - Postgres::create_database(&connection_string).await?; - } - - let pool = Pool::connect(&connection_string).await?; - - if let Err(err) = MIGRATOR.run(&pool).await { - tracing::error!("{err:?}"); - tracing::warn!("Migration mismatch dropping previosu db"); - drop(pool); - // Drop the DB if it's out of date - ONLY FOR TESTING - Postgres::drop_database(&connection_string).await?; - } else { - break pool; - } - }; + if !Postgres::database_exists(&connection_string).await? { + Postgres::create_database(&connection_string).await?; + } + + let pool = Pool::connect(&connection_string).await?; + + MIGRATOR.run(&pool).await?; Ok(Self { pool }) } From bc37a74297c2d1597846d258e81437178c1d8841 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 19:54:14 +0100 Subject: [PATCH 058/135] Remove redundant relayer id --- src/server/routes/transaction.rs | 3 +-- tests/send_many_txs.rs | 1 - tests/send_tx.rs | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index b51c151..b26a29b 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -14,7 +14,6 @@ use crate::types::TransactionPriority; #[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SendTxRequest { - pub relayer_id: String, pub to: Address, #[serde(with = "crate::serde_utils::decimal_u256")] pub value: U256, @@ -92,7 +91,7 @@ pub async fn send_tx( req.value, req.gas_limit, req.priority, - &req.relayer_id, + &api_token.relayer_id, ) .await?; diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 44106ee..ed6cfb3 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -64,7 +64,6 @@ async fn send_many_txs() -> eyre::Result<()> { .send_tx( &api_key, &SendTxRequest { - relayer_id: relayer_id.clone(), to: ARBITRARY_ADDRESS, value, gas_limit: U256::from(21_000), diff --git a/tests/send_tx.rs b/tests/send_tx.rs index 625356d..cff3991 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -61,7 +61,6 @@ async fn send_tx() -> eyre::Result<()> { .send_tx( &api_key, &SendTxRequest { - relayer_id, to: ARBITRARY_ADDRESS, value, gas_limit: U256::from(21_000), From 6afcde035439b4a70eee67d1b1110158e87f1e42 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 7 Dec 2023 13:36:05 +0100 Subject: [PATCH 059/135] Try unnested fields --- Cargo.lock | 6 +++--- Cargo.toml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79627df..7f6c8be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3281,9 +3281,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" [[package]] name = "postgres-docker-utils" @@ -4513,7 +4513,7 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "telemetry-batteries" version = "0.1.0" -source = "git+https://github.com/worldcoin/telemetry-batteries#53c0934bf00e10c5326391e075be9cc625a6013a" +source = "git+https://github.com/worldcoin/telemetry-batteries?branch=dzejkop/unnest-fields#8fa8fe137610b0e1e85a86a4cc9f6e4a78b23bbb" dependencies = [ "chrono", "dirs", diff --git a/Cargo.toml b/Cargo.toml index d5bfd8b..7e50cbd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,8 @@ itertools = "0.12.0" base64 = "0.21.5" # Company -telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries" } +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", branch = "dzejkop/unnest-fields" } +# telemetry-batteries = { path = "../telemetry-batteries" } # Internal postgres-docker-utils = { path = "crates/postgres-docker-utils" } From d07d83b60f6851f088c35e7807b08039a10f572f Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 7 Dec 2023 21:52:48 +0100 Subject: [PATCH 060/135] Use new telemetry-batteries --- Cargo.lock | 2 +- src/main.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f6c8be..cde4346 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4513,7 +4513,7 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "telemetry-batteries" version = "0.1.0" -source = "git+https://github.com/worldcoin/telemetry-batteries?branch=dzejkop/unnest-fields#8fa8fe137610b0e1e85a86a4cc9f6e4a78b23bbb" +source = "git+https://github.com/worldcoin/telemetry-batteries?branch=dzejkop/unnest-fields#4d64684669879cefae4b3c5ca047fdf4c2c62a7b" dependencies = [ "chrono", "dirs", diff --git a/src/main.rs b/src/main.rs index 2047a59..bc2c25c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,7 +59,8 @@ async fn main() -> eyre::Result<()> { if config.service.datadog_enabled { let datadog_battery = - DatadogBattery::new(None, Level::DEBUG, "tx-sitter-monolith", None); + DatadogBattery::new(None, "tx-sitter-monolith", None) + .with_location(); datadog_battery.init()?; } else { From 792b858e9ae01f7dcba18d63052d04a5115ca260 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 7 Dec 2023 21:59:36 +0100 Subject: [PATCH 061/135] Fix --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index bc2c25c..a8cd9e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,6 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; use telemetry_batteries::tracing::batteries::datadog::DatadogBattery; -use tracing::Level; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; From 5a576a4169b5ee09861762711105d3d2e4539e76 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 8 Dec 2023 00:37:18 +0100 Subject: [PATCH 062/135] Emit metrics periodically --- Cargo.lock | 1 + Cargo.toml | 1 + src/config.rs | 7 ++++ src/db.rs | 80 +++++++++++++++++++++++++++++++++++++++++++- src/db/data.rs | 9 +++++ src/main.rs | 14 ++++++++ src/service.rs | 4 +++ src/tasks.rs | 2 ++ src/tasks/metrics.rs | 36 ++++++++++++++++++++ tests/common/mod.rs | 1 + 10 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 src/tasks/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index cde4346..68fd11b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5050,6 +5050,7 @@ dependencies = [ "hyper", "indoc", "itertools 0.12.0", + "metrics", "num-bigint", "postgres-docker-utils", "rand", diff --git a/Cargo.toml b/Cargo.toml index 7e50cbd..f1f02e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ sqlx = { version = "0.7.2", features = [ "migrate", "bigdecimal", ] } +metrics = "0.21.1" num-bigint = "0.4.4" bigdecimal = "0.4.2" spki = "0.7.2" diff --git a/src/config.rs b/src/config.rs index 5813ba4..53ad624 100644 --- a/src/config.rs +++ b/src/config.rs @@ -20,6 +20,9 @@ pub struct TxSitterConfig { #[serde(default)] pub datadog_enabled: bool, + + #[serde(default)] + pub statsd_enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -112,6 +115,7 @@ mod tests { [service] escalation_interval = "1h" datadog_enabled = false + statsd_enabled = false [server] host = "127.0.0.1:3000" @@ -128,6 +132,7 @@ mod tests { [service] escalation_interval = "1h" datadog_enabled = false + statsd_enabled = false [server] host = "127.0.0.1:3000" @@ -150,6 +155,7 @@ mod tests { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), datadog_enabled: false, + statsd_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), @@ -174,6 +180,7 @@ mod tests { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), datadog_enabled: false, + statsd_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), diff --git a/src/db.rs b/src/db.rs index 550844c..8323a16 100644 --- a/src/db.rs +++ b/src/db.rs @@ -14,7 +14,9 @@ use crate::types::{RelayerInfo, RelayerUpdate, TransactionPriority}; pub mod data; -use self::data::{AddressWrapper, BlockFees, H256Wrapper, ReadTxData, RpcKind}; +use self::data::{ + AddressWrapper, BlockFees, H256Wrapper, NetworkStats, ReadTxData, RpcKind, +}; pub use self::data::{TxForEscalation, TxStatus, UnsentTx}; // Statically link in migration files @@ -924,6 +926,82 @@ impl Database { Ok(is_valid) } + + pub async fn get_stats(&self, chain_id: u64) -> eyre::Result { + let (pending_txs,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM transactions t + JOIN relayers r ON (t.relayer_id = r.id) + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + WHERE s.tx_id IS NULL + AND r.chain_id = $1 + "#, + ) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + let (mined_txs,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM transactions t + JOIN relayers r ON (t.relayer_id = r.id) + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + WHERE s.status = $1 + AND r.chain_id = $2 + "#, + ) + .bind(TxStatus::Mined) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + let (finalized_txs,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM transactions t + JOIN relayers r ON (t.relayer_id = r.id) + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + WHERE s.status = $1 + AND r.chain_id = $2 + "#, + ) + .bind(TxStatus::Finalized) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + let (total_indexed_blocks,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM blocks + WHERE chain_id = $1 + "#, + ) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + let (block_txs,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM block_txs + WHERE chain_id = $1 + "#, + ) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + Ok(NetworkStats { + pending_txs: pending_txs as u64, + mined_txs: mined_txs as u64, + finalized_txs: finalized_txs as u64, + total_indexed_blocks: total_indexed_blocks as u64, + block_txs: block_txs as u64, + }) + } } #[cfg(test)] diff --git a/src/db/data.rs b/src/db/data.rs index 5fccf40..7e8bd80 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -58,6 +58,15 @@ pub struct ReadTxData { pub status: Option, } +#[derive(Debug, Clone)] +pub struct NetworkStats { + pub pending_txs: u64, + pub mined_txs: u64, + pub finalized_txs: u64, + pub total_indexed_blocks: u64, + pub block_txs: u64, +} + #[derive(Debug, Clone)] pub struct BlockFees { pub fee_estimates: FeesEstimate, diff --git a/src/main.rs b/src/main.rs index a8cd9e9..18e8cca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; +use telemetry_batteries::metrics::statsd::StatsdBattery; +use telemetry_batteries::metrics::MetricsBattery; use telemetry_batteries::tracing::batteries::datadog::DatadogBattery; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; @@ -69,6 +71,18 @@ async fn main() -> eyre::Result<()> { .init(); } + if config.service.statsd_enabled { + let statsd_battery = StatsdBattery::new( + "localhost", + 8125, + 5000, + 1024, + Some("tx_sitter_monolith"), + )?; + + statsd_battery.init()?; + } + let service = Service::new(config).await?; service.wait().await?; diff --git a/src/service.rs b/src/service.rs index cef7e49..599b0ac 100644 --- a/src/service.rs +++ b/src/service.rs @@ -29,6 +29,10 @@ impl Service { task_runner.add_task("Handle soft reorgs", tasks::handle_soft_reorgs); task_runner.add_task("Handle hard reorgs", tasks::handle_hard_reorgs); + if app.config.service.statsd_enabled { + task_runner.add_task("Emit metrics", tasks::emit_metrics); + } + for chain_id in chain_ids { Self::spawn_chain_tasks(&task_runner, chain_id)?; } diff --git a/src/tasks.rs b/src/tasks.rs index e15f411..b86014b 100644 --- a/src/tasks.rs +++ b/src/tasks.rs @@ -3,6 +3,7 @@ pub mod escalate; pub mod finalize; pub mod handle_reorgs; pub mod index; +pub mod metrics; pub mod prune; pub use self::broadcast::broadcast_txs; @@ -10,4 +11,5 @@ pub use self::escalate::escalate_txs; pub use self::finalize::finalize_txs; pub use self::handle_reorgs::{handle_hard_reorgs, handle_soft_reorgs}; pub use self::index::index_chain; +pub use self::metrics::emit_metrics; pub use self::prune::{prune_blocks, prune_txs}; diff --git a/src/tasks/metrics.rs b/src/tasks/metrics.rs new file mode 100644 index 0000000..4584815 --- /dev/null +++ b/src/tasks/metrics.rs @@ -0,0 +1,36 @@ +use std::sync::Arc; +use std::time::Duration; + +use crate::app::App; + +const EMIT_METRICS_INTERVAL: Duration = Duration::from_secs(1); + +pub async fn emit_metrics(app: Arc) -> eyre::Result<()> { + loop { + let chain_ids = app.db.get_network_chain_ids().await?; + + for chain_id in chain_ids { + let stats = app.db.get_stats(chain_id).await?; + + // TODO: Add labels for env, etc. + let labels = [("chain_id", chain_id.to_string())]; + + metrics::gauge!("pending_txs", stats.pending_txs as f64, &labels); + metrics::gauge!("mined_txs", stats.mined_txs as f64, &labels); + metrics::gauge!( + "finalized_txs", + stats.finalized_txs as f64, + &labels + ); + metrics::gauge!( + "total_indexed_blocks", + stats.total_indexed_blocks as f64, + &labels + ); + metrics::gauge!("block_fees", stats.block_txs as f64, &labels); + metrics::gauge!("block_txs", stats.block_txs as f64, &labels); + } + + tokio::time::sleep(EMIT_METRICS_INTERVAL).await; + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6620594..c24a844 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -142,6 +142,7 @@ pub async fn setup_service( service: TxSitterConfig { escalation_interval, datadog_enabled: false, + statsd_enabled: false, }, server: ServerConfig { host: SocketAddr::V4(SocketAddrV4::new( From 07439b18825d8b2d20077b282309588ff2ace5a3 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Fri, 8 Dec 2023 01:39:46 +0100 Subject: [PATCH 063/135] Fix get_txs bug --- src/server/routes/transaction.rs | 4 ++-- src/tasks/metrics.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index b26a29b..0ea5513 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -101,13 +101,13 @@ pub async fn send_tx( #[tracing::instrument(skip(app, api_token))] pub async fn get_txs( State(app): State>, - Path((api_token, tx_id)): Path<(ApiKey, String)>, + Path(api_token): Path, ) -> Result>, ApiError> { if !app.is_authorized(&api_token).await? { return Err(ApiError::Unauthorized); } - let txs = app.db.read_txs(&tx_id).await?; + let txs = app.db.read_txs(&api_token.relayer_id).await?; let txs = txs.into_iter() diff --git a/src/tasks/metrics.rs b/src/tasks/metrics.rs index 4584815..fb945b1 100644 --- a/src/tasks/metrics.rs +++ b/src/tasks/metrics.rs @@ -27,7 +27,6 @@ pub async fn emit_metrics(app: Arc) -> eyre::Result<()> { stats.total_indexed_blocks as f64, &labels ); - metrics::gauge!("block_fees", stats.block_txs as f64, &labels); metrics::gauge!("block_txs", stats.block_txs as f64, &labels); } From 2e79308becfa0920f9d2f6a090d8feb9f7600985 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 10:24:43 +0100 Subject: [PATCH 064/135] misc --- config.toml | 3 ++- src/db.rs | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config.toml b/config.toml index d34c83f..f2f5ec2 100644 --- a/config.toml +++ b/config.toml @@ -1,6 +1,7 @@ [service] escalation_interval = "1m" -datadog_enabled = true +datadog_enabled = false +statsd_enabled = false [server] host = "127.0.0.1:3000" diff --git a/src/db.rs b/src/db.rs index 8323a16..7cad2df 100644 --- a/src/db.rs +++ b/src/db.rs @@ -507,6 +507,9 @@ impl Database { Ok(()) } + /// Marks txs as mined if the associated tx hash is present in a block + /// + /// returns cumulative gas used for all txs pub async fn mine_txs(&self, chain_id: u64) -> eyre::Result<()> { let mut tx = self.pool.begin().await?; From a82dcd99f992eeccae285354fe21cef4b95eb06a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 10:46:56 +0100 Subject: [PATCH 065/135] Big query & mining metrics --- src/db.rs | 77 ++++++++++++++++++---------------------------- src/tasks/index.rs | 23 ++++++++++---- 2 files changed, 47 insertions(+), 53 deletions(-) diff --git a/src/db.rs b/src/db.rs index 7cad2df..c0905ed 100644 --- a/src/db.rs +++ b/src/db.rs @@ -509,61 +509,44 @@ impl Database { /// Marks txs as mined if the associated tx hash is present in a block /// - /// returns cumulative gas used for all txs - pub async fn mine_txs(&self, chain_id: u64) -> eyre::Result<()> { - let mut tx = self.pool.begin().await?; - - // Fetch txs which are marked as pending but have an associated tx - // present in in one of the block txs - let items: Vec<(String, H256Wrapper, DateTime)> = sqlx::query_as( + /// returns the tx ids and hashes for all mined txs + pub async fn mine_txs( + &self, + chain_id: u64, + ) -> eyre::Result> { + let updated_txs: Vec<(String, H256Wrapper)> = sqlx::query_as( r#" - SELECT t.id, h.tx_hash, b.timestamp - FROM transactions t - JOIN sent_transactions s ON t.id = s.tx_id - JOIN tx_hashes h ON t.id = h.tx_id - JOIN block_txs bt ON h.tx_hash = bt.tx_hash - JOIN blocks b ON bt.block_number = b.block_number AND bt.chain_id = b.chain_id - WHERE s.status = $1 - AND b.chain_id = $2 + WITH cte AS ( + SELECT t.id, h.tx_hash, b.timestamp + FROM transactions t + JOIN sent_transactions s ON t.id = s.tx_id + JOIN tx_hashes h ON t.id = h.tx_id + JOIN block_txs bt ON h.tx_hash = bt.tx_hash + JOIN blocks b ON + bt.block_number = b.block_number + AND bt.chain_id = b.chain_id + WHERE s.status = $1 + AND b.chain_id = $2 + ) + UPDATE sent_transactions + SET status = $3, + valid_tx_hash = cte.tx_hash, + mined_at = cte.timestamp + FROM cte + WHERE sent_transactions.tx_id = cte.id + RETURNING sent_transactions.tx_id, sent_transactions.valid_tx_hash "#, ) .bind(TxStatus::Pending) .bind(chain_id as i64) - .fetch_all(tx.as_mut()) - .await?; - - let mut tx_ids = Vec::new(); - let mut tx_hashes = Vec::new(); - let mut timestamps = Vec::new(); - - for (tx_id, tx_hash, timestamp) in items { - tx_ids.push(tx_id); - tx_hashes.push(tx_hash); - timestamps.push(timestamp); - } - - sqlx::query( - r#" - UPDATE sent_transactions s - SET status = $1, - valid_tx_hash = mined.tx_hash, - mined_at = mined.timestamp - FROM transactions t, - UNNEST($2::TEXT[], $3::BYTEA[], $4::TIMESTAMPTZ[]) AS mined(tx_id, tx_hash, timestamp) - WHERE t.id = mined.tx_id - AND t.id = s.tx_id - "#, - ) .bind(TxStatus::Mined) - .bind(&tx_ids) - .bind(&tx_hashes) - .bind(×tamps) - .execute(tx.as_mut()) + .fetch_all(&self.pool) .await?; - tx.commit().await?; - - Ok(()) + Ok(updated_txs + .into_iter() + .map(|(id, hash)| (id, hash.0)) + .collect()) } pub async fn finalize_txs( diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 903f99a..fe430b2 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -51,7 +51,18 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { ) .await?; - app.db.mine_txs(chain_id).await?; + let mined_txs = app.db.mine_txs(chain_id).await?; + + let metric_labels = [("chain_id", chain_id.to_string())]; + for tx in mined_txs { + tracing::info!( + id = tx.0, + hash = ?tx.1, + "Tx mined" + ); + + metrics::increment_counter!("tx_mined", &metric_labels); + } let relayer_addresses = app.db.get_relayer_addresses(chain_id).await?; @@ -118,11 +129,11 @@ async fn update_relayer_nonces( let tx_count = rpc.get_transaction_count(relayer_address, None).await?; - // tracing::info!( - // nonce = ?tx_count, - // ?relayer_address, - // "Updating relayer nonce" - // ); + tracing::info!( + nonce = ?tx_count, + ?relayer_address, + "Updating relayer nonce" + ); app.db .update_relayer_nonce( From dad7ff525b8e5000935a3e5f44da0a8a831ad55e Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 10:53:19 +0100 Subject: [PATCH 066/135] More logging & metrics --- src/db.rs | 18 ++++++++++-------- src/tasks/handle_reorgs.rs | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/db.rs b/src/db.rs index c0905ed..9a68140 100644 --- a/src/db.rs +++ b/src/db.rs @@ -410,7 +410,8 @@ impl Database { Ok(()) } - pub async fn handle_soft_reorgs(&self) -> eyre::Result<()> { + /// Returns a list of soft reorged txs + pub async fn handle_soft_reorgs(&self) -> eyre::Result> { let mut tx = self.pool.begin().await?; // Fetch txs which have valid tx hash different than what is actually mined @@ -448,10 +449,11 @@ impl Database { tx.commit().await?; - Ok(()) + Ok(tx_ids) } - pub async fn handle_hard_reorgs(&self) -> eyre::Result<()> { + /// Returns a list of hard reorged txs + pub async fn handle_hard_reorgs(&self) -> eyre::Result> { let mut tx = self.pool.begin().await?; // Fetch txs which are marked as mined @@ -466,10 +468,10 @@ impl Database { LEFT JOIN block_txs bt ON h.tx_hash = bt.tx_hash WHERE s.status = $1 ) - SELECT t.id - FROM reorg_candidates t - GROUP BY t.id - HAVING COUNT(t.chain_id) = 0 + SELECT r.id + FROM reorg_candidates r + GROUP BY r.id + HAVING COUNT(r.chain_id) = 0 "#, ) .bind(TxStatus::Mined) @@ -504,7 +506,7 @@ impl Database { tx.commit().await?; - Ok(()) + Ok(tx_ids) } /// Marks txs as mined if the associated tx hash is present in a block diff --git a/src/tasks/handle_reorgs.rs b/src/tasks/handle_reorgs.rs index 699ef5c..dd91dab 100644 --- a/src/tasks/handle_reorgs.rs +++ b/src/tasks/handle_reorgs.rs @@ -11,7 +11,14 @@ pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { loop { tracing::info!("Handling hard reorgs"); - app.db.handle_hard_reorgs().await?; + let reorged_txs = app.db.handle_hard_reorgs().await?; + + for tx in reorged_txs { + tracing::info!( + id = tx, + "Tx hard reorged" + ); + } tokio::time::sleep(Duration::from_secs( TIME_BETWEEN_HARD_REORGS_SECONDS as u64, @@ -24,7 +31,14 @@ pub async fn handle_soft_reorgs(app: Arc) -> eyre::Result<()> { loop { tracing::info!("Handling soft reorgs"); - app.db.handle_soft_reorgs().await?; + let txs = app.db.handle_soft_reorgs().await?; + + for tx in txs { + tracing::info!( + id = tx, + "Tx soft reorged" + ); + } tokio::time::sleep(Duration::from_secs( TIME_BETWEEN_SOFT_REORGS_SECONDS as u64, From 956ea8d9b16c8767a8fd6c980a4f461ce83ac538 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 11:02:16 +0100 Subject: [PATCH 067/135] Fix escalation fee logic --- src/tasks/escalate.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 9725842..59797eb 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -37,22 +37,19 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { .context("Missing block")?; // Min increase of 20% on the priority fee required for a replacement tx + let factor = U256::from(100); let increased_gas_price_percentage = - U256::from(100 + (10 * (1 + escalation))); + factor + U256::from(10 * (1 + escalation)); - let factor = U256::from(100); + let max_fee_per_gas_increase = tx.initial_max_fee_per_gas.0 + * increased_gas_price_percentage + / factor; - let max_priority_fee_per_gas_increase = - tx.initial_max_priority_fee_per_gas.0 - * increased_gas_price_percentage - / factor; + let max_fee_per_gas = + tx.initial_max_fee_per_gas.0 + max_fee_per_gas_increase; let max_priority_fee_per_gas = - tx.initial_max_priority_fee_per_gas.0 - + max_priority_fee_per_gas_increase; - - let max_fee_per_gas = - fees.fee_estimates.base_fee_per_gas + max_priority_fee_per_gas; + max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; let eip1559_tx = Eip1559TransactionRequest { from: None, @@ -84,8 +81,6 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { let tx_hash = pending_tx.tx_hash(); - tracing::info!(?tx.id, ?tx_hash, "Tx escalated"); - app.db .escalate_tx( &tx.id, @@ -94,6 +89,8 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { max_priority_fee_per_gas, ) .await?; + + tracing::info!(?tx.id, ?tx_hash, "Tx escalated"); } tokio::time::sleep(app.config.service.escalation_interval).await; From 34848e387307074ab6a87ddd88cf08b8565cc51b Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 11:02:34 +0100 Subject: [PATCH 068/135] fmt --- src/tasks/handle_reorgs.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/tasks/handle_reorgs.rs b/src/tasks/handle_reorgs.rs index dd91dab..43c5f56 100644 --- a/src/tasks/handle_reorgs.rs +++ b/src/tasks/handle_reorgs.rs @@ -14,10 +14,7 @@ pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { let reorged_txs = app.db.handle_hard_reorgs().await?; for tx in reorged_txs { - tracing::info!( - id = tx, - "Tx hard reorged" - ); + tracing::info!(id = tx, "Tx hard reorged"); } tokio::time::sleep(Duration::from_secs( @@ -34,10 +31,7 @@ pub async fn handle_soft_reorgs(app: Arc) -> eyre::Result<()> { let txs = app.db.handle_soft_reorgs().await?; for tx in txs { - tracing::info!( - id = tx, - "Tx soft reorged" - ); + tracing::info!(id = tx, "Tx soft reorged"); } tokio::time::sleep(Duration::from_secs( From 7ce62ba672597c4bfe2ff0994f2a8f810369bbb8 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 11:13:30 +0100 Subject: [PATCH 069/135] Metrics for gas --- src/tasks/index.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/tasks/index.rs b/src/tasks/index.rs index fe430b2..0483938 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -17,6 +17,8 @@ const BLOCK_FEE_HISTORY_SIZE: usize = 10; const FEE_PERCENTILES: [f64; 5] = [5.0, 25.0, 50.0, 75.0, 95.0]; const TIME_BETWEEN_FEE_ESTIMATION_SECONDS: u64 = 30; +const GAS_PRICE_FOR_METRICS_FACTOR: f64 = 1e-9; + pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { loop { let ws_rpc = app.ws_provider(chain_id).await?; @@ -107,6 +109,32 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { ) .await?; + let labels = [("chain_id", chain_id.to_string())]; + metrics::gauge!( + "gas_price", + gas_price.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR, + &labels + ); + metrics::gauge!( + "base_fee_per_gas", + fee_estimates.base_fee_per_gas.as_u64() as f64 + * GAS_PRICE_FOR_METRICS_FACTOR, + &labels + ); + + for (i, percentile) in FEE_PERCENTILES.iter().enumerate() { + let percentile_fee = fee_estimates.percentile_fees[i]; + + metrics::gauge!( + "percentile_fee", + percentile_fee.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR, + &[ + ("chain_id", chain_id.to_string()), + ("percentile", percentile.to_string()), + ] + ); + } + tokio::time::sleep(Duration::from_secs( TIME_BETWEEN_FEE_ESTIMATION_SECONDS, )) From 997ecff50c7bcfbffaa69ec02da2e57bcf86b9e4 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 11:18:06 +0100 Subject: [PATCH 070/135] Better logging --- src/tasks/broadcast.rs | 6 +++--- src/tasks/escalate.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index c99e9cb..9418ded 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -76,7 +76,7 @@ async fn broadcast_relayer_txs( } for tx in txs { - tracing::info!(tx.id, "Sending tx"); + tracing::info!(id = tx.id, "Sending tx"); let middleware = app .signer_middleware(tx.chain_id, tx.key_id.clone()) @@ -134,8 +134,6 @@ async fn broadcast_relayer_txs( let tx_hash = pending_tx.tx_hash(); - tracing::info!(?tx.id, ?tx_hash, "Tx sent successfully"); - app.db .insert_tx_broadcast( &tx.id, @@ -144,6 +142,8 @@ async fn broadcast_relayer_txs( max_priority_fee_per_gas, ) .await?; + + tracing::info!(id = tx.id, hash = ?tx_hash, "Tx broadcast"); } Ok(()) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 59797eb..76d5d3c 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -17,7 +17,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { .await?; for tx in txs_for_escalation { - tracing::info!(tx.id, tx.escalation_count, "Escalating tx"); + tracing::info!(id = tx.id, tx.escalation_count, "Escalating tx"); if !should_send_transaction(&app, &tx.relayer_id).await? { tracing::warn!(id = tx.id, "Skipping transaction broadcast"); @@ -90,7 +90,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { ) .await?; - tracing::info!(?tx.id, ?tx_hash, "Tx escalated"); + tracing::info!(id = ?tx.id, hash = ?tx_hash, "Tx escalated"); } tokio::time::sleep(app.config.service.escalation_interval).await; From 1b9b4077d01944f6cf022b18248b363101ffbd9b Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 11:18:56 +0100 Subject: [PATCH 071/135] Add tx created log --- src/server/routes/transaction.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 0ea5513..a1dbca7 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -95,6 +95,8 @@ pub async fn send_tx( ) .await?; + tracing::info!(id = tx_id, "Tx created"); + Ok(Json(SendTxResponse { tx_id })) } From b0d0a0b13a3c7042bd3f8695af7f0ab40e1572e9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 12:03:37 +0100 Subject: [PATCH 072/135] Add relayers endpoint --- src/db.rs | 38 ++++++++++++++++++++++++++++++++---- src/server.rs | 5 +++-- src/server/routes/relayer.rs | 10 ++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/db.rs b/src/db.rs index 9a68140..dc86a15 100644 --- a/src/db.rs +++ b/src/db.rs @@ -120,16 +120,46 @@ impl Database { Ok(()) } + pub async fn get_relayers(&self) -> eyre::Result> { + Ok(sqlx::query_as( + r#" + SELECT + id, + name, + chain_id, + key_id, + address, + nonce, + current_nonce, + max_inflight_txs, + gas_limits + FROM relayers + "#, + ) + .fetch_all(&self.pool) + .await?) + } + pub async fn get_relayer(&self, id: &str) -> eyre::Result { Ok(sqlx::query_as( r#" - SELECT id, name, chain_id, key_id, address, nonce, current_nonce, max_inflight_txs, gas_limits + SELECT + id, + name, + chain_id, + key_id, + address, + nonce, + current_nonce, + max_inflight_txs, + gas_limits FROM relayers WHERE id = $1 - "#) - .bind(id) - .fetch_one(&self.pool).await? + "#, ) + .bind(id) + .fetch_one(&self.pool) + .await?) } pub async fn create_transaction( diff --git a/src/server.rs b/src/server.rs index fb9a3dd..247a604 100644 --- a/src/server.rs +++ b/src/server.rs @@ -9,8 +9,8 @@ use thiserror::Error; use tower_http::validate_request::ValidateRequestHeaderLayer; use self::routes::relayer::{ - create_relayer, create_relayer_api_key, get_relayer, relayer_rpc, - update_relayer, + create_relayer, create_relayer_api_key, get_relayer, get_relayers, + relayer_rpc, update_relayer, }; use self::routes::transaction::{get_tx, get_txs, send_tx}; use crate::app::App; @@ -77,6 +77,7 @@ pub async fn spawn_server( let mut admin_routes = Router::new() .route("/relayer", post(create_relayer)) + .route("/relayers", get(get_relayers)) .route( "/relayer/:relayer_id", post(update_relayer).get(get_relayer), diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 873facd..3573f8f 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -89,6 +89,16 @@ pub async fn update_relayer( Ok(()) } +#[tracing::instrument(skip(app))] +pub async fn get_relayers( + State(app): State>, + Path(relayer_id): Path, +) -> Result>, ApiError> { + let relayer_info = app.db.get_relayers().await?; + + Ok(Json(relayer_info)) +} + #[tracing::instrument(skip(app))] pub async fn get_relayer( State(app): State>, From 13f18e20f22455371471a094d1d1aaf465388ccc Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 12:18:44 +0100 Subject: [PATCH 073/135] Fix --- src/server/routes/relayer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 3573f8f..306c6c4 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -92,7 +92,6 @@ pub async fn update_relayer( #[tracing::instrument(skip(app))] pub async fn get_relayers( State(app): State>, - Path(relayer_id): Path, ) -> Result>, ApiError> { let relayer_info = app.db.get_relayers().await?; From b4d2f04145965dd03060a7958faf6ae0b5f5e643 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 12:50:31 +0100 Subject: [PATCH 074/135] Fix race condition --- src/db.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/db.rs b/src/db.rs index dc86a15..cf9adad 100644 --- a/src/db.rs +++ b/src/db.rs @@ -173,16 +173,30 @@ impl Database { relayer_id: &str, ) -> eyre::Result<()> { let mut tx = self.pool.begin().await?; + let mut value_bytes = [0u8; 32]; value.to_big_endian(&mut value_bytes); let mut gas_limit_bytes = [0u8; 32]; gas_limit.to_big_endian(&mut gas_limit_bytes); + let (nonce,): (i64,) = sqlx::query_as( + r#" + UPDATE relayers + SET nonce = nonce + 1, + updated_at = now() + WHERE id = $1 + RETURNING nonce - 1 + "#, + ) + .bind(relayer_id) + .fetch_one(tx.as_mut()) + .await?; + sqlx::query( r#" INSERT INTO transactions (id, tx_to, data, value, gas_limit, priority, relayer_id, nonce) - VALUES ($1, $2, $3, $4, $5, $6, $7, (SELECT nonce FROM relayers WHERE id = $7)) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) "#, ) .bind(tx_id) @@ -192,18 +206,7 @@ impl Database { .bind(gas_limit_bytes) .bind(priority) .bind(relayer_id) - .execute(tx.as_mut()) - .await?; - - sqlx::query( - r#" - UPDATE relayers - SET nonce = nonce + 1, - updated_at = now() - WHERE id = $1 - "#, - ) - .bind(relayer_id) + .bind(nonce) .execute(tx.as_mut()) .await?; From 03176ec430307f829d7894c5c477ce48b98e9f56 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 13:11:00 +0100 Subject: [PATCH 075/135] Add relayer reset endpoint --- src/db.rs | 33 +++++++++++++++++++++++++++++++++ src/server.rs | 3 ++- src/server/routes/relayer.rs | 13 +++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/db.rs b/src/db.rs index cf9adad..df84750 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1023,6 +1023,39 @@ impl Database { block_txs: block_txs as u64, }) } + + pub async fn purge_unsent_txs(&self, relayer_id: &str) -> eyre::Result<()> { + let unsent_txs = self.get_unsent_txs().await?; + + let unsent_tx_ids: Vec<_> = unsent_txs + .into_iter() + .filter(|tx| tx.relayer_id == relayer_id) + .map(|tx| tx.id) + .collect(); + + sqlx::query( + r#" + DELETE FROM transactions + WHERE id = ANY($1::TEXT[]) + "#, + ) + .bind(&unsent_tx_ids) + .execute(&self.pool) + .await?; + + sqlx::query( + r#" + UPDATE relayers + SET nonce = current_nonce + WHERE id = $1 + "#, + ) + .bind(relayer_id) + .execute(&self.pool) + .await?; + + Ok(()) + } } #[cfg(test)] diff --git a/src/server.rs b/src/server.rs index 247a604..11803d1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -10,7 +10,7 @@ use tower_http::validate_request::ValidateRequestHeaderLayer; use self::routes::relayer::{ create_relayer, create_relayer_api_key, get_relayer, get_relayers, - relayer_rpc, update_relayer, + purge_unsent_txs, relayer_rpc, update_relayer, }; use self::routes::transaction::{get_tx, get_txs, send_tx}; use crate::app::App; @@ -77,6 +77,7 @@ pub async fn spawn_server( let mut admin_routes = Router::new() .route("/relayer", post(create_relayer)) + .route("/relayer/:relayer_id/reset", post(purge_unsent_txs)) .route("/relayers", get(get_relayers)) .route( "/relayer/:relayer_id", diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 306c6c4..d262cdc 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -108,6 +108,19 @@ pub async fn get_relayer( Ok(Json(relayer_info)) } +/// Resets the relayer +/// deletes all unsent txs +/// and resets nonce to the current confirmed nonce +#[tracing::instrument(skip(app))] +pub async fn purge_unsent_txs( + State(app): State>, + Path(relayer_id): Path, +) -> Result<(), ApiError> { + app.db.purge_unsent_txs(&relayer_id).await?; + + Ok(()) +} + #[tracing::instrument(skip(app))] pub async fn relayer_rpc( State(app): State>, From 6de73df5180f3f00a3486f0a8d82044265b27809 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 13:26:53 +0100 Subject: [PATCH 076/135] Fix --- src/db.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/db.rs b/src/db.rs index df84750..f4262f3 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1025,32 +1025,28 @@ impl Database { } pub async fn purge_unsent_txs(&self, relayer_id: &str) -> eyre::Result<()> { - let unsent_txs = self.get_unsent_txs().await?; - - let unsent_tx_ids: Vec<_> = unsent_txs - .into_iter() - .filter(|tx| tx.relayer_id == relayer_id) - .map(|tx| tx.id) - .collect(); - - sqlx::query( + let (nonce,): (i64,) = sqlx::query_as( r#" - DELETE FROM transactions - WHERE id = ANY($1::TEXT[]) + UPDATE relayers + SET nonce = current_nonce + WHERE id = $1 + RETURNING nonce "#, ) - .bind(&unsent_tx_ids) - .execute(&self.pool) + .bind(relayer_id) + .fetch_one(&self.pool) .await?; sqlx::query( r#" - UPDATE relayers - SET nonce = current_nonce - WHERE id = $1 + DELETE FROM transactions + WHERE nonce > $1 + AND id NOT IN ( + SELECT tx_id FROM sent_transactions + ) "#, ) - .bind(relayer_id) + .bind(nonce) .execute(&self.pool) .await?; From e0e43509d8141aabd845b521fc57d73c6bb56dad Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 13:38:41 +0100 Subject: [PATCH 077/135] Fix --- src/db.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/db.rs b/src/db.rs index f4262f3..768a60d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1025,12 +1025,11 @@ impl Database { } pub async fn purge_unsent_txs(&self, relayer_id: &str) -> eyre::Result<()> { - let (nonce,): (i64,) = sqlx::query_as( + sqlx::query( r#" UPDATE relayers SET nonce = current_nonce WHERE id = $1 - RETURNING nonce "#, ) .bind(relayer_id) @@ -1040,13 +1039,13 @@ impl Database { sqlx::query( r#" DELETE FROM transactions - WHERE nonce > $1 + WHERE t.relayer_id = $1 AND id NOT IN ( SELECT tx_id FROM sent_transactions ) "#, ) - .bind(nonce) + .bind(relayer_id) .execute(&self.pool) .await?; From 3812432f50db6b12c43295eb88ce8a4c50bcf4ca Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 14:18:16 +0100 Subject: [PATCH 078/135] Fix --- src/db.rs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/db.rs b/src/db.rs index 768a60d..9676582 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1033,13 +1033,13 @@ impl Database { "#, ) .bind(relayer_id) - .fetch_one(&self.pool) + .execute(&self.pool) .await?; sqlx::query( r#" DELETE FROM transactions - WHERE t.relayer_id = $1 + WHERE relayer_id = $1 AND id NOT IN ( SELECT tx_id FROM sent_transactions ) @@ -1096,6 +1096,32 @@ mod tests { Ok(()) } + #[tokio::test] + async fn reset_relayer() -> eyre::Result<()> { + let (db, _db_container) = setup_db().await?; + + let chain_id = 123; + let network_name = "network_name"; + let http_rpc = "http_rpc"; + let ws_rpc = "ws_rpc"; + + db.create_network(chain_id, network_name, http_rpc, ws_rpc) + .await?; + + let relayer_id = uuid(); + let relayer_id = relayer_id.as_str(); + let relayer_name = "relayer_name"; + let key_id = "key_id"; + let address = Address::from_low_u64_be(1); + + db.create_relayer(relayer_id, relayer_name, chain_id, key_id, address) + .await?; + + db.purge_unsent_txs(relayer_id).await?; + + Ok(()) + } + #[tokio::test] async fn save_and_prune_blocks() -> eyre::Result<()> { let (db, _db_container) = setup_db().await?; From 27d0325b3937773cef80a8362c0da13ae7c1ee7d Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 16:39:46 +0100 Subject: [PATCH 079/135] Remove hard coded gas limit --- src/broadcast_utils.rs | 26 ++++---------------------- src/tasks/broadcast.rs | 4 ++-- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index c4159f0..ff47905 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -8,45 +8,27 @@ pub mod gas_estimation; const BASE_FEE_PER_GAS_SURGE_FACTOR: u64 = 2; -// TODO: Adjust -const MIN_PRIORITY_FEE: U256 = U256([10, 0, 0, 0]); -const MAX_GAS_PRICE: U256 = U256([100_000_000_000, 0, 0, 0]); /// Returns a tuple of max and max priority fee per gas pub fn calculate_gas_fees_from_estimates( estimates: &FeesEstimate, tx_priority_index: usize, max_base_fee_per_gas: U256, -) -> eyre::Result<(U256, U256)> { +) -> (U256, U256) { let max_priority_fee_per_gas = estimates.percentile_fees[tx_priority_index]; - let max_priority_fee_per_gas = - std::cmp::max(max_priority_fee_per_gas, MIN_PRIORITY_FEE); - let max_fee_per_gas = max_base_fee_per_gas + max_priority_fee_per_gas; - let max_fee_per_gas = std::cmp::min(max_fee_per_gas, MAX_GAS_PRICE); - Ok((max_fee_per_gas, max_priority_fee_per_gas)) + (max_fee_per_gas, max_priority_fee_per_gas) } /// Calculates the max base fee per gas -/// Returns an error if the base fee per gas is too high -/// /// i.e. the base fee from estimates surged by a factor -pub fn calculate_max_base_fee_per_gas( - estimates: &FeesEstimate, -) -> eyre::Result { +pub fn calculate_max_base_fee_per_gas(estimates: &FeesEstimate) -> U256 { let base_fee_per_gas = estimates.base_fee_per_gas; - if base_fee_per_gas > MAX_GAS_PRICE { - tracing::warn!("Base fee per gas is too high, retrying later"); - eyre::bail!("Base fee per gas is too high"); - } - // Surge the base fee per gas - let max_base_fee_per_gas = base_fee_per_gas * BASE_FEE_PER_GAS_SURGE_FACTOR; - - Ok(max_base_fee_per_gas) + base_fee_per_gas * BASE_FEE_PER_GAS_SURGE_FACTOR } pub fn escalate_priority_fee( diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 9418ded..3fcb698 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -89,14 +89,14 @@ async fn broadcast_relayer_txs( .context("Missing block fees")?; let max_base_fee_per_gas = - calculate_max_base_fee_per_gas(&fees.fee_estimates)?; + calculate_max_base_fee_per_gas(&fees.fee_estimates); let (max_fee_per_gas, max_priority_fee_per_gas) = calculate_gas_fees_from_estimates( &fees.fee_estimates, tx.priority.to_percentile_index(), max_base_fee_per_gas, - )?; + ); let eip1559_tx = Eip1559TransactionRequest { from: None, From fc36d12b23f14264a2352bb1c3c41f3dcd2f6420 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 11 Dec 2023 16:40:59 +0100 Subject: [PATCH 080/135] Remove surge logic --- src/broadcast_utils.rs | 12 ------------ src/tasks/broadcast.rs | 6 ++---- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index ff47905..a0182d4 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -6,9 +6,6 @@ use crate::app::App; pub mod gas_estimation; -const BASE_FEE_PER_GAS_SURGE_FACTOR: u64 = 2; - - /// Returns a tuple of max and max priority fee per gas pub fn calculate_gas_fees_from_estimates( estimates: &FeesEstimate, @@ -22,15 +19,6 @@ pub fn calculate_gas_fees_from_estimates( (max_fee_per_gas, max_priority_fee_per_gas) } -/// Calculates the max base fee per gas -/// i.e. the base fee from estimates surged by a factor -pub fn calculate_max_base_fee_per_gas(estimates: &FeesEstimate) -> U256 { - let base_fee_per_gas = estimates.base_fee_per_gas; - - // Surge the base fee per gas - base_fee_per_gas * BASE_FEE_PER_GAS_SURGE_FACTOR -} - pub fn escalate_priority_fee( max_base_fee_per_gas: U256, max_network_fee_per_gas: U256, diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 3fcb698..d541928 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -13,8 +13,7 @@ use itertools::Itertools; use crate::app::App; use crate::broadcast_utils::{ - calculate_gas_fees_from_estimates, calculate_max_base_fee_per_gas, - should_send_transaction, + calculate_gas_fees_from_estimates, should_send_transaction, }; use crate::db::UnsentTx; @@ -88,8 +87,7 @@ async fn broadcast_relayer_txs( .await? .context("Missing block fees")?; - let max_base_fee_per_gas = - calculate_max_base_fee_per_gas(&fees.fee_estimates); + let max_base_fee_per_gas = fees.fee_estimates.base_fee_per_gas; let (max_fee_per_gas, max_priority_fee_per_gas) = calculate_gas_fees_from_estimates( From 4dd4b518e20a49b418d28bec14d85324889bca82 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 13:37:56 -0500 Subject: [PATCH 081/135] added .gitignore, increased allowable db setup time --- .gitignore | 1 + src/db.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f97022 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/src/db.rs b/src/db.rs index 9676582..ac0da7f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1069,9 +1069,17 @@ mod tests { let url = format!("postgres://postgres:postgres@{db_socket_addr}/database"); - let db = Database::new(&DatabaseConfig::connection_string(url)).await?; + for _ in 0..5 { + match Database::new(&DatabaseConfig::connection_string(&url)).await + { + Ok(db) => return Ok((db, db_container)), + Err(_) => { + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } - Ok((db, db_container)) + Err(eyre::eyre!("Failed to connect to the database")) } async fn full_update( From 2f2766e15863c5628f57266c6203e77d2d93dd98 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 13:39:21 -0500 Subject: [PATCH 082/135] fix: typo --- src/tasks/finalize.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tasks/finalize.rs b/src/tasks/finalize.rs index 4695153..9ee3d87 100644 --- a/src/tasks/finalize.rs +++ b/src/tasks/finalize.rs @@ -7,15 +7,15 @@ const TIME_BETWEEN_FINALIZATIONS_SECONDS: i64 = 60; pub async fn finalize_txs(app: Arc) -> eyre::Result<()> { loop { - let finalization_timestmap = + let finalization_timestamp = chrono::Utc::now() - chrono::Duration::seconds(60 * 60); tracing::info!( "Finalizing txs mined before {}", - finalization_timestmap + finalization_timestamp ); - app.db.finalize_txs(finalization_timestmap).await?; + app.db.finalize_txs(finalization_timestamp).await?; tokio::time::sleep(Duration::from_secs( TIME_BETWEEN_FINALIZATIONS_SECONDS as u64, From 9139fafce18362fbbd96745ea8bf0ca5453b1cd9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 12 Dec 2023 20:24:37 +0100 Subject: [PATCH 083/135] Make transfers in tests more parallel --- tests/send_many_txs.rs | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index ed6cfb3..5476a3e 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -1,5 +1,7 @@ mod common; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use tx_sitter::server::routes::relayer::CreateApiKeyResponse; use crate::common::prelude::*; @@ -59,18 +61,29 @@ async fn send_many_txs() -> eyre::Result<()> { let value: U256 = parse_units("10", "ether")?.into(); let num_transfers = 10; + let mut tasks = FuturesUnordered::new(); for _ in 0..num_transfers { - client - .send_tx( - &api_key, - &SendTxRequest { - to: ARBITRARY_ADDRESS, - value, - gas_limit: U256::from(21_000), - ..Default::default() - }, - ) - .await?; + let client = &client; + tasks.push(async { + client + .send_tx( + &api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) + .await?; + + Ok(()) + }); + } + + while let Some(result) = tasks.next().await { + let result: eyre::Result<()> = result; + result?; } let expected_balance = value * num_transfers; From 3d369f329191e912629abb037785dfe166e180af Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 20:56:22 -0500 Subject: [PATCH 084/135] updated logic to back fill blocks that have been missed --- .gitignore | 1 + src/db.rs | 20 +++++++++ src/tasks/index.rs | 106 +++++++++++++++++++++++++++++---------------- 3 files changed, 89 insertions(+), 38 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f97022 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/src/db.rs b/src/db.rs index 9676582..b724e24 100644 --- a/src/db.rs +++ b/src/db.rs @@ -303,6 +303,26 @@ impl Database { Ok(block_number.map(|(n,)| n as u64)) } + pub async fn get_latest_block_number( + &self, + chain_id: u64, + ) -> eyre::Result { + let (block_number,): (i64,) = sqlx::query_as( + r#" + SELECT block_number + FROM blocks + WHERE chain_id = $1 + ORDER BY block_number DESC + LIMIT 1 + "#, + ) + .bind(chain_id as i64) + .fetch_one(&self.pool) + .await?; + + Ok(block_number as u64) + } + pub async fn get_latest_block_fees_by_chain_id( &self, chain_id: u64, diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 0483938..554bdba 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -3,7 +3,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use ethers::providers::{Http, Middleware, Provider}; -use ethers::types::BlockNumber; +use ethers::types::{Block, BlockNumber, H256}; use eyre::{Context, ContextCompat}; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -26,53 +26,83 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { let mut blocks_stream = ws_rpc.subscribe_blocks().await?; + let next_block_number = + app.db.get_latest_block_number(chain_id).await? + 1; + + if let Some(latest_block) = blocks_stream.next().await { + let latest_block_number = latest_block + .number + .context("Missing block number")? + .as_u64(); + + if latest_block_number > next_block_number { + for block_number in next_block_number..=latest_block_number { + let block = rpc + .get_block::(block_number.into()) + .await? + .context(format!( + "Could not get block at height {}", + block_number + ))?; + + index_block(app.clone(), chain_id, &rpc, block).await?; + } + } + } + while let Some(block) = blocks_stream.next().await { - let block_number = - block.number.context("Missing block number")?.as_u64(); + index_block(app.clone(), chain_id, &rpc, block).await?; + } + } +} - tracing::info!(block_number, "Indexing block"); +pub async fn index_block( + app: Arc, + chain_id: u64, + rpc: &Provider, + block: Block, +) -> eyre::Result<()> { + let block_number = block.number.context("Missing block number")?.as_u64(); - let block_timestamp_seconds = block.timestamp.as_u64(); - let block_timestamp = DateTime::::from_timestamp( - block_timestamp_seconds as i64, - 0, - ) - .context("Invalid timestamp")?; + tracing::info!(block_number, "Indexing block"); - let block = rpc - .get_block(block_number) - .await? - .context("Missing block")?; + let block_timestamp_seconds = block.timestamp.as_u64(); + let block_timestamp = + DateTime::::from_timestamp(block_timestamp_seconds as i64, 0) + .context("Invalid timestamp")?; - app.db - .save_block( - block.number.unwrap().as_u64(), - chain_id, - block_timestamp, - &block.transactions, - ) - .await?; + let block = rpc + .get_block(block_number) + .await? + .context("Missing block")?; + + app.db + .save_block( + block.number.unwrap().as_u64(), + chain_id, + block_timestamp, + &block.transactions, + ) + .await?; - let mined_txs = app.db.mine_txs(chain_id).await?; + let mined_txs = app.db.mine_txs(chain_id).await?; - let metric_labels = [("chain_id", chain_id.to_string())]; - for tx in mined_txs { - tracing::info!( - id = tx.0, - hash = ?tx.1, - "Tx mined" - ); + let metric_labels: [(&str, String); 1] = + [("chain_id", chain_id.to_string())]; + for tx in mined_txs { + tracing::info!( + id = tx.0, + hash = ?tx.1, + "Tx mined" + ); - metrics::increment_counter!("tx_mined", &metric_labels); - } + metrics::increment_counter!("tx_mined", &metric_labels); + } - let relayer_addresses = - app.db.get_relayer_addresses(chain_id).await?; + let relayer_addresses = app.db.get_relayer_addresses(chain_id).await?; - update_relayer_nonces(relayer_addresses, &app, &rpc, chain_id) - .await?; - } - } + update_relayer_nonces(relayer_addresses, &app, &rpc, chain_id).await?; + Ok(()) } pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { From d8eec4908a41f8331799537aba752a582bf04a23 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 21:18:32 -0500 Subject: [PATCH 085/135] adding comments --- src/tasks/index.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 554bdba..9f88bb0 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -24,11 +24,14 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { let ws_rpc = app.ws_provider(chain_id).await?; let rpc = app.http_provider(chain_id).await?; + // Subscribe to new block with the WS client which uses an unbounded receiver, buffering the stream let mut blocks_stream = ws_rpc.subscribe_blocks().await?; + // Get the latest block from the db let next_block_number = app.db.get_latest_block_number(chain_id).await? + 1; + // Get the first block from the stream and backfill any missing blocks if let Some(latest_block) = blocks_stream.next().await { let latest_block_number = latest_block .number @@ -36,7 +39,8 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { .as_u64(); if latest_block_number > next_block_number { - for block_number in next_block_number..=latest_block_number { + // Backfill blocks between the last synced block and the chain head + for block_number in next_block_number..latest_block_number { let block = rpc .get_block::(block_number.into()) .await? @@ -47,9 +51,13 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { index_block(app.clone(), chain_id, &rpc, block).await?; } + + // Index the latest block after backfilling + index_block(app.clone(), chain_id, &rpc, latest_block).await?; } } + // Index incoming blocks from the stream while let Some(block) = blocks_stream.next().await { index_block(app.clone(), chain_id, &rpc, block).await?; } From 80656c1242ab763eda3a32abb70712f49eef6099 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 21:28:53 -0500 Subject: [PATCH 086/135] updated tests --- src/db.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/db.rs b/src/db.rs index b724e24..662ffa2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1089,9 +1089,17 @@ mod tests { let url = format!("postgres://postgres:postgres@{db_socket_addr}/database"); - let db = Database::new(&DatabaseConfig::connection_string(url)).await?; + for _ in 0..5 { + match Database::new(&DatabaseConfig::connection_string(&url)).await + { + Ok(db) => return Ok((db, db_container)), + Err(_) => { + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } - Ok((db, db_container)) + Err(eyre::eyre!("Failed to connect to the database")) } async fn full_update( @@ -1406,7 +1414,7 @@ mod tests { async fn blocks() -> eyre::Result<()> { let (db, _db_container) = setup_db().await?; - let block_number = 1; + let block_numbers = vec![0, 1]; let chain_id = 1; let timestamp = ymd_hms(2023, 11, 23, 12, 32, 2); let txs = &[ @@ -1415,7 +1423,10 @@ mod tests { H256::from_low_u64_be(3), ]; - db.save_block(block_number, chain_id, timestamp, txs) + db.save_block(block_numbers[0], chain_id, timestamp, txs) + .await?; + + db.save_block(block_numbers[1], chain_id, timestamp, txs) .await?; let fee_estimates = FeesEstimate { @@ -1425,13 +1436,19 @@ mod tests { let gas_price = U256::from(1_000_000_007); - db.save_block_fees(block_number, chain_id, &fee_estimates, gas_price) - .await?; + db.save_block_fees( + block_numbers[1], + chain_id, + &fee_estimates, + gas_price, + ) + .await?; + let latest_block_number = db.get_latest_block_number(chain_id).await?; let block_fees = db.get_latest_block_fees_by_chain_id(chain_id).await?; - let block_fees = block_fees.context("Missing fees")?; + assert_eq!(latest_block_number, block_numbers[1]); assert_eq!( block_fees.fee_estimates.base_fee_per_gas, fee_estimates.base_fee_per_gas From b73098c0b679f763015c4731765bf1734f46ca21 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 22:07:04 -0500 Subject: [PATCH 087/135] updated get_latest_block_number --- src/db.rs | 13 +++++---- src/tasks/index.rs | 71 ++++++++++++++++++++++++++++------------------ 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/src/db.rs b/src/db.rs index 662ffa2..1f1e07e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -306,8 +306,8 @@ impl Database { pub async fn get_latest_block_number( &self, chain_id: u64, - ) -> eyre::Result { - let (block_number,): (i64,) = sqlx::query_as( + ) -> eyre::Result> { + let block_number: Option<(i64,)> = sqlx::query_as( r#" SELECT block_number FROM blocks @@ -317,10 +317,10 @@ impl Database { "#, ) .bind(chain_id as i64) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; - Ok(block_number as u64) + Ok(block_number.map(|(n,)| n as u64)) } pub async fn get_latest_block_fees_by_chain_id( @@ -1444,7 +1444,10 @@ mod tests { ) .await?; - let latest_block_number = db.get_latest_block_number(chain_id).await?; + let latest_block_number = + db.get_latest_block_number(chain_id) + .await? + .context("Could not get latest block number")?; let block_fees = db.get_latest_block_fees_by_chain_id(chain_id).await?; let block_fees = block_fees.context("Missing fees")?; diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 9f88bb0..3882d06 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use std::time::Duration; use chrono::{DateTime, Utc}; -use ethers::providers::{Http, Middleware, Provider}; +use ethers::providers::{Http, Middleware, Provider, SubscriptionStream, Ws}; use ethers::types::{Block, BlockNumber, H256}; use eyre::{Context, ContextCompat}; use futures::stream::FuturesUnordered; @@ -27,34 +27,12 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { // Subscribe to new block with the WS client which uses an unbounded receiver, buffering the stream let mut blocks_stream = ws_rpc.subscribe_blocks().await?; - // Get the latest block from the db - let next_block_number = - app.db.get_latest_block_number(chain_id).await? + 1; + // Get the first block from the stream, backfilling any missing blocks between the latest block in the db - // Get the first block from the stream and backfill any missing blocks + //TODO: note in the comments that this fills the block if let Some(latest_block) = blocks_stream.next().await { - let latest_block_number = latest_block - .number - .context("Missing block number")? - .as_u64(); - - if latest_block_number > next_block_number { - // Backfill blocks between the last synced block and the chain head - for block_number in next_block_number..latest_block_number { - let block = rpc - .get_block::(block_number.into()) - .await? - .context(format!( - "Could not get block at height {}", - block_number - ))?; - - index_block(app.clone(), chain_id, &rpc, block).await?; - } - - // Index the latest block after backfilling - index_block(app.clone(), chain_id, &rpc, latest_block).await?; - } + backfill_to_block(app.clone(), chain_id, &rpc, latest_block) + .await?; } // Index incoming blocks from the stream @@ -113,6 +91,45 @@ pub async fn index_block( Ok(()) } +pub async fn backfill_to_block( + app: Arc, + chain_id: u64, + rpc: &Provider, + latest_block: Block, +) -> eyre::Result<()> { + // Get the latest block from the db + if let Some(latest_db_block_number) = + app.db.get_latest_block_number(chain_id).await? + { + let next_block_number: u64 = latest_db_block_number + 1; + + // Get the first block from the stream and backfill any missing blocks + let latest_block_number = latest_block + .number + .context("Missing block number")? + .as_u64(); + + if latest_block_number > next_block_number { + // Backfill blocks between the last synced block and the chain head, non inclusive + for block_number in next_block_number..latest_block_number { + let block = rpc + .get_block::(block_number.into()) + .await? + .context(format!( + "Could not get block at height {}", + block_number + ))?; + + index_block(app.clone(), chain_id, &rpc, block).await?; + } + } + + // Index the latest block after backfilling + index_block(app.clone(), chain_id, &rpc, latest_block).await?; + }; + Ok(()) +} + pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { let rpc = app.http_provider(chain_id).await?; From 06116d81aa6c73e68b714df207b8d4d43a665e63 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 22:11:26 -0500 Subject: [PATCH 088/135] updated comments --- src/tasks/index.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 3882d06..b54676f 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -27,9 +27,7 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { // Subscribe to new block with the WS client which uses an unbounded receiver, buffering the stream let mut blocks_stream = ws_rpc.subscribe_blocks().await?; - // Get the first block from the stream, backfilling any missing blocks between the latest block in the db - - //TODO: note in the comments that this fills the block + // Get the first block from the stream, backfilling any missing blocks from the latest block in the db to the chain head if let Some(latest_block) = blocks_stream.next().await { backfill_to_block(app.clone(), chain_id, &rpc, latest_block) .await?; From 299a8568623bb08b2939cdb5dfb389a3293a1c72 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Tue, 12 Dec 2023 22:18:53 -0500 Subject: [PATCH 089/135] cargo clippy --- src/db.rs | 2 +- src/tasks/index.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/db.rs b/src/db.rs index 1f1e07e..965117a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1414,7 +1414,7 @@ mod tests { async fn blocks() -> eyre::Result<()> { let (db, _db_container) = setup_db().await?; - let block_numbers = vec![0, 1]; + let block_numbers = [0, 1]; let chain_id = 1; let timestamp = ymd_hms(2023, 11, 23, 12, 32, 2); let txs = &[ diff --git a/src/tasks/index.rs b/src/tasks/index.rs index b54676f..eca129f 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use std::time::Duration; use chrono::{DateTime, Utc}; -use ethers::providers::{Http, Middleware, Provider, SubscriptionStream, Ws}; +use ethers::providers::{Http, Middleware, Provider}; use ethers::types::{Block, BlockNumber, H256}; use eyre::{Context, ContextCompat}; use futures::stream::FuturesUnordered; @@ -85,7 +85,7 @@ pub async fn index_block( let relayer_addresses = app.db.get_relayer_addresses(chain_id).await?; - update_relayer_nonces(relayer_addresses, &app, &rpc, chain_id).await?; + update_relayer_nonces(relayer_addresses, &app, rpc, chain_id).await?; Ok(()) } @@ -118,12 +118,12 @@ pub async fn backfill_to_block( block_number ))?; - index_block(app.clone(), chain_id, &rpc, block).await?; + index_block(app.clone(), chain_id, rpc, block).await?; } } // Index the latest block after backfilling - index_block(app.clone(), chain_id, &rpc, latest_block).await?; + index_block(app.clone(), chain_id, rpc, latest_block).await?; }; Ok(()) } From 41fd8af3bfaeba250026452ba28519b8d93dd728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 13 Dec 2023 16:17:23 +0100 Subject: [PATCH 090/135] Preconfigured start - for dockerization (#6) * Allow preconfigured networks & relayers * Only allow single network and relayer --- config.toml | 10 ++++ src/config.rs | 42 +++++++++++++- src/keys.rs | 125 ++--------------------------------------- src/keys/kms_keys.rs | 59 +++++++++++++++++++ src/keys/local_keys.rs | 69 +++++++++++++++++++++++ src/service.rs | 43 ++++++++++++++ tests/common/mod.rs | 52 +++++++++++------ tests/rpc_access.rs | 9 +-- tests/send_many_txs.rs | 41 ++------------ tests/send_tx.rs | 39 +------------ 10 files changed, 269 insertions(+), 220 deletions(-) create mode 100644 src/keys/kms_keys.rs create mode 100644 src/keys/local_keys.rs diff --git a/config.toml b/config.toml index f2f5ec2..cc2e14e 100644 --- a/config.toml +++ b/config.toml @@ -3,6 +3,16 @@ escalation_interval = "1m" datadog_enabled = false statsd_enabled = false +[predefined.network] +chain_id = 31337 +http_url = "http://127.0.0.1:8545" +ws_url = "ws://127.0.0.1:8545" + +[predefined.relayer] +id = "1b908a34-5dc1-4d2d-a146-5eb46e975830" +chain_id = 31337 +key_id = "d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa" + [server] host = "127.0.0.1:3000" disable_auth = false diff --git a/src/config.rs b/src/config.rs index 53ad624..7e1e534 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,6 +23,34 @@ pub struct TxSitterConfig { #[serde(default)] pub statsd_enabled: bool, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub predefined: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct Predefined { + pub network: PredefinedNetwork, + pub relayer: PredefinedRelayer, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct PredefinedNetwork { + pub chain_id: u64, + pub name: String, + pub http_rpc: String, + pub ws_rpc: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct PredefinedRelayer { + pub id: String, + pub name: String, + pub key_id: String, + pub chain_id: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -101,10 +129,16 @@ pub enum KeysConfig { #[serde(rename_all = "snake_case")] pub struct KmsKeysConfig {} -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct LocalKeysConfig {} +impl KeysConfig { + pub fn is_local(&self) -> bool { + matches!(self, Self::Local(_)) + } +} + #[cfg(test)] mod tests { use indoc::indoc; @@ -156,6 +190,7 @@ mod tests { escalation_interval: Duration::from_secs(60 * 60), datadog_enabled: false, statsd_enabled: false, + predefined: None, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), @@ -166,7 +201,7 @@ mod tests { "postgres://postgres:postgres@127.0.0.1:52804/database" .to_string(), ), - keys: KeysConfig::Local(LocalKeysConfig {}), + keys: KeysConfig::Local(LocalKeysConfig::default()), }; let toml = toml::to_string_pretty(&config).unwrap(); @@ -181,6 +216,7 @@ mod tests { escalation_interval: Duration::from_secs(60 * 60), datadog_enabled: false, statsd_enabled: false, + predefined: None, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), @@ -194,7 +230,7 @@ mod tests { password: "pass".to_string(), database: "db".to_string(), }), - keys: KeysConfig::Local(LocalKeysConfig {}), + keys: KeysConfig::Local(LocalKeysConfig::default()), }; let toml = toml::to_string_pretty(&config).unwrap(); diff --git a/src/keys.rs b/src/keys.rs index 01c4647..3dc171c 100644 --- a/src/keys.rs +++ b/src/keys.rs @@ -1,14 +1,10 @@ -use aws_config::BehaviorVersion; -use aws_sdk_kms::types::{KeySpec, KeyUsageType}; -use ethers::core::k256::ecdsa::SigningKey; -use ethers::signers::Wallet; -use eyre::{Context, ContextCompat}; -pub use universal_signer::UniversalSigner; - -use crate::aws::ethers_signer::AwsSigner; -use crate::config::{KmsKeysConfig, LocalKeysConfig}; +pub mod kms_keys; +pub mod local_keys; +pub mod universal_signer; -mod universal_signer; +pub use kms_keys::KmsKeys; +pub use local_keys::LocalKeys; +pub use universal_signer::UniversalSigner; #[async_trait::async_trait] pub trait KeysSource: Send + Sync + 'static { @@ -18,112 +14,3 @@ pub trait KeysSource: Send + Sync + 'static { /// Loads the key using the provided id async fn load_signer(&self, id: String) -> eyre::Result; } - -pub struct KmsKeys { - kms_client: aws_sdk_kms::Client, -} - -impl KmsKeys { - pub async fn new(_config: &KmsKeysConfig) -> eyre::Result { - let aws_config = - aws_config::load_defaults(BehaviorVersion::latest()).await; - - let kms_client = aws_sdk_kms::Client::new(&aws_config); - - Ok(Self { kms_client }) - } -} - -#[async_trait::async_trait] -impl KeysSource for KmsKeys { - async fn new_signer(&self) -> eyre::Result<(String, UniversalSigner)> { - let kms_key = self - .kms_client - .create_key() - .key_spec(KeySpec::EccSecgP256K1) - .key_usage(KeyUsageType::SignVerify) - .send() - .await - .context("AWS Error")?; - - let key_id = - kms_key.key_metadata.context("Missing key metadata")?.key_id; - - let signer = AwsSigner::new( - self.kms_client.clone(), - key_id.clone(), - 1, // TODO: get chain id from provider - ) - .await?; - - Ok((key_id, UniversalSigner::Aws(signer))) - } - - async fn load_signer(&self, id: String) -> eyre::Result { - let signer = AwsSigner::new( - self.kms_client.clone(), - id.clone(), - 1, // TODO: get chain id from provider - ) - .await?; - - Ok(UniversalSigner::Aws(signer)) - } -} - -pub struct LocalKeys { - rng: rand::rngs::OsRng, -} - -impl LocalKeys { - pub fn new(_config: &LocalKeysConfig) -> Self { - Self { - rng: rand::rngs::OsRng, - } - } -} - -#[async_trait::async_trait] -impl KeysSource for LocalKeys { - async fn new_signer(&self) -> eyre::Result<(String, UniversalSigner)> { - let signing_key = SigningKey::random(&mut self.rng.clone()); - - let key_id = signing_key.to_bytes().to_vec(); - let key_id = hex::encode(key_id); - - let signer = Wallet::from(signing_key); - - Ok((key_id, UniversalSigner::Local(signer))) - } - - async fn load_signer(&self, id: String) -> eyre::Result { - let key_id = hex::decode(id)?; - let signing_key = SigningKey::from_slice(key_id.as_slice())?; - - let signer = Wallet::from(signing_key); - - Ok(UniversalSigner::Local(signer)) - } -} - -#[cfg(test)] -mod tests { - use ethers::signers::Signer; - - use super::*; - - #[tokio::test] - async fn local_roundtrip() -> eyre::Result<()> { - let keys_source = LocalKeys::new(&LocalKeysConfig {}); - - let (id, signer) = keys_source.new_signer().await?; - - let address = signer.address(); - - let signer = keys_source.load_signer(id).await?; - - assert_eq!(address, signer.address()); - - Ok(()) - } -} diff --git a/src/keys/kms_keys.rs b/src/keys/kms_keys.rs new file mode 100644 index 0000000..baabb29 --- /dev/null +++ b/src/keys/kms_keys.rs @@ -0,0 +1,59 @@ +use aws_config::BehaviorVersion; +use aws_sdk_kms::types::{KeySpec, KeyUsageType}; +use eyre::{Context, ContextCompat}; + +use super::{KeysSource, UniversalSigner}; +use crate::aws::ethers_signer::AwsSigner; +use crate::config::KmsKeysConfig; + +pub struct KmsKeys { + kms_client: aws_sdk_kms::Client, +} + +impl KmsKeys { + pub async fn new(_config: &KmsKeysConfig) -> eyre::Result { + let aws_config = + aws_config::load_defaults(BehaviorVersion::latest()).await; + + let kms_client = aws_sdk_kms::Client::new(&aws_config); + + Ok(Self { kms_client }) + } +} + +#[async_trait::async_trait] +impl KeysSource for KmsKeys { + async fn new_signer(&self) -> eyre::Result<(String, UniversalSigner)> { + let kms_key = self + .kms_client + .create_key() + .key_spec(KeySpec::EccSecgP256K1) + .key_usage(KeyUsageType::SignVerify) + .send() + .await + .context("AWS Error")?; + + let key_id = + kms_key.key_metadata.context("Missing key metadata")?.key_id; + + let signer = AwsSigner::new( + self.kms_client.clone(), + key_id.clone(), + 1, // TODO: get chain id from provider + ) + .await?; + + Ok((key_id, UniversalSigner::Aws(signer))) + } + + async fn load_signer(&self, id: String) -> eyre::Result { + let signer = AwsSigner::new( + self.kms_client.clone(), + id.clone(), + 1, // TODO: get chain id from provider + ) + .await?; + + Ok(UniversalSigner::Aws(signer)) + } +} diff --git a/src/keys/local_keys.rs b/src/keys/local_keys.rs new file mode 100644 index 0000000..8b6e334 --- /dev/null +++ b/src/keys/local_keys.rs @@ -0,0 +1,69 @@ +use ethers::core::k256::ecdsa::SigningKey; +use ethers::signers::Wallet; + +use super::universal_signer::UniversalSigner; +use super::KeysSource; +use crate::config::LocalKeysConfig; + +pub struct LocalKeys { + rng: rand::rngs::OsRng, +} + +impl LocalKeys { + pub fn new(_config: &LocalKeysConfig) -> Self { + Self { + rng: rand::rngs::OsRng, + } + } +} + +#[async_trait::async_trait] +impl KeysSource for LocalKeys { + async fn new_signer(&self) -> eyre::Result<(String, UniversalSigner)> { + let signing_key = SigningKey::random(&mut self.rng.clone()); + + let key_id = signing_key.to_bytes().to_vec(); + let key_id = hex::encode(key_id); + + let signer = Wallet::from(signing_key); + + Ok((key_id, UniversalSigner::Local(signer))) + } + + async fn load_signer(&self, id: String) -> eyre::Result { + let signing_key = signing_key_from_hex(&id)?; + + let signer = Wallet::from(signing_key); + + Ok(UniversalSigner::Local(signer)) + } +} + +pub fn signing_key_from_hex(s: &str) -> eyre::Result { + let key_id = hex::decode(s)?; + let signing_key = SigningKey::from_slice(key_id.as_slice())?; + + Ok(signing_key) +} + +#[cfg(test)] +mod tests { + use ethers::signers::Signer; + + use super::*; + + #[tokio::test] + async fn local_roundtrip() -> eyre::Result<()> { + let keys_source = LocalKeys::new(&LocalKeysConfig::default()); + + let (id, signer) = keys_source.new_signer().await?; + + let address = signer.address(); + + let signer = keys_source.load_signer(id).await?; + + assert_eq!(address, signer.address()); + + Ok(()) + } +} diff --git a/src/service.rs b/src/service.rs index 599b0ac..a14598d 100644 --- a/src/service.rs +++ b/src/service.rs @@ -1,10 +1,12 @@ use std::net::SocketAddr; use std::sync::Arc; +use ethers::signers::{Signer, Wallet}; use tokio::task::JoinHandle; use crate::app::App; use crate::config::Config; +use crate::keys::local_keys::signing_key_from_hex; use crate::task_runner::TaskRunner; use crate::tasks; @@ -44,6 +46,8 @@ impl Service { Ok(()) }); + initialize_predefined_values(&app).await?; + Ok(Self { _app: app, local_addr, @@ -78,3 +82,42 @@ impl Service { Ok(()) } } + +async fn initialize_predefined_values( + app: &Arc, +) -> Result<(), eyre::Error> { + if app.config.service.predefined.is_some() && !app.config.keys.is_local() { + eyre::bail!("Predefined relayers are only supported with local keys"); + } + + let predefined = app.config.service.predefined.as_ref().unwrap(); + + app.db + .create_network( + predefined.network.chain_id, + &predefined.network.name, + &predefined.network.http_rpc, + &predefined.network.ws_rpc, + ) + .await?; + + let task_runner = TaskRunner::new(app.clone()); + Service::spawn_chain_tasks(&task_runner, predefined.network.chain_id)?; + + let secret_key = signing_key_from_hex(&predefined.relayer.key_id)?; + + let signer = Wallet::from(secret_key); + let address = signer.address(); + + app.db + .create_relayer( + &predefined.relayer.id, + &predefined.relayer.name, + predefined.relayer.chain_id, + &predefined.relayer.key_id, + address, + ) + .await?; + + Ok(()) +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c24a844..033ea32 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -18,10 +18,9 @@ use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; use tx_sitter::client::TxSitterClient; use tx_sitter::config::{ - Config, DatabaseConfig, KeysConfig, LocalKeysConfig, ServerConfig, - TxSitterConfig, + Config, DatabaseConfig, KeysConfig, LocalKeysConfig, Predefined, + PredefinedNetwork, PredefinedRelayer, ServerConfig, TxSitterConfig, }; -use tx_sitter::server::routes::network::NewNetworkInfo; use tx_sitter::service::Service; pub type AppMiddleware = SignerMiddleware>, LocalWallet>; @@ -49,12 +48,18 @@ pub const DEFAULT_ANVIL_PRIVATE_KEY: &[u8] = &hex_literal::hex!( "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" ); +pub const SECONDARY_ANVIL_PRIVATE_KEY: &[u8] = &hex_literal::hex!( + "59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" +); + pub const ARBITRARY_ADDRESS: Address = H160(hex_literal::hex!( "1Ed53d680B8890DAe2a63f673a85fFDE1FD5C7a2" )); pub const DEFAULT_ANVIL_CHAIN_ID: u64 = 31337; +pub const DEFAULT_RELAYER_ID: &str = "1b908a34-5dc1-4d2d-a146-5eb46e975830"; + pub struct DoubleAnvilHandle { pub double_anvil: Arc, ws_addr: String, @@ -104,7 +109,7 @@ pub async fn setup_double_anvil() -> eyre::Result { let middleware = setup_middleware( format!("http://{local_addr}"), - DEFAULT_ANVIL_PRIVATE_KEY, + SECONDARY_ANVIL_PRIVATE_KEY, ) .await?; @@ -138,11 +143,27 @@ pub async fn setup_service( ) -> eyre::Result<(Service, TxSitterClient)> { let rpc_url = anvil_handle.local_addr(); + let anvil_private_key = hex::encode(DEFAULT_ANVIL_PRIVATE_KEY); + let config = Config { service: TxSitterConfig { escalation_interval, datadog_enabled: false, statsd_enabled: false, + predefined: Some(Predefined { + network: PredefinedNetwork { + chain_id: DEFAULT_ANVIL_CHAIN_ID, + name: "Anvil".to_string(), + http_rpc: format!("http://{}", rpc_url), + ws_rpc: anvil_handle.ws_addr(), + }, + relayer: PredefinedRelayer { + name: "Anvil".to_string(), + id: DEFAULT_RELAYER_ID.to_string(), + key_id: anvil_private_key, + chain_id: DEFAULT_ANVIL_CHAIN_ID, + }, + }), }, server: ServerConfig { host: SocketAddr::V4(SocketAddrV4::new( @@ -153,7 +174,7 @@ pub async fn setup_service( password: None, }, database: DatabaseConfig::connection_string(db_connection_url), - keys: KeysConfig::Local(LocalKeysConfig {}), + keys: KeysConfig::Local(LocalKeysConfig::default()), }; let service = Service::new(config).await?; @@ -161,17 +182,6 @@ pub async fn setup_service( let client = TxSitterClient::new(format!("http://{}", service.local_addr())); - client - .create_network( - DEFAULT_ANVIL_CHAIN_ID, - &NewNetworkInfo { - name: "Anvil".to_string(), - http_rpc: format!("http://{}", rpc_url), - ws_rpc: anvil_handle.ws_addr(), - }, - ) - .await?; - Ok((service, client)) } @@ -179,7 +189,7 @@ pub async fn setup_middleware( rpc_url: impl AsRef, private_key: &[u8], ) -> eyre::Result { - let provider = Provider::::new(rpc_url.as_ref().parse()?); + let provider = setup_provider(rpc_url).await?; let wallet = LocalWallet::from(SigningKey::from_slice(private_key)?) .with_chain_id(provider.get_chainid().await?.as_u64()); @@ -188,3 +198,11 @@ pub async fn setup_middleware( Ok(middleware) } + +pub async fn setup_provider( + rpc_url: impl AsRef, +) -> eyre::Result> { + let provider = Provider::::new(rpc_url.as_ref().parse()?); + + Ok(provider) +} diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index 81509cc..c4ab6fe 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -18,15 +18,8 @@ async fn rpc_access() -> eyre::Result<()> { let (service, client) = setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let CreateRelayerResponse { relayer_id, .. } = client - .create_relayer(&CreateRelayerRequest { - name: "Test relayer".to_string(), - chain_id: DEFAULT_ANVIL_CHAIN_ID, - }) - .await?; - let CreateApiKeyResponse { api_key } = - client.create_relayer_api_key(&relayer_id).await?; + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; let rpc_url = format!("http://{}/1/api/{api_key}/rpc", service.local_addr()); diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 5476a3e..1e0226c 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -18,44 +18,11 @@ async fn send_many_txs() -> eyre::Result<()> { let (_service, client) = setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let CreateRelayerResponse { - address: relayer_address, - relayer_id, - } = client - .create_relayer(&CreateRelayerRequest { - name: "Test relayer".to_string(), - chain_id: DEFAULT_ANVIL_CHAIN_ID, - }) - .await?; - let CreateApiKeyResponse { api_key } = - client.create_relayer_api_key(&relayer_id).await?; - - // Fund the relayer - let middleware = setup_middleware( - format!("http://{}", double_anvil.local_addr()), - DEFAULT_ANVIL_PRIVATE_KEY, - ) - .await?; - - let amount: U256 = parse_units("1000", "ether")?.into(); - - middleware - .send_transaction( - Eip1559TransactionRequest { - to: Some(relayer_address.into()), - value: Some(amount), - ..Default::default() - }, - None, - ) - .await? - .await?; - - let provider = middleware.provider(); - - let current_balance = provider.get_balance(relayer_address, None).await?; - assert_eq!(current_balance, amount); + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; + + let provider = + setup_provider(format!("http://{}", double_anvil.local_addr())).await?; // Send a transaction let value: U256 = parse_units("10", "ether")?.into(); diff --git a/tests/send_tx.rs b/tests/send_tx.rs index cff3991..b4236ca 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -16,44 +16,11 @@ async fn send_tx() -> eyre::Result<()> { let (_service, client) = setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - let CreateRelayerResponse { - address: relayer_address, - relayer_id, - } = client - .create_relayer(&CreateRelayerRequest { - name: "Test relayer".to_string(), - chain_id: DEFAULT_ANVIL_CHAIN_ID, - }) - .await?; - let CreateApiKeyResponse { api_key } = - client.create_relayer_api_key(&relayer_id).await?; - - // Fund the relayer - let middleware = setup_middleware( - format!("http://{}", double_anvil.local_addr()), - DEFAULT_ANVIL_PRIVATE_KEY, - ) - .await?; - - let amount: U256 = parse_units("100", "ether")?.into(); - - middleware - .send_transaction( - Eip1559TransactionRequest { - to: Some(relayer_address.into()), - value: Some(amount), - ..Default::default() - }, - None, - ) - .await? - .await?; - - let provider = middleware.provider(); + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; - let current_balance = provider.get_balance(relayer_address, None).await?; - assert_eq!(current_balance, amount); + let provider = + setup_provider(format!("http://{}", double_anvil.local_addr())).await?; // Send a transaction let value: U256 = parse_units("1", "ether")?.into(); From 74301e65d9fea173aedf540d95a1799decded434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 13 Dec 2023 18:28:29 +0100 Subject: [PATCH 091/135] Allow filtering txs by status (#7) * Allow filtering txs by status * Minor refactor --- src/db.rs | 24 ++++++++++++++++++++++++ src/db/data.rs | 2 +- src/server/routes/transaction.rs | 22 ++++++++++++++++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/db.rs b/src/db.rs index 965117a..40457f1 100644 --- a/src/db.rs +++ b/src/db.rs @@ -754,7 +754,13 @@ impl Database { pub async fn read_txs( &self, relayer_id: &str, + tx_status_filter: Option>, ) -> eyre::Result> { + let (should_filter, status_filter) = match tx_status_filter { + Some(status) => (true, status), + None => (false, None), + }; + Ok(sqlx::query_as( r#" SELECT t.id as tx_id, t.tx_to as to, t.data, t.value, t.gas_limit, t.nonce, @@ -763,9 +769,12 @@ impl Database { LEFT JOIN sent_transactions s ON t.id = s.tx_id LEFT JOIN tx_hashes h ON s.valid_tx_hash = h.tx_hash WHERE t.relayer_id = $1 + AND ($2 = true AND s.status = $3) OR $2 = false "#, ) .bind(relayer_id) + .bind(should_filter) + .bind(status_filter) .fetch_all(&self.pool) .await?) } @@ -1310,6 +1319,9 @@ mod tests { assert_eq!(tx.nonce, 0); assert_eq!(tx.tx_hash, None); + let unsent_txs = db.read_txs(relayer_id, None).await?; + assert_eq!(unsent_txs.len(), 1, "1 unsent tx"); + let tx_hash_1 = H256::from_low_u64_be(1); let tx_hash_2 = H256::from_low_u64_be(2); let initial_max_fee_per_gas = U256::from(1); @@ -1328,6 +1340,18 @@ mod tests { assert_eq!(tx.tx_hash.unwrap().0, tx_hash_1); assert_eq!(tx.status, Some(TxStatus::Pending)); + let unsent_txs = db.read_txs(relayer_id, Some(None)).await?; + assert_eq!(unsent_txs.len(), 0, "0 unsent tx"); + + let pending_txs = db + .read_txs(relayer_id, Some(Some(TxStatus::Pending))) + .await?; + assert_eq!(pending_txs.len(), 1, "1 pending tx"); + + let all_txs = db.read_txs(relayer_id, None).await?; + + assert_eq!(all_txs, pending_txs); + db.escalate_tx( tx_id, tx_hash_2, diff --git a/src/db/data.rs b/src/db/data.rs index 7e8bd80..058feb0 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -43,7 +43,7 @@ pub struct TxForEscalation { pub escalation_count: usize, } -#[derive(Debug, Clone, FromRow)] +#[derive(Debug, Clone, FromRow, PartialEq, Eq)] pub struct ReadTxData { pub tx_id: String, pub to: AddressWrapper, diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index a1dbca7..33f7d9e 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use axum::extract::{Json, Path, State}; +use axum::extract::{Json, Path, Query, State}; use ethers::types::{Address, Bytes, H256, U256}; use eyre::Result; use serde::{Deserialize, Serialize}; @@ -33,6 +33,13 @@ pub struct SendTxResponse { pub tx_id: String, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetTxQuery { + #[serde(default)] + pub status: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetTxResponse { @@ -104,12 +111,23 @@ pub async fn send_tx( pub async fn get_txs( State(app): State>, Path(api_token): Path, + Query(query): Query, ) -> Result>, ApiError> { if !app.is_authorized(&api_token).await? { return Err(ApiError::Unauthorized); } - let txs = app.db.read_txs(&api_token.relayer_id).await?; + let txs = match query.status { + Some(GetTxResponseStatus::TxStatus(status)) => { + app.db + .read_txs(&api_token.relayer_id, Some(Some(status))) + .await? + } + Some(GetTxResponseStatus::Unsent(_)) => { + app.db.read_txs(&api_token.relayer_id, Some(None)).await? + } + None => app.db.read_txs(&api_token.relayer_id, None).await?, + }; let txs = txs.into_iter() From 524199b73b139e06a079dbf6926ea2fb62bf441a Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 17:48:58 -0500 Subject: [PATCH 092/135] insert into tx_hashes and sent_transactions seperatley, added raw_signed_tx for UniversalSigner --- src/db.rs | 38 ++++++++++++++---- src/keys/universal_signer.rs | 15 ++++++++ src/tasks/broadcast.rs | 74 +++++++++++++++++++++++++----------- 3 files changed, 97 insertions(+), 30 deletions(-) diff --git a/src/db.rs b/src/db.rs index ac0da7f..269e9b6 100644 --- a/src/db.rs +++ b/src/db.rs @@ -231,7 +231,7 @@ impl Database { .await?) } - pub async fn insert_tx_broadcast( + pub async fn insert_into_tx_hashes( &self, tx_id: &str, tx_hash: H256, @@ -246,8 +246,6 @@ impl Database { initial_max_priority_fee_per_gas .to_big_endian(&mut initial_max_priority_fee_per_gas_bytes); - let mut tx = self.pool.begin().await?; - sqlx::query( r#" INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) @@ -258,9 +256,27 @@ impl Database { .bind(tx_hash.as_bytes()) .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) - .execute(tx.as_mut()) + .execute(&self.pool) .await?; + Ok(()) + } + + pub async fn insert_into_sent_transactions( + &self, + tx_id: &str, + tx_hash: H256, + initial_max_fee_per_gas: U256, + initial_max_priority_fee_per_gas: U256, + ) -> eyre::Result<()> { + let mut initial_max_fee_per_gas_bytes = [0u8; 32]; + initial_max_fee_per_gas + .to_big_endian(&mut initial_max_fee_per_gas_bytes); + + let mut initial_max_priority_fee_per_gas_bytes = [0u8; 32]; + initial_max_priority_fee_per_gas + .to_big_endian(&mut initial_max_priority_fee_per_gas_bytes); + sqlx::query( r#" INSERT INTO sent_transactions (tx_id, initial_max_fee_per_gas, initial_max_priority_fee_per_gas, valid_tx_hash) @@ -271,9 +287,7 @@ impl Database { .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) .bind(tx_hash.as_bytes()) - .execute(tx.as_mut()).await?; - - tx.commit().await?; + .execute(&self.pool).await?; Ok(()) } @@ -1295,7 +1309,15 @@ mod tests { let initial_max_fee_per_gas = U256::from(1); let initial_max_priority_fee_per_gas = U256::from(1); - db.insert_tx_broadcast( + db.insert_into_tx_hashes( + tx_id, + tx_hash_1, + initial_max_fee_per_gas, + initial_max_priority_fee_per_gas, + ) + .await?; + + db.insert_into_sent_transactions( tx_id, tx_hash_1, initial_max_fee_per_gas, diff --git a/src/keys/universal_signer.rs b/src/keys/universal_signer.rs index 6bfd718..c54635c 100644 --- a/src/keys/universal_signer.rs +++ b/src/keys/universal_signer.rs @@ -3,6 +3,7 @@ use ethers::core::types::transaction::eip2718::TypedTransaction; use ethers::core::types::transaction::eip712::Eip712; use ethers::core::types::{Address, Signature as EthSig}; use ethers::signers::{Signer, Wallet, WalletError}; +use ethers::types::{Bytes, H256}; use thiserror::Error; use crate::aws::ethers_signer::AwsSigner; @@ -13,6 +14,20 @@ pub enum UniversalSigner { Local(Wallet), } +impl UniversalSigner { + pub async fn raw_signed_tx( + &self, + tx: &TypedTransaction, + ) -> eyre::Result { + let signature = match self { + Self::Aws(signer) => signer.sign_transaction(tx).await?, + Self::Local(signer) => signer.sign_transaction(tx).await?, + }; + + Ok(tx.rlp_signed(&signature)) + } +} + #[derive(Debug, Error)] pub enum UniversalError { #[error("AWS Signer Error: {0}")] diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index d541928..154276b 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -5,7 +5,7 @@ use std::time::Duration; use ethers::providers::Middleware; use ethers::types::transaction::eip2718::TypedTransaction; use ethers::types::transaction::eip2930::AccessList; -use ethers::types::{Address, Eip1559TransactionRequest, NameOrAddress}; +use ethers::types::{Address, Eip1559TransactionRequest, NameOrAddress, H256}; use eyre::ContextCompat; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -96,44 +96,74 @@ async fn broadcast_relayer_txs( max_base_fee_per_gas, ); - let eip1559_tx = Eip1559TransactionRequest { - from: None, - to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), - gas: Some(tx.gas_limit.0), - value: Some(tx.value.0), - data: Some(tx.data.into()), - nonce: Some(tx.nonce.into()), - access_list: AccessList::default(), - max_priority_fee_per_gas: Some(max_priority_fee_per_gas), - max_fee_per_gas: Some(max_fee_per_gas), - chain_id: Some(tx.chain_id.into()), + let mut typed_transaction = + TypedTransaction::Eip1559(Eip1559TransactionRequest { + from: None, + to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), + gas: Some(tx.gas_limit.0), + value: Some(tx.value.0), + data: Some(tx.data.into()), + nonce: Some(tx.nonce.into()), + access_list: AccessList::default(), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + max_fee_per_gas: Some(max_fee_per_gas), + chain_id: Some(tx.chain_id.into()), + }); + + // Fill and simulate the transaction + middleware + .fill_transaction(&mut typed_transaction, None) + .await?; + + // Simulate the transaction + match middleware.call(&typed_transaction, None).await { + Ok(_) => { + tracing::info!(?tx.id, "Tx simulated successfully"); + } + Err(err) => { + tracing::error!(?tx.id, error = ?err, "Failed to simulate tx"); + continue; + } }; - tracing::debug!(?eip1559_tx, "Sending tx"); + // Get the raw signed tx and derive the tx hash + let raw_signed_tx = middleware + .signer() + .raw_signed_tx(&typed_transaction) + .await?; + + let tx_hash = H256::from(ethers::utils::keccak256(&raw_signed_tx)); + + app.db + .insert_into_tx_hashes( + &tx.id, + tx_hash, + max_fee_per_gas, + max_priority_fee_per_gas, + ) + .await?; + + tracing::debug!(?tx.id, "Sending tx"); // TODO: Is it possible that we send a tx but don't store it in the DB? // TODO: Be smarter about error handling - a tx can fail to be sent // e.g. because the relayer is out of funds // but we don't want to retry it forever - let pending_tx = middleware - .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await; + let pending_tx = middleware.send_raw_transaction(raw_signed_tx).await; - let pending_tx = match pending_tx { + match pending_tx { Ok(pending_tx) => { tracing::info!(?pending_tx, "Tx sent successfully"); - pending_tx } Err(err) => { - tracing::error!(error = ?err, "Failed to send tx"); + tracing::error!(?tx.id, error = ?err, "Failed to send tx"); continue; } }; - let tx_hash = pending_tx.tx_hash(); - + // Insert the tx into app.db - .insert_tx_broadcast( + .insert_into_sent_transactions( &tx.id, tx_hash, max_fee_per_gas, From 38da96958f4585fa84d2c5cbf5d23fa3d0d5eef3 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 13 Dec 2023 23:53:06 +0100 Subject: [PATCH 093/135] Example .env --- .env.example | 1 + .gitignore | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..943896d --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +RUST_LOG=info,tx_sitter=debug,fake_rpc=debug,tower_http=debug diff --git a/.gitignore b/.gitignore index 9f97022..14ee500 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -target/ \ No newline at end of file +target/ +.env From dc962f01de207d10e127a05730f19dcaf06d67b2 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 13 Dec 2023 23:53:11 +0100 Subject: [PATCH 094/135] Fix --- src/service.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service.rs b/src/service.rs index a14598d..37a71c1 100644 --- a/src/service.rs +++ b/src/service.rs @@ -90,7 +90,9 @@ async fn initialize_predefined_values( eyre::bail!("Predefined relayers are only supported with local keys"); } - let predefined = app.config.service.predefined.as_ref().unwrap(); + let Some(predefined) = app.config.service.predefined.as_ref() else { + return Ok(()); + }; app.db .create_network( From 479e335a75080c224b3d97a3f7ee58f7c13cffbb Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:28:02 -0500 Subject: [PATCH 095/135] added logic to recover simulated txs --- src/db.rs | 17 +++++++++++ src/tasks/broadcast.rs | 67 ++++++++++++++++++++++++++---------------- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/src/db.rs b/src/db.rs index 269e9b6..fa413f1 100644 --- a/src/db.rs +++ b/src/db.rs @@ -292,6 +292,23 @@ impl Database { Ok(()) } + // Gets all transactions that were simulated but not sent + pub async fn recover_simulated_txs(&self) -> eyre::Result> { + Ok(sqlx::query_as( + r#" + SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.priority, t.nonce, r.key_id, r.chain_id + FROM transactions t + INNER JOIN tx_hashes h ON (h.tx_id = t.id) + INNER JOIN relayers r ON (t.relayer_id = r.id + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + WHERE s.tx_id IS NULL + ORDER BY r.id, t.nonce ASC; + "#, + ) + .fetch_all(&self.pool) + .await?) + } + pub async fn get_latest_block_number_without_fee_estimates( &self, chain_id: u64, diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 154276b..784f52c 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -18,39 +18,38 @@ use crate::broadcast_utils::{ use crate::db::UnsentTx; pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { - loop { - let mut txs = app.db.get_unsent_txs().await?; - - txs.sort_unstable_by_key(|tx| tx.relayer_id.clone()); + // Recovery any unsent transactions that were simulated but never sent + let recovered_txs = app.db.recover_simulated_txs().await?; + broadcast_unsent_txs(&app, recovered_txs).await?; - let txs_by_relayer = - txs.into_iter().group_by(|tx| tx.relayer_id.clone()); + loop { + // Get all unsent txs and broadcast + let txs = app.db.get_unsent_txs().await?; + broadcast_unsent_txs(&app, txs).await?; - let txs_by_relayer: HashMap<_, _> = txs_by_relayer - .into_iter() - .map(|(relayer_id, txs)| { - let mut txs = txs.collect_vec(); + tokio::time::sleep(Duration::from_secs(1)).await; + } +} - txs.sort_unstable_by_key(|tx| tx.nonce); +async fn broadcast_unsent_txs( + app: &App, + txs: Vec, +) -> eyre::Result<()> { + let txs_by_relayer = sort_txs_by_relayer(txs); - (relayer_id, txs) - }) - .collect(); + let mut futures = FuturesUnordered::new(); - let mut futures = FuturesUnordered::new(); + for (relayer_id, txs) in txs_by_relayer { + futures.push(broadcast_relayer_txs(&app, relayer_id, txs)); + } - for (relayer_id, txs) in txs_by_relayer { - futures.push(broadcast_relayer_txs(&app, relayer_id, txs)); + while let Some(result) = futures.next().await { + if let Err(err) = result { + tracing::error!(error = ?err, "Failed broadcasting txs"); } - - while let Some(result) = futures.next().await { - if let Err(err) = result { - tracing::error!(error = ?err, "Failed broadcasting txs"); - } - } - - tokio::time::sleep(Duration::from_secs(1)).await; } + + Ok(()) } #[tracing::instrument(skip(app, txs))] @@ -176,3 +175,21 @@ async fn broadcast_relayer_txs( Ok(()) } + +fn sort_txs_by_relayer( + mut txs: Vec, +) -> HashMap> { + txs.sort_unstable_by_key(|tx| tx.relayer_id.clone()); + let txs_by_relayer = txs.into_iter().group_by(|tx| tx.relayer_id.clone()); + + txs_by_relayer + .into_iter() + .map(|(relayer_id, txs)| { + let mut txs = txs.collect_vec(); + + txs.sort_unstable_by_key(|tx| tx.nonce); + + (relayer_id, txs) + }) + .collect() +} From 9343e424dc29aa97c69331ef579d2e82ad8c1eae Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:28:40 -0500 Subject: [PATCH 096/135] cargo clippy, cargo sort --- Cargo.toml | 90 ++++++++++++++++++------------------ src/keys/universal_signer.rs | 2 +- src/tasks/broadcast.rs | 2 +- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f1f02e5..191421c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,54 +8,50 @@ default-run = "tx-sitter" members = ["crates/*"] [dependencies] +async-trait = "0.1.74" # Third Party ## AWS aws-config = { version = "1.0.1" } -aws-sdk-kms = "1.3.0" -aws-smithy-types = "1.0.2" -aws-smithy-runtime-api = "1.0.2" -aws-types = "1.0.1" aws-credential-types = { version = "1.0.1", features = [ "hardcoded-credentials", ] } - -## Other -serde = "1.0.136" +aws-sdk-kms = "1.3.0" +aws-smithy-runtime-api = "1.0.2" +aws-smithy-types = "1.0.2" +aws-types = "1.0.1" axum = { version = "0.6.20", features = ["headers"] } -thiserror = "1.0.50" -headers = "0.3.9" -humantime = "2.1.0" -humantime-serde = "1.1.1" -hyper = "0.14.27" -dotenv = "0.15.0" +base64 = "0.21.5" +bigdecimal = "0.4.2" +chrono = "0.4" clap = { version = "4.3.0", features = ["env", "derive"] } +config = "0.13.3" +dotenv = "0.15.0" ethers = { version = "2.0.11", features = ["ws"] } eyre = "0.6.5" +futures = "0.3" +headers = "0.3.9" hex = "0.4.3" hex-literal = "0.4.1" +humantime = "2.1.0" +humantime-serde = "1.1.1" +hyper = "0.14.27" +itertools = "0.12.0" +metrics = "0.21.1" +num-bigint = "0.4.4" +# telemetry-batteries = { path = "../telemetry-batteries" } + +# Internal +postgres-docker-utils = { path = "crates/postgres-docker-utils" } +rand = "0.8.5" reqwest = { version = "0.11.13", default-features = false, features = [ "rustls-tls", ] } + +## Other +serde = "1.0.136" serde_json = "1.0.91" -strum = { version = "0.25.0", features = ["derive"] } -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -tracing = { version = "0.1", features = ["log"] } -tracing-subscriber = { version = "0.3", default-features = false, features = [ - "env-filter", - "std", - "fmt", - "json", - "ansi", -] } -tower-http = { version = "0.4.4", features = [ "trace", "auth" ] } -uuid = { version = "0.8", features = ["v4"] } -futures = "0.3" -chrono = "0.4" -rand = "0.8.5" sha3 = "0.10.8" -config = "0.13.3" -toml = "0.8.8" -url = "2.4.1" +spki = "0.7.2" sqlx = { version = "0.7.2", features = [ "time", "chrono", @@ -65,26 +61,30 @@ sqlx = { version = "0.7.2", features = [ "migrate", "bigdecimal", ] } -metrics = "0.21.1" -num-bigint = "0.4.4" -bigdecimal = "0.4.2" -spki = "0.7.2" -async-trait = "0.1.74" -itertools = "0.12.0" -base64 = "0.21.5" +strum = { version = "0.25.0", features = ["derive"] } # Company telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", branch = "dzejkop/unnest-fields" } -# telemetry-batteries = { path = "../telemetry-batteries" } - -# Internal -postgres-docker-utils = { path = "crates/postgres-docker-utils" } +thiserror = "1.0.50" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +toml = "0.8.8" +tower-http = { version = "0.4.4", features = [ "trace", "auth" ] } +tracing = { version = "0.1", features = ["log"] } +tracing-subscriber = { version = "0.3", default-features = false, features = [ + "env-filter", + "std", + "fmt", + "json", + "ansi", +] } +url = "2.4.1" +uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -test-case = "3.1.0" -indoc = "2.0.3" fake-rpc = { path = "crates/fake-rpc" } +indoc = "2.0.3" +test-case = "3.1.0" [features] -default = [ "default-config" ] +default = ["default-config"] default-config = [] diff --git a/src/keys/universal_signer.rs b/src/keys/universal_signer.rs index c54635c..bd96e80 100644 --- a/src/keys/universal_signer.rs +++ b/src/keys/universal_signer.rs @@ -3,7 +3,7 @@ use ethers::core::types::transaction::eip2718::TypedTransaction; use ethers::core::types::transaction::eip712::Eip712; use ethers::core::types::{Address, Signature as EthSig}; use ethers::signers::{Signer, Wallet, WalletError}; -use ethers::types::{Bytes, H256}; +use ethers::types::{Bytes}; use thiserror::Error; use crate::aws::ethers_signer::AwsSigner; diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 784f52c..c4b47c6 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -40,7 +40,7 @@ async fn broadcast_unsent_txs( let mut futures = FuturesUnordered::new(); for (relayer_id, txs) in txs_by_relayer { - futures.push(broadcast_relayer_txs(&app, relayer_id, txs)); + futures.push(broadcast_relayer_txs(app, relayer_id, txs)); } while let Some(result) = futures.next().await { From 89a93b159ed6d19133245f3d59cafbe45dd9ffad Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:29:04 -0500 Subject: [PATCH 097/135] formatting --- src/keys/universal_signer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keys/universal_signer.rs b/src/keys/universal_signer.rs index bd96e80..2a3db9d 100644 --- a/src/keys/universal_signer.rs +++ b/src/keys/universal_signer.rs @@ -3,7 +3,7 @@ use ethers::core::types::transaction::eip2718::TypedTransaction; use ethers::core::types::transaction::eip712::Eip712; use ethers::core::types::{Address, Signature as EthSig}; use ethers::signers::{Signer, Wallet, WalletError}; -use ethers::types::{Bytes}; +use ethers::types::Bytes; use thiserror::Error; use crate::aws::ethers_signer::AwsSigner; From 280abf4415d060bb93082521e6351671e5df9265 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:30:58 -0500 Subject: [PATCH 098/135] removed recover simulated tx --- src/db.rs | 17 ----------------- src/tasks/broadcast.rs | 5 ----- 2 files changed, 22 deletions(-) diff --git a/src/db.rs b/src/db.rs index fa413f1..269e9b6 100644 --- a/src/db.rs +++ b/src/db.rs @@ -292,23 +292,6 @@ impl Database { Ok(()) } - // Gets all transactions that were simulated but not sent - pub async fn recover_simulated_txs(&self) -> eyre::Result> { - Ok(sqlx::query_as( - r#" - SELECT r.id as relayer_id, t.id, t.tx_to, t.data, t.value, t.gas_limit, t.priority, t.nonce, r.key_id, r.chain_id - FROM transactions t - INNER JOIN tx_hashes h ON (h.tx_id = t.id) - INNER JOIN relayers r ON (t.relayer_id = r.id - LEFT JOIN sent_transactions s ON (t.id = s.tx_id) - WHERE s.tx_id IS NULL - ORDER BY r.id, t.nonce ASC; - "#, - ) - .fetch_all(&self.pool) - .await?) - } - pub async fn get_latest_block_number_without_fee_estimates( &self, chain_id: u64, diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index c4b47c6..eb3ab9a 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -18,15 +18,10 @@ use crate::broadcast_utils::{ use crate::db::UnsentTx; pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { - // Recovery any unsent transactions that were simulated but never sent - let recovered_txs = app.db.recover_simulated_txs().await?; - broadcast_unsent_txs(&app, recovered_txs).await?; - loop { // Get all unsent txs and broadcast let txs = app.db.get_unsent_txs().await?; broadcast_unsent_txs(&app, txs).await?; - tokio::time::sleep(Duration::from_secs(1)).await; } } From 7d9ebd2592d815b8b90376bf58bb50b6dd532abe Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:44:35 -0500 Subject: [PATCH 099/135] updated insert_into_tx_hashes to do nothing on conflict --- src/db.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/db.rs b/src/db.rs index 269e9b6..fe96728 100644 --- a/src/db.rs +++ b/src/db.rs @@ -250,6 +250,7 @@ impl Database { r#" INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) VALUES ($1, $2, $3, $4) + ON CONFLICT (tx_hash) DO NOTHING "#, ) .bind(tx_id) From 8488a9e3e2c61d6aebff5c9a2d26d626e52074a7 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 18:51:24 -0500 Subject: [PATCH 100/135] on conflict, update tx_hashes --- src/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.rs b/src/db.rs index fe96728..fd04681 100644 --- a/src/db.rs +++ b/src/db.rs @@ -250,7 +250,7 @@ impl Database { r#" INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) VALUES ($1, $2, $3, $4) - ON CONFLICT (tx_hash) DO NOTHING + ON CONFLICT (tx_hash) DO UPDATE "#, ) .bind(tx_id) From 9412b0888e265479ce69a7c8e5016cfa9a2d0ca3 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Wed, 13 Dec 2023 19:02:27 -0500 Subject: [PATCH 101/135] updated tx_hashes to add constraint on tx_id, do nothing on conflict when inserting into table --- db/migrations/001_init.sql | 3 +++ src/db.rs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 62a7209..4490b20 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -60,6 +60,9 @@ CREATE TABLE tx_hashes ( escalated BOOL NOT NULL DEFAULT FALSE ); +ALTER TABLE tx_hashes +ADD UNIQUE (tx_id); + -- Dynamic tx data & data used for escalations CREATE TABLE sent_transactions ( tx_id VARCHAR(255) PRIMARY KEY REFERENCES transactions(id) ON DELETE CASCADE, diff --git a/src/db.rs b/src/db.rs index fd04681..914661d 100644 --- a/src/db.rs +++ b/src/db.rs @@ -250,7 +250,7 @@ impl Database { r#" INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) VALUES ($1, $2, $3, $4) - ON CONFLICT (tx_hash) DO UPDATE + ON CONFLICT (tx_id) DO NOTHING "#, ) .bind(tx_id) From 00ff1c68723b3bfecc1bc9f519d03d4e317aef98 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Thu, 14 Dec 2023 08:59:19 -0500 Subject: [PATCH 102/135] added insert_tx_broadcast --- db/migrations/001_init.sql | 3 --- src/db.rs | 29 +++++++---------------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/db/migrations/001_init.sql b/db/migrations/001_init.sql index 4490b20..62a7209 100644 --- a/db/migrations/001_init.sql +++ b/db/migrations/001_init.sql @@ -60,9 +60,6 @@ CREATE TABLE tx_hashes ( escalated BOOL NOT NULL DEFAULT FALSE ); -ALTER TABLE tx_hashes -ADD UNIQUE (tx_id); - -- Dynamic tx data & data used for escalations CREATE TABLE sent_transactions ( tx_id VARCHAR(255) PRIMARY KEY REFERENCES transactions(id) ON DELETE CASCADE, diff --git a/src/db.rs b/src/db.rs index 914661d..ace95df 100644 --- a/src/db.rs +++ b/src/db.rs @@ -231,7 +231,7 @@ impl Database { .await?) } - pub async fn insert_into_tx_hashes( + pub async fn insert_tx_broadcast( &self, tx_id: &str, tx_hash: H256, @@ -246,38 +246,21 @@ impl Database { initial_max_priority_fee_per_gas .to_big_endian(&mut initial_max_priority_fee_per_gas_bytes); + let mut tx = self.pool.begin().await?; + sqlx::query( r#" INSERT INTO tx_hashes (tx_id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) VALUES ($1, $2, $3, $4) - ON CONFLICT (tx_id) DO NOTHING "#, ) .bind(tx_id) .bind(tx_hash.as_bytes()) .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) - .execute(&self.pool) + .execute(tx.as_mut()) .await?; - Ok(()) - } - - pub async fn insert_into_sent_transactions( - &self, - tx_id: &str, - tx_hash: H256, - initial_max_fee_per_gas: U256, - initial_max_priority_fee_per_gas: U256, - ) -> eyre::Result<()> { - let mut initial_max_fee_per_gas_bytes = [0u8; 32]; - initial_max_fee_per_gas - .to_big_endian(&mut initial_max_fee_per_gas_bytes); - - let mut initial_max_priority_fee_per_gas_bytes = [0u8; 32]; - initial_max_priority_fee_per_gas - .to_big_endian(&mut initial_max_priority_fee_per_gas_bytes); - sqlx::query( r#" INSERT INTO sent_transactions (tx_id, initial_max_fee_per_gas, initial_max_priority_fee_per_gas, valid_tx_hash) @@ -288,7 +271,9 @@ impl Database { .bind(initial_max_fee_per_gas_bytes) .bind(initial_max_priority_fee_per_gas_bytes) .bind(tx_hash.as_bytes()) - .execute(&self.pool).await?; + .execute(tx.as_mut()).await?; + + tx.commit().await?; Ok(()) } From eb6b1fc7068027d7075b2377cc912f0551bd542c Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Thu, 14 Dec 2023 09:02:23 -0500 Subject: [PATCH 103/135] write to database after successful simulation --- src/db.rs | 10 +--------- src/tasks/broadcast.rs | 14 +++----------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/db.rs b/src/db.rs index ace95df..ac0da7f 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1295,15 +1295,7 @@ mod tests { let initial_max_fee_per_gas = U256::from(1); let initial_max_priority_fee_per_gas = U256::from(1); - db.insert_into_tx_hashes( - tx_id, - tx_hash_1, - initial_max_fee_per_gas, - initial_max_priority_fee_per_gas, - ) - .await?; - - db.insert_into_sent_transactions( + db.insert_tx_broadcast( tx_id, tx_hash_1, initial_max_fee_per_gas, diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index eb3ab9a..ee6159a 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -109,6 +109,8 @@ async fn broadcast_relayer_txs( .fill_transaction(&mut typed_transaction, None) .await?; + tracing::debug!(?tx.id, "Simulating tx"); + // Simulate the transaction match middleware.call(&typed_transaction, None).await { Ok(_) => { @@ -129,7 +131,7 @@ async fn broadcast_relayer_txs( let tx_hash = H256::from(ethers::utils::keccak256(&raw_signed_tx)); app.db - .insert_into_tx_hashes( + .insert_tx_broadcast( &tx.id, tx_hash, max_fee_per_gas, @@ -155,16 +157,6 @@ async fn broadcast_relayer_txs( } }; - // Insert the tx into - app.db - .insert_into_sent_transactions( - &tx.id, - tx_hash, - max_fee_per_gas, - max_priority_fee_per_gas, - ) - .await?; - tracing::info!(id = tx.id, hash = ?tx_hash, "Tx broadcast"); } From d74d027505969418d965f6840f8f54467058d15c Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Thu, 14 Dec 2023 09:13:34 -0500 Subject: [PATCH 104/135] removed unneeded function --- src/tasks/broadcast.rs | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index ee6159a..9822cc4 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -21,30 +21,22 @@ pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { loop { // Get all unsent txs and broadcast let txs = app.db.get_unsent_txs().await?; - broadcast_unsent_txs(&app, txs).await?; - tokio::time::sleep(Duration::from_secs(1)).await; - } -} + let txs_by_relayer = sort_txs_by_relayer(txs); -async fn broadcast_unsent_txs( - app: &App, - txs: Vec, -) -> eyre::Result<()> { - let txs_by_relayer = sort_txs_by_relayer(txs); - - let mut futures = FuturesUnordered::new(); + let mut futures = FuturesUnordered::new(); - for (relayer_id, txs) in txs_by_relayer { - futures.push(broadcast_relayer_txs(app, relayer_id, txs)); - } + for (relayer_id, txs) in txs_by_relayer { + futures.push(broadcast_relayer_txs(&app, relayer_id, txs)); + } - while let Some(result) = futures.next().await { - if let Err(err) = result { - tracing::error!(error = ?err, "Failed broadcasting txs"); + while let Some(result) = futures.next().await { + if let Err(err) = result { + tracing::error!(error = ?err, "Failed broadcasting txs"); + } } - } - Ok(()) + tokio::time::sleep(Duration::from_secs(1)).await; + } } #[tracing::instrument(skip(app, txs))] From 2f73406ae6d76d68628d8f9f2ae000188d70e69f Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Thu, 14 Dec 2023 09:25:54 -0500 Subject: [PATCH 105/135] updated cargo toml to match dev branch --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 191421c..6051d7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries" thiserror = "1.0.50" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } toml = "0.8.8" -tower-http = { version = "0.4.4", features = [ "trace", "auth" ] } +tower-http = { version = "0.4.4", features = ["trace", "auth"] } tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", default-features = false, features = [ "env-filter", From 9cb1e3e0a5303e7f738bd0ec45acaf6a8b73ec01 Mon Sep 17 00:00:00 2001 From: 0xKitsune <0xKitsune@protonmail.com> Date: Thu, 14 Dec 2023 10:25:30 -0500 Subject: [PATCH 106/135] removed comments and todo --- Cargo.toml | 2 +- src/tasks/broadcast.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6051d7e..192f45c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = ["crates/*"] [dependencies] async-trait = "0.1.74" -# Third Party + ## AWS aws-config = { version = "1.0.1" } aws-credential-types = { version = "1.0.1", features = [ diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 9822cc4..7f33f21 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -133,7 +133,6 @@ async fn broadcast_relayer_txs( tracing::debug!(?tx.id, "Sending tx"); - // TODO: Is it possible that we send a tx but don't store it in the DB? // TODO: Be smarter about error handling - a tx can fail to be sent // e.g. because the relayer is out of funds // but we don't want to retry it forever From c5ee7fb098cc55631afa30aaadc62f9fd3659210 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 18 Dec 2023 13:24:12 +0100 Subject: [PATCH 107/135] Build for multiple platforms --- .github/workflows/build-image-and-publish.yml | 62 ++++++++++++------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index e3191c3..647fe82 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -4,28 +4,48 @@ on: push: branches: [ main, dev ] +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: build-and-push: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: [linux/amd64, linux/arm64, linux/arm/v7] steps: - - name: Check Out Repo - uses: actions/checkout@v2 - - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: . - file: Dockerfile - push: true - tags: dzejkop/tx-sitter:latest - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Check Out Repo + uses: actions/checkout@v2 + + - name: Log in to Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v1 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: ${{ matrix.platform }} + cache-from: type=gha + cache-to: type=gha,mode=max From 4fcf48b8de717e947f43cf1b452969fca4aa839e Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 18 Dec 2023 13:55:29 +0100 Subject: [PATCH 108/135] Don't build for arm/v7 --- .github/workflows/build-image-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 647fe82..ac09a94 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - platform: [linux/amd64, linux/arm64, linux/arm/v7] + platform: [linux/amd64, linux/arm64] steps: - name: Check Out Repo uses: actions/checkout@v2 From 986f865230544edd50b63c1fbb1a8da76f6b3921 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 18 Dec 2023 22:23:15 +0100 Subject: [PATCH 109/135] Tags --- .github/workflows/build-image-and-publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index ac09a94..9731fbc 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -31,6 +31,11 @@ jobs: uses: docker/metadata-action@v1 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha + type=raw,value=latest-${{ matrix.platform }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 877922c0bf17237e8a4c6c14805634853f1ddc18 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Mon, 18 Dec 2023 23:54:23 +0100 Subject: [PATCH 110/135] TEMP: Don't build for arm64 --- .github/workflows/build-image-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 9731fbc..b2c3607 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - platform: [linux/amd64, linux/arm64] + platform: [linux/amd64] steps: - name: Check Out Repo uses: actions/checkout@v2 From e2afc2dc86551c4cb6117b6afb95916904b143d2 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 19 Dec 2023 00:13:00 +0100 Subject: [PATCH 111/135] Bring back arm64 --- .github/workflows/build-image-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index b2c3607..9731fbc 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - platform: [linux/amd64] + platform: [linux/amd64, linux/arm64] steps: - name: Check Out Repo uses: actions/checkout@v2 From 3e5b12246a8b19920e981cac39a4f8b4168a4d19 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 19 Dec 2023 12:09:51 +0100 Subject: [PATCH 112/135] Update action versions --- .github/workflows/build-image-and-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 9731fbc..af02a18 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -17,10 +17,10 @@ jobs: platform: [linux/amd64, linux/arm64] steps: - name: Check Out Repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Log in to Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -28,7 +28,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v1 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -44,7 +44,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: . file: Dockerfile From 81cb3b0f7132dfe4c590c65c47a8bc84b6838bcb Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 19 Dec 2023 12:38:39 +0100 Subject: [PATCH 113/135] Add annotations --- .github/workflows/build-image-and-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index af02a18..54f7414 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -52,5 +52,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ matrix.platform }} + annotations: ${{ steps.meta.outputs.annotations }} cache-from: type=gha cache-to: type=gha,mode=max From 82cc394c1bf700eccd2ecfa4b0c250904f5b35f2 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 19 Dec 2023 13:23:39 +0100 Subject: [PATCH 114/135] No more matrix --- .github/workflows/build-image-and-publish.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 54f7414..63d2d39 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -11,10 +11,6 @@ env: jobs: build-and-push: runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: [linux/amd64, linux/arm64] steps: - name: Check Out Repo uses: actions/checkout@v4 @@ -51,7 +47,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: ${{ matrix.platform }} + platforms: linux/amd64,linux/arm64 annotations: ${{ steps.meta.outputs.annotations }} cache-from: type=gha cache-to: type=gha,mode=max From b1751a004ec407d8bececf55cbddb9b7a541d06c Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Tue, 19 Dec 2023 13:24:33 +0100 Subject: [PATCH 115/135] Remove matrix reference --- .github/workflows/build-image-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image-and-publish.yml b/.github/workflows/build-image-and-publish.yml index 63d2d39..0fb4553 100644 --- a/.github/workflows/build-image-and-publish.yml +++ b/.github/workflows/build-image-and-publish.yml @@ -31,7 +31,7 @@ jobs: type=ref,event=branch type=ref,event=pr type=sha - type=raw,value=latest-${{ matrix.platform }} + type=raw,value=latest - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 0feee6f7bead53234feddb5009fe8764c41370cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 27 Dec 2023 18:05:37 +0100 Subject: [PATCH 116/135] Reorg & Escalation testing (#11) --- Cargo.lock | 38 -------- Cargo.toml | 1 - crates/fake-rpc/Cargo.toml | 52 ----------- crates/fake-rpc/src/lib.rs | 153 -------------------------------- crates/fake-rpc/src/main.rs | 30 ------- src/broadcast_utils.rs | 31 +------ src/client.rs | 29 +++++- src/config.rs | 22 +++++ src/tasks/escalate.rs | 9 +- src/tasks/handle_reorgs.rs | 15 +--- tests/common/anvil_builder.rs | 67 ++++++++++++++ tests/common/mod.rs | 136 ++++------------------------ tests/common/service_builder.rs | 96 ++++++++++++++++++++ tests/create_relayer.rs | 6 +- tests/escalation.rs | 86 ++++++++++++++++++ tests/reorg.rs | 74 +++++++++++++++ tests/rpc_access.rs | 10 +-- tests/send_many_txs.rs | 13 +-- tests/send_tx.rs | 12 +-- 19 files changed, 411 insertions(+), 469 deletions(-) delete mode 100644 crates/fake-rpc/Cargo.toml delete mode 100644 crates/fake-rpc/src/lib.rs delete mode 100644 crates/fake-rpc/src/main.rs create mode 100644 tests/common/anvil_builder.rs create mode 100644 tests/common/service_builder.rs create mode 100644 tests/escalation.rs create mode 100644 tests/reorg.rs diff --git a/Cargo.lock b/Cargo.lock index 68fd11b..698e24e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1705,43 +1705,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fake-rpc" -version = "0.1.0" -dependencies = [ - "async-trait", - "axum", - "chrono", - "clap", - "config", - "dotenv", - "ethers", - "ethers-signers", - "eyre", - "futures", - "headers", - "hex", - "hex-literal", - "humantime", - "humantime-serde", - "hyper", - "rand", - "reqwest", - "serde", - "serde_json", - "sha3", - "spki", - "sqlx", - "strum", - "thiserror", - "tokio", - "toml 0.8.8", - "tower-http", - "tracing", - "tracing-subscriber", - "uuid 0.8.2", -] - [[package]] name = "fastrand" version = "2.0.1" @@ -5040,7 +5003,6 @@ dependencies = [ "dotenv", "ethers", "eyre", - "fake-rpc", "futures", "headers", "hex", diff --git a/Cargo.toml b/Cargo.toml index 192f45c..f9f6b47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,6 @@ url = "2.4.1" uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -fake-rpc = { path = "crates/fake-rpc" } indoc = "2.0.3" test-case = "3.1.0" diff --git a/crates/fake-rpc/Cargo.toml b/crates/fake-rpc/Cargo.toml deleted file mode 100644 index da94d7a..0000000 --- a/crates/fake-rpc/Cargo.toml +++ /dev/null @@ -1,52 +0,0 @@ -[package] -name = "fake-rpc" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -serde = "1.0.136" -axum = { version = "0.6.20", features = ["headers"] } -thiserror = "1.0.50" -headers = "0.3.9" -humantime = "2.1.0" -humantime-serde = "1.1.1" -hyper = "0.14.27" -dotenv = "0.15.0" -clap = { version = "4.3.0", features = ["env", "derive"] } -ethers = { version = "2.0.11" } -ethers-signers = { version = "2.0.11" } -eyre = "0.6.5" -hex = "0.4.3" -hex-literal = "0.4.1" -reqwest = { version = "0.11.13", default-features = false, features = [ - "rustls-tls", -] } -serde_json = "1.0.91" -strum = { version = "0.25.0", features = ["derive"] } -tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -tracing = { version = "0.1", features = ["log"] } -tracing-subscriber = { version = "0.3", default-features = false, features = [ - "env-filter", - "std", - "fmt", - "json", - "ansi", -] } -tower-http = { version = "0.4.4", features = ["trace"] } -uuid = { version = "0.8", features = ["v4"] } -futures = "0.3" -chrono = "0.4" -rand = "0.8.5" -sha3 = "0.10.8" -config = "0.13.3" -toml = "0.8.8" -sqlx = { version = "0.7.2", features = [ - "runtime-tokio", - "tls-rustls", - "postgres", - "migrate", -] } -spki = "0.7.2" -async-trait = "0.1.74" diff --git a/crates/fake-rpc/src/lib.rs b/crates/fake-rpc/src/lib.rs deleted file mode 100644 index 81c334e..0000000 --- a/crates/fake-rpc/src/lib.rs +++ /dev/null @@ -1,153 +0,0 @@ -use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::atomic::AtomicBool; -use std::sync::Arc; -use std::time::Duration; - -use axum::extract::State; -use axum::routing::{post, IntoMakeService}; -use axum::{Json, Router}; -use ethers::utils::{Anvil, AnvilInstance}; -use hyper::server::conn::AddrIncoming; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tokio::sync::Mutex; - -pub const BLOCK_TIME_SECONDS: u64 = 2; - -pub struct DoubleAnvil { - main_anvil: Mutex, - reference_anvil: Mutex, - held_back_txs: Mutex>, - - auto_advance: AtomicBool, -} - -impl DoubleAnvil { - pub async fn drop_txs(&self) -> eyre::Result<()> { - let mut held_back_txs = self.held_back_txs.lock().await; - held_back_txs.clear(); - Ok(()) - } - - pub async fn advance(&self) -> eyre::Result<()> { - let mut held_back_txs = self.held_back_txs.lock().await; - - for req in held_back_txs.drain(..) { - tracing::info!(?req, "eth_sendRawTransaction"); - - let response = reqwest::Client::new() - .post(&self.main_anvil.lock().await.endpoint()) - .json(&req) - .send() - .await - .unwrap(); - - let resp = response.json::().await.unwrap(); - - tracing::info!(?resp, "eth_sendRawTransaction.response"); - } - - Ok(()) - } - - pub fn set_auto_advance(&self, auto_advance: bool) { - self.auto_advance - .store(auto_advance, std::sync::atomic::Ordering::SeqCst); - } - - pub async fn ws_endpoint(&self) -> String { - self.main_anvil.lock().await.ws_endpoint() - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct JsonRpcReq { - pub id: u64, - pub jsonrpc: String, - pub method: String, - #[serde(default)] - pub params: Value, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct JsonRpcResponse { - pub id: u64, - pub jsonrpc: String, - pub result: Value, -} - -async fn advance(State(anvil): State>) { - anvil.advance().await.unwrap(); -} - -async fn rpc( - State(anvil): State>, - Json(req): Json, -) -> Json { - let method = req.method.as_str(); - let anvil_instance = match method { - "eth_sendRawTransaction" => { - anvil.held_back_txs.lock().await.push(req.clone()); - - if anvil.auto_advance.load(std::sync::atomic::Ordering::SeqCst) { - anvil.advance().await.unwrap(); - } - - anvil.reference_anvil.lock().await - } - "eth_getTransactionReceipt" => anvil.main_anvil.lock().await, - "eth_getTransactionByHash" => anvil.reference_anvil.lock().await, - _ => anvil.main_anvil.lock().await, - }; - - tracing::info!(?req, "{}", method); - - let response = reqwest::Client::new() - .post(&anvil_instance.endpoint()) - .json(&req) - .send() - .await - .unwrap(); - - let resp = response.json::().await.unwrap(); - - tracing::info!(?resp, "{}.response", method); - - Json(resp) -} - -pub async fn serve( - port: u16, -) -> ( - Arc, - axum::Server>, -) { - let main_anvil = Anvil::new().block_time(BLOCK_TIME_SECONDS).spawn(); - let reference_anvil = Anvil::new().block_time(BLOCK_TIME_SECONDS).spawn(); - - tokio::time::sleep(Duration::from_secs(BLOCK_TIME_SECONDS)).await; - - tracing::info!("Main anvil instance: {}", main_anvil.endpoint()); - tracing::info!("Reference anvil instance: {}", reference_anvil.endpoint()); - - let state = Arc::new(DoubleAnvil { - main_anvil: Mutex::new(main_anvil), - reference_anvil: Mutex::new(reference_anvil), - held_back_txs: Mutex::new(Vec::new()), - auto_advance: AtomicBool::new(true), - }); - - let router = Router::new() - .route("/", post(rpc)) - .route("/advance", post(advance)) - .with_state(state.clone()) - .layer(tower_http::trace::TraceLayer::new_for_http()); - - let host = Ipv4Addr::new(127, 0, 0, 1); - let socket_addr = SocketAddr::new(host.into(), port); - - let server = - axum::Server::bind(&socket_addr).serve(router.into_make_service()); - - (state, server) -} diff --git a/crates/fake-rpc/src/main.rs b/crates/fake-rpc/src/main.rs deleted file mode 100644 index a20b425..0000000 --- a/crates/fake-rpc/src/main.rs +++ /dev/null @@ -1,30 +0,0 @@ -use clap::Parser; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::EnvFilter; - -#[derive(Debug, Clone, Parser)] -struct Args { - #[clap(short, long, default_value = "8545")] - port: u16, -} - -#[tokio::main] -async fn main() -> eyre::Result<()> { - dotenv::dotenv().ok(); - - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().pretty().compact()) - .with(EnvFilter::from_default_env()) - .init(); - - let args = Args::parse(); - - let (_app, server) = fake_rpc::serve(args.port).await; - - tracing::info!("Serving fake RPC at {}", server.local_addr()); - - server.await?; - - Ok(()) -} diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index a0182d4..35c9318 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -1,4 +1,4 @@ -use ethers::types::{Eip1559TransactionRequest, U256}; +use ethers::types::U256; use eyre::ContextCompat; use self::gas_estimation::FeesEstimate; @@ -19,35 +19,6 @@ pub fn calculate_gas_fees_from_estimates( (max_fee_per_gas, max_priority_fee_per_gas) } -pub fn escalate_priority_fee( - max_base_fee_per_gas: U256, - max_network_fee_per_gas: U256, - current_max_priority_fee_per_gas: U256, - escalation_count: usize, - tx: &mut Eip1559TransactionRequest, -) { - // Min increase of 20% on the priority fee required for a replacement tx - let increased_gas_price_percentage = - U256::from(100 + (10 * (1 + escalation_count))); - - let factor = U256::from(100); - - let new_max_priority_fee_per_gas = current_max_priority_fee_per_gas - * increased_gas_price_percentage - / factor; - - let new_max_priority_fee_per_gas = - std::cmp::min(new_max_priority_fee_per_gas, max_network_fee_per_gas); - - let new_max_fee_per_gas = - max_base_fee_per_gas + new_max_priority_fee_per_gas; - let new_max_fee_per_gas = - std::cmp::min(new_max_fee_per_gas, max_network_fee_per_gas); - - tx.max_fee_per_gas = Some(new_max_fee_per_gas); - tx.max_priority_fee_per_gas = Some(new_max_priority_fee_per_gas); -} - pub async fn should_send_transaction( app: &App, relayer_id: &str, diff --git a/src/client.rs b/src/client.rs index b479e5d..49aa7d1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,7 +5,9 @@ use crate::server::routes::network::NewNetworkInfo; use crate::server::routes::relayer::{ CreateApiKeyResponse, CreateRelayerRequest, CreateRelayerResponse, }; -use crate::server::routes::transaction::{SendTxRequest, SendTxResponse}; +use crate::server::routes::transaction::{ + GetTxResponse, SendTxRequest, SendTxResponse, +}; pub struct TxSitterClient { client: reqwest::Client, @@ -43,6 +45,17 @@ impl TxSitterClient { Ok(response.json().await?) } + async fn json_get(&self, url: &str) -> eyre::Result + where + R: serde::de::DeserializeOwned, + { + let response = self.client.get(url).send().await?; + + let response = Self::validate_response(response).await?; + + Ok(response.json().await?) + } + async fn validate_response(response: Response) -> eyre::Result { if !response.status().is_success() { let body = response.text().await?; @@ -77,6 +90,20 @@ impl TxSitterClient { .await } + pub async fn get_tx( + &self, + api_key: &ApiKey, + tx_id: &str, + ) -> eyre::Result { + self.json_get(&format!( + "{}/1/api/{api_key}/tx/{tx_id}", + self.url, + api_key = api_key, + tx_id = tx_id + )) + .await + } + pub async fn create_network( &self, chain_id: u64, diff --git a/src/config.rs b/src/config.rs index 7e1e534..2869c1e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,6 +18,12 @@ pub struct TxSitterConfig { #[serde(with = "humantime_serde")] pub escalation_interval: Duration, + #[serde(with = "humantime_serde", default = "default_soft_reorg_interval")] + pub soft_reorg_interval: Duration, + + #[serde(with = "humantime_serde", default = "default_hard_reorg_interval")] + pub hard_reorg_interval: Duration, + #[serde(default)] pub datadog_enabled: bool, @@ -28,6 +34,14 @@ pub struct TxSitterConfig { pub predefined: Option, } +const fn default_soft_reorg_interval() -> Duration { + Duration::from_secs(60) +} + +const fn default_hard_reorg_interval() -> Duration { + Duration::from_secs(60 * 60) +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct Predefined { @@ -148,6 +162,8 @@ mod tests { const WITH_DB_CONNECTION_STRING: &str = indoc! {r#" [service] escalation_interval = "1h" + soft_reorg_interval = "1m" + hard_reorg_interval = "1h" datadog_enabled = false statsd_enabled = false @@ -165,6 +181,8 @@ mod tests { const WITH_DB_PARTS: &str = indoc! {r#" [service] escalation_interval = "1h" + soft_reorg_interval = "1m" + hard_reorg_interval = "1h" datadog_enabled = false statsd_enabled = false @@ -188,6 +206,8 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + soft_reorg_interval: default_soft_reorg_interval(), + hard_reorg_interval: default_hard_reorg_interval(), datadog_enabled: false, statsd_enabled: false, predefined: None, @@ -214,6 +234,8 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + soft_reorg_interval: default_soft_reorg_interval(), + hard_reorg_interval: default_hard_reorg_interval(), datadog_enabled: false, statsd_enabled: false, predefined: None, diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 76d5d3c..3a8c51f 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -39,7 +39,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { // Min increase of 20% on the priority fee required for a replacement tx let factor = U256::from(100); let increased_gas_price_percentage = - factor + U256::from(10 * (1 + escalation)); + factor + U256::from(20 * (1 + escalation)); let max_fee_per_gas_increase = tx.initial_max_fee_per_gas.0 * increased_gas_price_percentage @@ -51,6 +51,13 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { let max_priority_fee_per_gas = max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; + tracing::warn!( + "Initial tx fees are max = {}, priority = {}", + tx.initial_max_fee_per_gas.0, + tx.initial_max_priority_fee_per_gas.0 + ); + tracing::warn!("Escalating with max fee = {max_fee_per_gas} and max priority = {max_priority_fee_per_gas}"); + let eip1559_tx = Eip1559TransactionRequest { from: None, to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), diff --git a/src/tasks/handle_reorgs.rs b/src/tasks/handle_reorgs.rs index 43c5f56..7b9a12d 100644 --- a/src/tasks/handle_reorgs.rs +++ b/src/tasks/handle_reorgs.rs @@ -1,12 +1,7 @@ use std::sync::Arc; -use std::time::Duration; use crate::app::App; -// TODO: Make this configurable -const TIME_BETWEEN_HARD_REORGS_SECONDS: i64 = 60 * 60; // Once every hour -const TIME_BETWEEN_SOFT_REORGS_SECONDS: i64 = 60; // Once every minute - pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { loop { tracing::info!("Handling hard reorgs"); @@ -17,10 +12,7 @@ pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { tracing::info!(id = tx, "Tx hard reorged"); } - tokio::time::sleep(Duration::from_secs( - TIME_BETWEEN_HARD_REORGS_SECONDS as u64, - )) - .await; + tokio::time::sleep(app.config.service.hard_reorg_interval).await; } } @@ -34,9 +26,6 @@ pub async fn handle_soft_reorgs(app: Arc) -> eyre::Result<()> { tracing::info!(id = tx, "Tx soft reorged"); } - tokio::time::sleep(Duration::from_secs( - TIME_BETWEEN_SOFT_REORGS_SECONDS as u64, - )) - .await; + tokio::time::sleep(app.config.service.soft_reorg_interval).await; } } diff --git a/tests/common/anvil_builder.rs b/tests/common/anvil_builder.rs new file mode 100644 index 0000000..ac1fd55 --- /dev/null +++ b/tests/common/anvil_builder.rs @@ -0,0 +1,67 @@ +use std::time::Duration; + +use ethers::providers::Middleware; +use ethers::types::{Eip1559TransactionRequest, U256}; +use ethers::utils::{Anvil, AnvilInstance}; + +use super::prelude::{ + setup_middleware, DEFAULT_ANVIL_ACCOUNT, DEFAULT_ANVIL_BLOCK_TIME, + SECONDARY_ANVIL_PRIVATE_KEY, +}; + +#[derive(Debug, Clone, Default)] +pub struct AnvilBuilder { + pub block_time: Option, + pub port: Option, +} + +impl AnvilBuilder { + pub fn block_time(mut self, block_time: u64) -> Self { + self.block_time = Some(block_time); + self + } + + pub fn port(mut self, port: u16) -> Self { + self.port = Some(port); + self + } + + pub async fn spawn(self) -> eyre::Result { + let mut anvil = Anvil::new(); + + let block_time = if let Some(block_time) = self.block_time { + block_time + } else { + DEFAULT_ANVIL_BLOCK_TIME + }; + anvil = anvil.block_time(block_time); + + if let Some(port) = self.port { + anvil = anvil.port(port); + } + + let anvil = anvil.spawn(); + + let middleware = + setup_middleware(anvil.endpoint(), SECONDARY_ANVIL_PRIVATE_KEY) + .await?; + + // Wait for the chain to start and produce at least one block + tokio::time::sleep(Duration::from_secs(block_time)).await; + + // We need to seed some transactions so we can get fee estimates on the first block + middleware + .send_transaction( + Eip1559TransactionRequest { + to: Some(DEFAULT_ANVIL_ACCOUNT.into()), + value: Some(U256::from(100u64)), + ..Default::default() + }, + None, + ) + .await? + .await?; + + Ok(anvil) + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 033ea32..3d5f121 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,41 +1,43 @@ #![allow(dead_code)] // Needed because this module is imported as module by many test crates -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::sync::Arc; -use std::time::Duration; use ethers::core::k256::ecdsa::SigningKey; use ethers::middleware::SignerMiddleware; use ethers::providers::{Http, Middleware, Provider}; use ethers::signers::{LocalWallet, Signer}; -use ethers::types::{Address, Eip1559TransactionRequest, H160, U256}; -use fake_rpc::DoubleAnvil; +use ethers::types::{Address, H160}; use postgres_docker_utils::DockerContainerGuard; -use tokio::task::JoinHandle; use tracing::level_filters::LevelFilter; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; -use tx_sitter::client::TxSitterClient; -use tx_sitter::config::{ - Config, DatabaseConfig, KeysConfig, LocalKeysConfig, Predefined, - PredefinedNetwork, PredefinedRelayer, ServerConfig, TxSitterConfig, -}; -use tx_sitter::service::Service; pub type AppMiddleware = SignerMiddleware>, LocalWallet>; +mod anvil_builder; +mod service_builder; + +pub use self::anvil_builder::AnvilBuilder; +pub use self::service_builder::ServiceBuilder; + #[allow(unused_imports)] pub mod prelude { pub use std::time::Duration; + pub use ethers::prelude::{Http, Provider}; pub use ethers::providers::Middleware; - pub use ethers::types::{Eip1559TransactionRequest, U256}; + pub use ethers::types::{Eip1559TransactionRequest, H256, U256}; pub use ethers::utils::parse_units; + pub use futures::stream::FuturesUnordered; + pub use futures::StreamExt; + pub use tx_sitter::api_key::ApiKey; + pub use tx_sitter::client::TxSitterClient; pub use tx_sitter::server::routes::relayer::{ - CreateRelayerRequest, CreateRelayerResponse, + CreateApiKeyResponse, CreateRelayerRequest, CreateRelayerResponse, }; pub use tx_sitter::server::routes::transaction::SendTxRequest; + pub use url::Url; pub use super::*; } @@ -57,26 +59,10 @@ pub const ARBITRARY_ADDRESS: Address = H160(hex_literal::hex!( )); pub const DEFAULT_ANVIL_CHAIN_ID: u64 = 31337; +pub const DEFAULT_ANVIL_BLOCK_TIME: u64 = 2; pub const DEFAULT_RELAYER_ID: &str = "1b908a34-5dc1-4d2d-a146-5eb46e975830"; -pub struct DoubleAnvilHandle { - pub double_anvil: Arc, - ws_addr: String, - local_addr: SocketAddr, - server_handle: JoinHandle>, -} - -impl DoubleAnvilHandle { - pub fn local_addr(&self) -> String { - self.local_addr.to_string() - } - - pub fn ws_addr(&self) -> String { - self.ws_addr.clone() - } -} - pub fn setup_tracing() { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().pretty().compact()) @@ -84,7 +70,7 @@ pub fn setup_tracing() { EnvFilter::builder() .with_default_directive(LevelFilter::INFO.into()) // Logging from fake_rpc can get very messy so we set it to warn only - .parse_lossy("info,fake_rpc=warn"), + .parse_lossy("info,tx_sitter=debug,fake_rpc=warn"), ) .init(); } @@ -97,94 +83,6 @@ pub async fn setup_db() -> eyre::Result<(String, DockerContainerGuard)> { Ok((url, db_container)) } -pub async fn setup_double_anvil() -> eyre::Result { - let (double_anvil, server) = fake_rpc::serve(0).await; - - let local_addr = server.local_addr(); - - let server_handle = tokio::spawn(async move { - server.await?; - Ok(()) - }); - - let middleware = setup_middleware( - format!("http://{local_addr}"), - SECONDARY_ANVIL_PRIVATE_KEY, - ) - .await?; - - // We need to seed some transactions so we can get fee estimates on the first block - middleware - .send_transaction( - Eip1559TransactionRequest { - to: Some(DEFAULT_ANVIL_ACCOUNT.into()), - value: Some(U256::from(100u64)), - ..Default::default() - }, - None, - ) - .await? - .await?; - - let ws_addr = double_anvil.ws_endpoint().await; - - Ok(DoubleAnvilHandle { - double_anvil, - ws_addr, - local_addr, - server_handle, - }) -} - -pub async fn setup_service( - anvil_handle: &DoubleAnvilHandle, - db_connection_url: &str, - escalation_interval: Duration, -) -> eyre::Result<(Service, TxSitterClient)> { - let rpc_url = anvil_handle.local_addr(); - - let anvil_private_key = hex::encode(DEFAULT_ANVIL_PRIVATE_KEY); - - let config = Config { - service: TxSitterConfig { - escalation_interval, - datadog_enabled: false, - statsd_enabled: false, - predefined: Some(Predefined { - network: PredefinedNetwork { - chain_id: DEFAULT_ANVIL_CHAIN_ID, - name: "Anvil".to_string(), - http_rpc: format!("http://{}", rpc_url), - ws_rpc: anvil_handle.ws_addr(), - }, - relayer: PredefinedRelayer { - name: "Anvil".to_string(), - id: DEFAULT_RELAYER_ID.to_string(), - key_id: anvil_private_key, - chain_id: DEFAULT_ANVIL_CHAIN_ID, - }, - }), - }, - server: ServerConfig { - host: SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - 0, - )), - username: None, - password: None, - }, - database: DatabaseConfig::connection_string(db_connection_url), - keys: KeysConfig::Local(LocalKeysConfig::default()), - }; - - let service = Service::new(config).await?; - - let client = - TxSitterClient::new(format!("http://{}", service.local_addr())); - - Ok((service, client)) -} - pub async fn setup_middleware( rpc_url: impl AsRef, private_key: &[u8], diff --git a/tests/common/service_builder.rs b/tests/common/service_builder.rs new file mode 100644 index 0000000..492e538 --- /dev/null +++ b/tests/common/service_builder.rs @@ -0,0 +1,96 @@ +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; +use std::time::Duration; + +use ethers::utils::AnvilInstance; +use tx_sitter::client::TxSitterClient; +use tx_sitter::config::{ + Config, DatabaseConfig, KeysConfig, LocalKeysConfig, Predefined, + PredefinedNetwork, PredefinedRelayer, ServerConfig, TxSitterConfig, +}; +use tx_sitter::service::Service; + +use super::prelude::{ + DEFAULT_ANVIL_CHAIN_ID, DEFAULT_ANVIL_PRIVATE_KEY, DEFAULT_RELAYER_ID, +}; + +pub struct ServiceBuilder { + escalation_interval: Duration, + soft_reorg_interval: Duration, + hard_reorg_interval: Duration, +} + +impl Default for ServiceBuilder { + fn default() -> Self { + Self { + escalation_interval: Duration::from_secs(30), + soft_reorg_interval: Duration::from_secs(45), + hard_reorg_interval: Duration::from_secs(60), + } + } +} + +impl ServiceBuilder { + pub fn escalation_interval(mut self, interval: Duration) -> Self { + self.escalation_interval = interval; + self + } + + pub fn soft_reorg_interval(mut self, interval: Duration) -> Self { + self.soft_reorg_interval = interval; + self + } + + pub fn hard_reorg_interval(mut self, interval: Duration) -> Self { + self.hard_reorg_interval = interval; + self + } + + pub async fn build( + self, + anvil: &AnvilInstance, + db_url: &str, + ) -> eyre::Result<(Service, TxSitterClient)> { + let anvil_private_key = hex::encode(DEFAULT_ANVIL_PRIVATE_KEY); + + let config = Config { + service: TxSitterConfig { + escalation_interval: self.escalation_interval, + soft_reorg_interval: self.soft_reorg_interval, + hard_reorg_interval: self.hard_reorg_interval, + datadog_enabled: false, + statsd_enabled: false, + predefined: Some(Predefined { + network: PredefinedNetwork { + chain_id: DEFAULT_ANVIL_CHAIN_ID, + name: "Anvil".to_string(), + http_rpc: anvil.endpoint(), + ws_rpc: anvil.ws_endpoint(), + }, + relayer: PredefinedRelayer { + name: "Anvil".to_string(), + id: DEFAULT_RELAYER_ID.to_string(), + key_id: anvil_private_key, + chain_id: DEFAULT_ANVIL_CHAIN_ID, + }, + }), + }, + server: ServerConfig { + host: SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::new(127, 0, 0, 1), + 0, + )), + username: None, + password: None, + }, + database: DatabaseConfig::connection_string(db_url), + keys: KeysConfig::Local(LocalKeysConfig::default()), + }; + + let service = Service::new(config).await?; + + let client = + TxSitterClient::new(format!("http://{}", service.local_addr())); + + Ok((service, client)) + } +} diff --git a/tests/create_relayer.rs b/tests/create_relayer.rs index e71732a..17a8233 100644 --- a/tests/create_relayer.rs +++ b/tests/create_relayer.rs @@ -2,17 +2,15 @@ mod common; use crate::common::prelude::*; -const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); - #[tokio::test] async fn create_relayer() -> eyre::Result<()> { setup_tracing(); let (db_url, _db_container) = setup_db().await?; - let double_anvil = setup_double_anvil().await?; + let anvil = AnvilBuilder::default().spawn().await?; let (_service, client) = - setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; + ServiceBuilder::default().build(&anvil, &db_url).await?; let CreateRelayerResponse { .. } = client .create_relayer(&CreateRelayerRequest { diff --git a/tests/escalation.rs b/tests/escalation.rs new file mode 100644 index 0000000..cab10c6 --- /dev/null +++ b/tests/escalation.rs @@ -0,0 +1,86 @@ +mod common; + +use crate::common::prelude::*; + +const ESCALATION_INTERVAL: Duration = Duration::from_secs(2); +const ANVIL_BLOCK_TIME: u64 = 6; + +#[tokio::test] +async fn escalation() -> eyre::Result<()> { + setup_tracing(); + + let (db_url, _db_container) = setup_db().await?; + let anvil = AnvilBuilder::default() + .block_time(ANVIL_BLOCK_TIME) + .spawn() + .await?; + + let (_service, client) = ServiceBuilder::default() + .escalation_interval(ESCALATION_INTERVAL) + .build(&anvil, &db_url) + .await?; + + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; + + let provider = setup_provider(anvil.endpoint()).await?; + + // Send a transaction + let value: U256 = parse_units("1", "ether")?.into(); + let tx = client + .send_tx( + &api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) + .await?; + + let initial_tx_hash = get_tx_hash(&client, &api_key, &tx.tx_id).await?; + + await_balance(&provider, value).await?; + let final_tx_hash = get_tx_hash(&client, &api_key, &tx.tx_id).await?; + + assert_ne!( + initial_tx_hash, final_tx_hash, + "Escalation should have occurred" + ); + + Ok(()) +} + +async fn await_balance( + provider: &Provider, + value: U256, +) -> eyre::Result<()> { + for _ in 0..24 { + let balance = provider.get_balance(ARBITRARY_ADDRESS, None).await?; + + if balance == value { + return Ok(()); + } else { + tokio::time::sleep(Duration::from_secs(3)).await; + } + } + + eyre::bail!("Balance not updated in time"); +} + +async fn get_tx_hash( + client: &TxSitterClient, + api_key: &ApiKey, + tx_id: &str, +) -> eyre::Result { + loop { + let tx = client.get_tx(api_key, tx_id).await?; + + if let Some(tx_hash) = tx.tx_hash { + return Ok(tx_hash); + } else { + tokio::time::sleep(Duration::from_secs(3)).await; + } + } +} diff --git a/tests/reorg.rs b/tests/reorg.rs new file mode 100644 index 0000000..a25ae9a --- /dev/null +++ b/tests/reorg.rs @@ -0,0 +1,74 @@ +mod common; + +use crate::common::prelude::*; + +#[tokio::test] +async fn reorg() -> eyre::Result<()> { + setup_tracing(); + + let (db_url, _db_container) = setup_db().await?; + let anvil = AnvilBuilder::default().spawn().await?; + let anvil_port = anvil.port(); + + let (_service, client) = ServiceBuilder::default() + .hard_reorg_interval(Duration::from_secs(2)) + .build(&anvil, &db_url) + .await?; + + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; + + let provider = setup_provider(anvil.endpoint()).await?; + + // Send a transaction + let value: U256 = parse_units("1", "ether")?.into(); + client + .send_tx( + &api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) + .await?; + + await_balance(&provider, value).await?; + + // Drop anvil to simulate a reorg + tracing::warn!("Dropping anvil & restarting at port {anvil_port}"); + drop(anvil); + + let anvil = AnvilBuilder::default().port(anvil_port).spawn().await?; + let provider = setup_provider(anvil.endpoint()).await?; + + await_balance(&provider, value).await?; + + Ok(()) +} + +async fn await_balance( + provider: &Provider, + value: U256, +) -> eyre::Result<()> { + for _ in 0..24 { + let balance = match provider.get_balance(ARBITRARY_ADDRESS, None).await + { + Ok(balance) => balance, + Err(err) => { + tracing::warn!("Error getting balance: {:?}", err); + tokio::time::sleep(Duration::from_secs(3)).await; + continue; + } + }; + + if balance == value { + return Ok(()); + } else { + tokio::time::sleep(Duration::from_secs(3)).await; + } + } + + eyre::bail!("Balance not updated in time"); +} diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index c4ab6fe..f646210 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -1,22 +1,16 @@ mod common; -use ethers::prelude::*; -use tx_sitter::server::routes::relayer::CreateApiKeyResponse; -use url::Url; - use crate::common::prelude::*; -const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); - #[tokio::test] async fn rpc_access() -> eyre::Result<()> { setup_tracing(); let (db_url, _db_container) = setup_db().await?; - let double_anvil = setup_double_anvil().await?; + let anvil = AnvilBuilder::default().spawn().await?; let (service, client) = - setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; + ServiceBuilder::default().build(&anvil, &db_url).await?; let CreateApiKeyResponse { api_key } = client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 1e0226c..4325423 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -1,28 +1,21 @@ mod common; -use futures::stream::FuturesUnordered; -use futures::StreamExt; -use tx_sitter::server::routes::relayer::CreateApiKeyResponse; - use crate::common::prelude::*; -const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); - #[tokio::test] async fn send_many_txs() -> eyre::Result<()> { setup_tracing(); let (db_url, _db_container) = setup_db().await?; - let double_anvil = setup_double_anvil().await?; + let anvil = AnvilBuilder::default().spawn().await?; let (_service, client) = - setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; + ServiceBuilder::default().build(&anvil, &db_url).await?; let CreateApiKeyResponse { api_key } = client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; - let provider = - setup_provider(format!("http://{}", double_anvil.local_addr())).await?; + let provider = setup_provider(anvil.endpoint()).await?; // Send a transaction let value: U256 = parse_units("10", "ether")?.into(); diff --git a/tests/send_tx.rs b/tests/send_tx.rs index b4236ca..1a8fe62 100644 --- a/tests/send_tx.rs +++ b/tests/send_tx.rs @@ -1,26 +1,20 @@ mod common; -use tx_sitter::server::routes::relayer::CreateApiKeyResponse; - use crate::common::prelude::*; -const ESCALATION_INTERVAL: Duration = Duration::from_secs(30); - #[tokio::test] async fn send_tx() -> eyre::Result<()> { setup_tracing(); let (db_url, _db_container) = setup_db().await?; - let double_anvil = setup_double_anvil().await?; + let anvil = AnvilBuilder::default().spawn().await?; let (_service, client) = - setup_service(&double_anvil, &db_url, ESCALATION_INTERVAL).await?; - + ServiceBuilder::default().build(&anvil, &db_url).await?; let CreateApiKeyResponse { api_key } = client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; - let provider = - setup_provider(format!("http://{}", double_anvil.local_addr())).await?; + let provider = setup_provider(anvil.endpoint()).await?; // Send a transaction let value: U256 = parse_units("1", "ether")?.into(); From 17560e8940076a82ace702ef388e288436e95d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 27 Dec 2023 18:25:51 +0100 Subject: [PATCH 117/135] Further dockerization support (#12) * Predefined api key for dockerization * Fix Dockerfile --- Dockerfile | 3 --- config.toml | 1 + src/api_key.rs | 11 ++++++++++- src/config.rs | 3 +++ src/db.rs | 2 +- src/server/routes/relayer.rs | 4 ++-- src/service.rs | 9 +++++++++ tests/common/service_builder.rs | 3 +++ 8 files changed, 29 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae9dd47..b1b104f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,17 +24,14 @@ RUN rustup component add cargo # TODO: Hacky but it works RUN mkdir -p ./src RUN mkdir -p ./crates/postgres-docker-utils/src -RUN mkdir -p ./crates/fake-rpc/src # Copy only Cargo.toml for better caching COPY ./Cargo.toml ./Cargo.toml COPY ./Cargo.lock ./Cargo.lock COPY ./crates/postgres-docker-utils/Cargo.toml ./crates/postgres-docker-utils/Cargo.toml -COPY ./crates/fake-rpc/Cargo.toml ./crates/fake-rpc/Cargo.toml RUN echo "fn main() {}" > ./src/main.rs RUN echo "fn main() {}" > ./crates/postgres-docker-utils/src/main.rs -RUN echo "fn main() {}" > ./crates/fake-rpc/src/main.rs # Prebuild dependencies RUN cargo fetch diff --git a/config.toml b/config.toml index cc2e14e..3977e73 100644 --- a/config.toml +++ b/config.toml @@ -12,6 +12,7 @@ ws_url = "ws://127.0.0.1:8545" id = "1b908a34-5dc1-4d2d-a146-5eb46e975830" chain_id = 31337 key_id = "d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa" +api_key = "G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH" [server] host = "127.0.0.1:3000" diff --git a/src/api_key.rs b/src/api_key.rs index f180a83..a1dca7a 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -13,7 +13,16 @@ pub struct ApiKey { } impl ApiKey { - pub fn new(relayer_id: impl ToString) -> Self { + pub fn new(relayer_id: impl ToString, key: [u8; 32]) -> Self { + let relayer_id = relayer_id.to_string(); + + Self { + relayer_id, + api_key: key, + } + } + + pub fn random(relayer_id: impl ToString) -> Self { let relayer_id = relayer_id.to_string(); let mut api_key = [0u8; 32]; diff --git a/src/config.rs b/src/config.rs index 2869c1e..6d2ed1c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,6 +3,8 @@ use std::time::Duration; use serde::{Deserialize, Serialize}; +use crate::api_key::ApiKey; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct Config { @@ -65,6 +67,7 @@ pub struct PredefinedRelayer { pub name: String, pub key_id: String, pub chain_id: u64, + pub api_key: ApiKey, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/db.rs b/src/db.rs index 40457f1..0504c67 100644 --- a/src/db.rs +++ b/src/db.rs @@ -935,7 +935,7 @@ impl Database { Ok(items.into_iter().map(|(x,)| x as u64).collect()) } - pub async fn save_api_key( + pub async fn create_api_key( &self, relayer_id: &str, api_key_hash: [u8; 32], diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index d262cdc..88d923e 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -158,10 +158,10 @@ pub async fn create_relayer_api_key( State(app): State>, Path(relayer_id): Path, ) -> Result, ApiError> { - let api_key = ApiKey::new(&relayer_id); + let api_key = ApiKey::random(&relayer_id); app.db - .save_api_key(&relayer_id, api_key.api_key_hash()) + .create_api_key(&relayer_id, api_key.api_key_hash()) .await?; Ok(Json(CreateApiKeyResponse { api_key })) diff --git a/src/service.rs b/src/service.rs index 37a71c1..f08b663 100644 --- a/src/service.rs +++ b/src/service.rs @@ -94,6 +94,8 @@ async fn initialize_predefined_values( return Ok(()); }; + tracing::warn!("Running with predefined values is not recommended in a production environment"); + app.db .create_network( predefined.network.chain_id, @@ -121,5 +123,12 @@ async fn initialize_predefined_values( ) .await?; + app.db + .create_api_key( + &predefined.relayer.api_key.relayer_id, + predefined.relayer.api_key.api_key_hash(), + ) + .await?; + Ok(()) } diff --git a/tests/common/service_builder.rs b/tests/common/service_builder.rs index 492e538..b76e92e 100644 --- a/tests/common/service_builder.rs +++ b/tests/common/service_builder.rs @@ -2,6 +2,7 @@ use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::time::Duration; use ethers::utils::AnvilInstance; +use tx_sitter::api_key::ApiKey; use tx_sitter::client::TxSitterClient; use tx_sitter::config::{ Config, DatabaseConfig, KeysConfig, LocalKeysConfig, Predefined, @@ -71,6 +72,8 @@ impl ServiceBuilder { id: DEFAULT_RELAYER_ID.to_string(), key_id: anvil_private_key, chain_id: DEFAULT_ANVIL_CHAIN_ID, + // TODO: Use this key in tests + api_key: ApiKey::random(DEFAULT_RELAYER_ID), }, }), }, From 8a5f5eeb4513e38ebbc8782a2e88e034df97b1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Thu, 28 Dec 2023 19:19:12 +0100 Subject: [PATCH 118/135] Fix owned deserialization (#13) --- src/api_key.rs | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/api_key.rs b/src/api_key.rs index a1dca7a..02d5576 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::str::FromStr; use base64::Engine; @@ -53,7 +54,7 @@ impl<'de> serde::Deserialize<'de> for ApiKey { where D: serde::Deserializer<'de>, { - <&str>::deserialize(deserializer)? + >::deserialize(deserializer)? .parse() .map_err(serde::de::Error::custom) } @@ -105,16 +106,16 @@ mod tests { use super::*; + fn random_api_key() -> ApiKey { + let mut api_key = [0u8; 32]; + OsRng.fill_bytes(&mut api_key); + + ApiKey::new(uuid::Uuid::new_v4().to_string(), api_key) + } + #[test] fn from_to_str() { - let api_key = ApiKey { - relayer_id: uuid::Uuid::new_v4().to_string(), - api_key: { - let mut api_key = [0u8; 32]; - OsRng.fill_bytes(&mut api_key); - api_key - }, - }; + let api_key = random_api_key(); let api_key_str = api_key.to_string(); @@ -124,4 +125,33 @@ mod tests { assert_eq!(api_key, api_key_parsed); } + + #[test] + fn from_to_serde_json() { + let api_key = random_api_key(); + + let api_key_json = serde_json::to_string(&api_key).unwrap(); + + println!("api_key_str = {api_key_json}"); + + let api_key_parsed: ApiKey = + serde_json::from_str(&api_key_json).unwrap(); + + assert_eq!(api_key, api_key_parsed); + } + + #[test] + fn from_to_serde_json_owned() { + let api_key = random_api_key(); + + let api_key_json: serde_json::Value = + serde_json::to_value(&api_key).unwrap(); + + println!("api_key_str = {api_key_json}"); + + let api_key_parsed: ApiKey = + serde_json::from_value(api_key_json).unwrap(); + + assert_eq!(api_key, api_key_parsed); + } } From 0829d6b6f3c494f055124283694b521c35867c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Tue, 9 Jan 2024 20:50:56 +0100 Subject: [PATCH 119/135] Allow disabling relayers (#14) * Update DB * Integrate with db code * Escalate per relayer & don't if relayer disabled * clippy --- db/migrations/002_relayers_table_update.sql | 5 + manual_test_kms.nu | 2 +- src/broadcast_utils.rs | 15 +- src/db.rs | 23 ++- src/tasks/broadcast.rs | 6 +- src/tasks/escalate.rs | 210 ++++++++++++-------- src/types.rs | 19 +- 7 files changed, 176 insertions(+), 104 deletions(-) create mode 100644 db/migrations/002_relayers_table_update.sql diff --git a/db/migrations/002_relayers_table_update.sql b/db/migrations/002_relayers_table_update.sql new file mode 100644 index 0000000..24e3053 --- /dev/null +++ b/db/migrations/002_relayers_table_update.sql @@ -0,0 +1,5 @@ +ALTER TABLE relayers +RENAME COLUMN gas_limits TO gas_price_limits; + +ALTER TABLE relayers +ADD COLUMN enabled BOOL NOT NULL DEFAULT TRUE; diff --git a/manual_test_kms.nu b/manual_test_kms.nu index e426991..7ad746a 100644 --- a/manual_test_kms.nu +++ b/manual_test_kms.nu @@ -21,7 +21,7 @@ echo "Creating relayer" let relayer = http post -t application/json $"($txSitter)/1/admin/relayer" { "name": "My Relayer", "chainId": 11155111 } http post -t application/json $"($txSitter)/1/admin/relayer/($relayer.relayerId)" { - gasLimits: [ + gasPriceLimits: [ { chainId: 11155111, value: "0x123" } ] } diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index 35c9318..3de31b5 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -3,6 +3,7 @@ use eyre::ContextCompat; use self::gas_estimation::FeesEstimate; use crate::app::App; +use crate::types::RelayerInfo; pub mod gas_estimation; @@ -21,11 +22,19 @@ pub fn calculate_gas_fees_from_estimates( pub async fn should_send_transaction( app: &App, - relayer_id: &str, + relayer: &RelayerInfo, ) -> eyre::Result { - let relayer = app.db.get_relayer(relayer_id).await?; + if !relayer.enabled { + tracing::warn!( + relayer_id = relayer.id, + chain_id = relayer.chain_id, + "Relayer is disabled, skipping transactions broadcast" + ); + + return Ok(false); + } - for gas_limit in &relayer.gas_limits.0 { + for gas_limit in &relayer.gas_price_limits.0 { let chain_fees = app .db .get_latest_block_fees_by_chain_id(relayer.chain_id) diff --git a/src/db.rs b/src/db.rs index 0504c67..d935abc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -101,16 +101,16 @@ impl Database { .await?; } - if let Some(gas_limits) = &update.gas_limits { + if let Some(gas_price_limits) = &update.gas_price_limits { sqlx::query( r#" UPDATE relayers - SET gas_limits = $2 + SET gas_price_limits = $2 WHERE id = $1 "#, ) .bind(id) - .bind(Json(gas_limits)) + .bind(Json(gas_price_limits)) .execute(tx.as_mut()) .await?; } @@ -132,7 +132,8 @@ impl Database { nonce, current_nonce, max_inflight_txs, - gas_limits + gas_price_limits, + enabled FROM relayers "#, ) @@ -152,7 +153,8 @@ impl Database { nonce, current_nonce, max_inflight_txs, - gas_limits + gas_price_limits, + enabled FROM relayers WHERE id = $1 "#, @@ -1090,7 +1092,7 @@ mod tests { use super::*; use crate::db::data::U256Wrapper; - use crate::types::RelayerGasLimit; + use crate::types::RelayerGasPriceLimit; async fn setup_db() -> eyre::Result<(Database, DockerContainerGuard)> { let db_container = postgres_docker_utils::setup().await?; @@ -1230,17 +1232,18 @@ mod tests { assert_eq!(relayer.nonce, 0); assert_eq!(relayer.current_nonce, 0); assert_eq!(relayer.max_inflight_txs, 5); - assert_eq!(relayer.gas_limits.0, vec![]); + assert_eq!(relayer.gas_price_limits.0, vec![]); db.update_relayer( relayer_id, &RelayerUpdate { relayer_name: None, max_inflight_txs: Some(10), - gas_limits: Some(vec![RelayerGasLimit { + gas_price_limits: Some(vec![RelayerGasPriceLimit { chain_id: 1, value: U256Wrapper(U256::from(10_123u64)), }]), + enabled: None, }, ) .await?; @@ -1256,8 +1259,8 @@ mod tests { assert_eq!(relayer.current_nonce, 0); assert_eq!(relayer.max_inflight_txs, 10); assert_eq!( - relayer.gas_limits.0, - vec![RelayerGasLimit { + relayer.gas_price_limits.0, + vec![RelayerGasPriceLimit { chain_id: 1, value: U256Wrapper(U256::from(10_123u64)), }] diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 7f33f21..ce1f44a 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -44,14 +44,16 @@ async fn broadcast_relayer_txs( app: &App, relayer_id: String, txs: Vec, -) -> Result<(), eyre::Error> { +) -> eyre::Result<()> { if txs.is_empty() { return Ok(()); } tracing::info!(relayer_id, num_txs = txs.len(), "Broadcasting relayer txs"); - if !should_send_transaction(app, &relayer_id).await? { + let relayer = app.db.get_relayer(&relayer_id).await?; + + if !should_send_transaction(app, &relayer).await? { tracing::warn!( relayer_id = relayer_id, "Skipping transaction broadcasts" diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 3a8c51f..7d15ac1 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::Arc; use ethers::providers::Middleware; @@ -5,9 +6,12 @@ use ethers::types::transaction::eip2718::TypedTransaction; use ethers::types::transaction::eip2930::AccessList; use ethers::types::{Address, Eip1559TransactionRequest, NameOrAddress, U256}; use eyre::ContextCompat; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use crate::app::App; use crate::broadcast_utils::should_send_transaction; +use crate::db::TxForEscalation; pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { loop { @@ -16,90 +20,136 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { .get_txs_for_escalation(app.config.service.escalation_interval) .await?; - for tx in txs_for_escalation { - tracing::info!(id = tx.id, tx.escalation_count, "Escalating tx"); + let txs_for_escalation = split_txs_per_relayer(txs_for_escalation); - if !should_send_transaction(&app, &tx.relayer_id).await? { - tracing::warn!(id = tx.id, "Skipping transaction broadcast"); - continue; - } + let mut futures = FuturesUnordered::new(); - let escalation = tx.escalation_count + 1; - - let middleware = app - .signer_middleware(tx.chain_id, tx.key_id.clone()) - .await?; - - let fees = app - .db - .get_latest_block_fees_by_chain_id(tx.chain_id) - .await? - .context("Missing block")?; - - // Min increase of 20% on the priority fee required for a replacement tx - let factor = U256::from(100); - let increased_gas_price_percentage = - factor + U256::from(20 * (1 + escalation)); - - let max_fee_per_gas_increase = tx.initial_max_fee_per_gas.0 - * increased_gas_price_percentage - / factor; - - let max_fee_per_gas = - tx.initial_max_fee_per_gas.0 + max_fee_per_gas_increase; - - let max_priority_fee_per_gas = - max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; - - tracing::warn!( - "Initial tx fees are max = {}, priority = {}", - tx.initial_max_fee_per_gas.0, - tx.initial_max_priority_fee_per_gas.0 - ); - tracing::warn!("Escalating with max fee = {max_fee_per_gas} and max priority = {max_priority_fee_per_gas}"); - - let eip1559_tx = Eip1559TransactionRequest { - from: None, - to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), - gas: Some(tx.gas_limit.0), - value: Some(tx.value.0), - data: Some(tx.data.into()), - nonce: Some(tx.nonce.into()), - access_list: AccessList::default(), - max_priority_fee_per_gas: Some(max_priority_fee_per_gas), - max_fee_per_gas: Some(max_fee_per_gas), - chain_id: Some(tx.chain_id.into()), - }; - - let pending_tx = middleware - .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await; - - let pending_tx = match pending_tx { - Ok(pending_tx) => { - tracing::info!(?pending_tx, "Tx sent successfully"); - pending_tx - } - Err(err) => { - tracing::error!(error = ?err, "Failed to send tx"); - continue; - } - }; - - let tx_hash = pending_tx.tx_hash(); - - app.db - .escalate_tx( - &tx.id, - tx_hash, - max_fee_per_gas, - max_priority_fee_per_gas, - ) - .await?; - - tracing::info!(id = ?tx.id, hash = ?tx_hash, "Tx escalated"); + for (relayer_id, txs) in txs_for_escalation { + futures.push(escalate_relayer_txs(&app, relayer_id, txs)); + } + + while let Some(result) = futures.next().await { + if let Err(err) = result { + tracing::error!(error = ?err, "Failed escalating txs"); + } } tokio::time::sleep(app.config.service.escalation_interval).await; } } + +async fn escalate_relayer_txs( + app: &App, + relayer_id: String, + txs: Vec, +) -> eyre::Result<()> { + let relayer = app.db.get_relayer(&relayer_id).await?; + + for tx in txs { + tracing::info!(id = tx.id, tx.escalation_count, "Escalating tx"); + + if !should_send_transaction(app, &relayer).await? { + tracing::warn!(id = tx.id, "Skipping transaction broadcast"); + + return Ok(()); + } + + let escalation = tx.escalation_count + 1; + + let middleware = app + .signer_middleware(tx.chain_id, tx.key_id.clone()) + .await?; + + let fees = app + .db + .get_latest_block_fees_by_chain_id(tx.chain_id) + .await? + .context("Missing block")?; + + // Min increase of 20% on the priority fee required for a replacement tx + let factor = U256::from(100); + let increased_gas_price_percentage = + factor + U256::from(20 * (1 + escalation)); + + let max_fee_per_gas_increase = tx.initial_max_fee_per_gas.0 + * increased_gas_price_percentage + / factor; + + let max_fee_per_gas = + tx.initial_max_fee_per_gas.0 + max_fee_per_gas_increase; + + let max_priority_fee_per_gas = + max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; + + tracing::warn!( + "Initial tx fees are max = {}, priority = {}", + tx.initial_max_fee_per_gas.0, + tx.initial_max_priority_fee_per_gas.0 + ); + tracing::warn!("Escalating with max fee = {max_fee_per_gas} and max priority = {max_priority_fee_per_gas}"); + + let eip1559_tx = Eip1559TransactionRequest { + from: None, + to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), + gas: Some(tx.gas_limit.0), + value: Some(tx.value.0), + data: Some(tx.data.into()), + nonce: Some(tx.nonce.into()), + access_list: AccessList::default(), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + max_fee_per_gas: Some(max_fee_per_gas), + chain_id: Some(tx.chain_id.into()), + }; + + let pending_tx = middleware + .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) + .await; + + let pending_tx = match pending_tx { + Ok(pending_tx) => { + tracing::info!(?pending_tx, "Tx sent successfully"); + pending_tx + } + Err(err) => { + tracing::error!(error = ?err, "Failed to send tx"); + continue; + } + }; + + let tx_hash = pending_tx.tx_hash(); + + app.db + .escalate_tx( + &tx.id, + tx_hash, + max_fee_per_gas, + max_priority_fee_per_gas, + ) + .await?; + + tracing::info!(id = ?tx.id, hash = ?tx_hash, "Tx escalated"); + } + + Ok(()) +} + +fn split_txs_per_relayer( + txs: Vec, +) -> HashMap> { + let mut txs_per_relayer = HashMap::new(); + + for tx in txs { + let relayer_id = tx.relayer_id.clone(); + + let txs_for_relayer = + txs_per_relayer.entry(relayer_id).or_insert_with(Vec::new); + + txs_for_relayer.push(tx); + } + + for (_, txs) in txs_per_relayer.iter_mut() { + txs.sort_by_key(|tx| tx.escalation_count); + } + + txs_per_relayer +} diff --git a/src/types.rs b/src/types.rs index 47fd120..b349d08 100644 --- a/src/types.rs +++ b/src/types.rs @@ -42,7 +42,8 @@ pub struct RelayerInfo { pub current_nonce: u64, #[sqlx(try_from = "i64")] pub max_inflight_txs: u64, - pub gas_limits: Json>, + pub gas_price_limits: Json>, + pub enabled: bool, } #[derive(Deserialize, Serialize, Debug, Clone, Default)] @@ -50,17 +51,17 @@ pub struct RelayerInfo { pub struct RelayerUpdate { #[serde(default)] pub relayer_name: Option, - #[serde(default)] pub max_inflight_txs: Option, - #[serde(default)] - pub gas_limits: Option>, + pub gas_price_limits: Option>, + #[serde(default)] + pub enabled: Option, } #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct RelayerGasLimit { +pub struct RelayerGasPriceLimit { pub value: U256Wrapper, pub chain_id: i64, } @@ -82,10 +83,11 @@ mod tests { nonce: 0, current_nonce: 0, max_inflight_txs: 0, - gas_limits: Json(vec![RelayerGasLimit { + gas_price_limits: Json(vec![RelayerGasPriceLimit { value: U256Wrapper(U256::zero()), chain_id: 1, }]), + enabled: true, }; let json = serde_json::to_string_pretty(&info).unwrap(); @@ -100,12 +102,13 @@ mod tests { "nonce": 0, "currentNonce": 0, "maxInflightTxs": 0, - "gasLimits": [ + "gasPriceLimits": [ { "value": "0x0", "chainId": 1 } - ] + ], + "enabled": true } "#}; From f324c530a15135e61c477d550c4c4f39ef680a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 10 Jan 2024 15:06:53 +0100 Subject: [PATCH 120/135] Improve-logging (#15) * Improve logging * Clippy & fmt * Minor improvements * Remove redundant comment --- src/broadcast_utils.rs | 3 +- src/db.rs | 48 ++++++++++++----------- src/server/routes/transaction.rs | 2 +- src/task_runner.rs | 6 +-- src/tasks/broadcast.rs | 65 +++++++++++++++++++------------- src/tasks/escalate.rs | 53 +++++++++++++++----------- src/tasks/handle_reorgs.rs | 4 +- src/tasks/index.rs | 31 +++++++++------ 8 files changed, 124 insertions(+), 88 deletions(-) diff --git a/src/broadcast_utils.rs b/src/broadcast_utils.rs index 3de31b5..f1d791d 100644 --- a/src/broadcast_utils.rs +++ b/src/broadcast_utils.rs @@ -20,7 +20,7 @@ pub fn calculate_gas_fees_from_estimates( (max_fee_per_gas, max_priority_fee_per_gas) } -pub async fn should_send_transaction( +pub async fn should_send_relayer_transactions( app: &App, relayer: &RelayerInfo, ) -> eyre::Result { @@ -43,6 +43,7 @@ pub async fn should_send_transaction( if chain_fees.gas_price > gas_limit.value.0 { tracing::warn!( + relayer_id = relayer.id, chain_id = relayer.chain_id, gas_price = ?chain_fees.gas_price, gas_limit = ?gas_limit.value.0, diff --git a/src/db.rs b/src/db.rs index d935abc..43f9b9e 100644 --- a/src/db.rs +++ b/src/db.rs @@ -14,9 +14,7 @@ use crate::types::{RelayerInfo, RelayerUpdate, TransactionPriority}; pub mod data; -use self::data::{ - AddressWrapper, BlockFees, H256Wrapper, NetworkStats, ReadTxData, RpcKind, -}; +use self::data::{BlockFees, H256Wrapper, NetworkStats, ReadTxData, RpcKind}; pub use self::data::{TxForEscalation, TxStatus, UnsentTx}; // Statically link in migration files @@ -141,6 +139,32 @@ impl Database { .await?) } + pub async fn get_relayers_by_chain_id( + &self, + chain_id: u64, + ) -> eyre::Result> { + Ok(sqlx::query_as( + r#" + SELECT + id, + name, + chain_id, + key_id, + address, + nonce, + current_nonce, + max_inflight_txs, + gas_price_limits, + enabled + FROM relayers + WHERE chain_id = $1 + "#, + ) + .bind(chain_id as i64) + .fetch_all(&self.pool) + .await?) + } + pub async fn get_relayer(&self, id: &str) -> eyre::Result { Ok(sqlx::query_as( r#" @@ -781,24 +805,6 @@ impl Database { .await?) } - pub async fn get_relayer_addresses( - &self, - chain_id: u64, - ) -> eyre::Result> { - let items: Vec<(AddressWrapper,)> = sqlx::query_as( - r#" - SELECT address - FROM relayers - WHERE chain_id = $1 - "#, - ) - .bind(chain_id as i64) - .fetch_all(&self.pool) - .await?; - - Ok(items.into_iter().map(|(wrapper,)| wrapper.0).collect()) - } - pub async fn update_relayer_nonce( &self, chain_id: u64, diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 33f7d9e..75dceec 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -102,7 +102,7 @@ pub async fn send_tx( ) .await?; - tracing::info!(id = tx_id, "Tx created"); + tracing::info!(tx_id, "Transaction created"); Ok(Json(SendTxResponse { tx_id })) } diff --git a/src/task_runner.rs b/src/task_runner.rs index 2e4e096..f5b4367 100644 --- a/src/task_runner.rs +++ b/src/task_runner.rs @@ -33,12 +33,12 @@ where let mut failures = vec![]; loop { - tracing::info!(label, "Running task"); + tracing::info!(task_label = label, "Running task"); let result = task(app.clone()).await; if let Err(err) = result { - tracing::error!(label, error = ?err, "Task failed"); + tracing::error!(task_label = label, error = ?err, "Task failed"); failures.push(Instant::now()); let backoff = determine_backoff(&failures); @@ -47,7 +47,7 @@ where prune_failures(&mut failures); } else { - tracing::info!(label, "Task finished"); + tracing::info!(task_label = label, "Task finished"); break; } } diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index ce1f44a..673c02f 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -13,14 +13,17 @@ use itertools::Itertools; use crate::app::App; use crate::broadcast_utils::{ - calculate_gas_fees_from_estimates, should_send_transaction, + calculate_gas_fees_from_estimates, should_send_relayer_transactions, }; use crate::db::UnsentTx; +const NO_TXS_SLEEP_DURATION: Duration = Duration::from_secs(2); + pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { loop { - // Get all unsent txs and broadcast let txs = app.db.get_unsent_txs().await?; + let num_txs = txs.len(); + let txs_by_relayer = sort_txs_by_relayer(txs); let mut futures = FuturesUnordered::new(); @@ -31,11 +34,13 @@ pub async fn broadcast_txs(app: Arc) -> eyre::Result<()> { while let Some(result) = futures.next().await { if let Err(err) = result { - tracing::error!(error = ?err, "Failed broadcasting txs"); + tracing::error!(error = ?err, "Failed broadcasting transactions"); } } - tokio::time::sleep(Duration::from_secs(1)).await; + if num_txs == 0 { + tokio::time::sleep(NO_TXS_SLEEP_DURATION).await; + } } } @@ -49,21 +54,22 @@ async fn broadcast_relayer_txs( return Ok(()); } - tracing::info!(relayer_id, num_txs = txs.len(), "Broadcasting relayer txs"); - let relayer = app.db.get_relayer(&relayer_id).await?; - if !should_send_transaction(app, &relayer).await? { - tracing::warn!( - relayer_id = relayer_id, - "Skipping transaction broadcasts" - ); + if !should_send_relayer_transactions(app, &relayer).await? { + tracing::warn!(relayer_id = relayer_id, "Skipping relayer broadcasts"); return Ok(()); } + tracing::info!( + relayer_id, + num_txs = txs.len(), + "Broadcasting relayer transactions" + ); + for tx in txs { - tracing::info!(id = tx.id, "Sending tx"); + tracing::info!(tx_id = tx.id, nonce = tx.nonce, "Sending transaction"); let middleware = app .signer_middleware(tx.chain_id, tx.key_id.clone()) @@ -103,16 +109,22 @@ async fn broadcast_relayer_txs( .fill_transaction(&mut typed_transaction, None) .await?; - tracing::debug!(?tx.id, "Simulating tx"); + tracing::debug!(tx_id = tx.id, "Simulating transaction"); // Simulate the transaction match middleware.call(&typed_transaction, None).await { Ok(_) => { - tracing::info!(?tx.id, "Tx simulated successfully"); + tracing::info!( + tx_id = tx.id, + "Transaction simulated successfully" + ); } Err(err) => { - tracing::error!(?tx.id, error = ?err, "Failed to simulate tx"); - continue; + tracing::error!(tx_id = tx.id, error = ?err, "Failed to simulate transaction"); + + // If we fail while broadcasting a tx with nonce `n`, + // it doesn't make sense to broadcast tx with nonce `n + 1` + return Ok(()); } }; @@ -133,24 +145,25 @@ async fn broadcast_relayer_txs( ) .await?; - tracing::debug!(?tx.id, "Sending tx"); + tracing::debug!(tx_id = tx.id, "Sending transaction"); - // TODO: Be smarter about error handling - a tx can fail to be sent - // e.g. because the relayer is out of funds - // but we don't want to retry it forever let pending_tx = middleware.send_raw_transaction(raw_signed_tx).await; - match pending_tx { - Ok(pending_tx) => { - tracing::info!(?pending_tx, "Tx sent successfully"); - } + let pending_tx = match pending_tx { + Ok(pending_tx) => pending_tx, Err(err) => { - tracing::error!(?tx.id, error = ?err, "Failed to send tx"); + tracing::error!(tx_id = tx.id, error = ?err, "Failed to send transaction"); continue; } }; - tracing::info!(id = tx.id, hash = ?tx_hash, "Tx broadcast"); + tracing::info!( + tx_id = tx.id, + tx_nonce = tx.nonce, + tx_hash = ?tx_hash, + ?pending_tx, + "Transaction broadcast" + ); } Ok(()) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 7d15ac1..76cf4da 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -10,7 +10,7 @@ use futures::stream::FuturesUnordered; use futures::StreamExt; use crate::app::App; -use crate::broadcast_utils::should_send_transaction; +use crate::broadcast_utils::should_send_relayer_transactions; use crate::db::TxForEscalation; pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { @@ -46,14 +46,21 @@ async fn escalate_relayer_txs( let relayer = app.db.get_relayer(&relayer_id).await?; for tx in txs { - tracing::info!(id = tx.id, tx.escalation_count, "Escalating tx"); - - if !should_send_transaction(app, &relayer).await? { - tracing::warn!(id = tx.id, "Skipping transaction broadcast"); + if !should_send_relayer_transactions(app, &relayer).await? { + tracing::warn!( + relayer_id = relayer.id, + "Skipping relayer escalations" + ); return Ok(()); } + tracing::info!( + tx_id = tx.id, + escalation_count = tx.escalation_count, + "Escalating transaction" + ); + let escalation = tx.escalation_count + 1; let middleware = app @@ -71,23 +78,17 @@ async fn escalate_relayer_txs( let increased_gas_price_percentage = factor + U256::from(20 * (1 + escalation)); - let max_fee_per_gas_increase = tx.initial_max_fee_per_gas.0 - * increased_gas_price_percentage - / factor; + let initial_max_fee_per_gas = tx.initial_max_fee_per_gas.0; + + let max_fee_per_gas_increase = + initial_max_fee_per_gas * increased_gas_price_percentage / factor; let max_fee_per_gas = - tx.initial_max_fee_per_gas.0 + max_fee_per_gas_increase; + initial_max_fee_per_gas + max_fee_per_gas_increase; let max_priority_fee_per_gas = max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; - tracing::warn!( - "Initial tx fees are max = {}, priority = {}", - tx.initial_max_fee_per_gas.0, - tx.initial_max_priority_fee_per_gas.0 - ); - tracing::warn!("Escalating with max fee = {max_fee_per_gas} and max priority = {max_priority_fee_per_gas}"); - let eip1559_tx = Eip1559TransactionRequest { from: None, to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), @@ -106,18 +107,26 @@ async fn escalate_relayer_txs( .await; let pending_tx = match pending_tx { - Ok(pending_tx) => { - tracing::info!(?pending_tx, "Tx sent successfully"); - pending_tx - } + Ok(pending_tx) => pending_tx, Err(err) => { - tracing::error!(error = ?err, "Failed to send tx"); + tracing::error!(tx_id = tx.id, error = ?err, "Failed to escalate transaction"); continue; } }; let tx_hash = pending_tx.tx_hash(); + tracing::info!( + tx_id = tx.id, + ?tx_hash, + ?initial_max_fee_per_gas, + ?max_fee_per_gas_increase, + ?max_fee_per_gas, + ?max_priority_fee_per_gas, + ?pending_tx, + "Escalated transaction" + ); + app.db .escalate_tx( &tx.id, @@ -127,7 +136,7 @@ async fn escalate_relayer_txs( ) .await?; - tracing::info!(id = ?tx.id, hash = ?tx_hash, "Tx escalated"); + tracing::info!(tx_id = tx.id, "Escalated transaction saved"); } Ok(()) diff --git a/src/tasks/handle_reorgs.rs b/src/tasks/handle_reorgs.rs index 7b9a12d..a18aa15 100644 --- a/src/tasks/handle_reorgs.rs +++ b/src/tasks/handle_reorgs.rs @@ -9,7 +9,7 @@ pub async fn handle_hard_reorgs(app: Arc) -> eyre::Result<()> { let reorged_txs = app.db.handle_hard_reorgs().await?; for tx in reorged_txs { - tracing::info!(id = tx, "Tx hard reorged"); + tracing::info!(tx_id = tx, "Transaction hard reorged"); } tokio::time::sleep(app.config.service.hard_reorg_interval).await; @@ -23,7 +23,7 @@ pub async fn handle_soft_reorgs(app: Arc) -> eyre::Result<()> { let txs = app.db.handle_soft_reorgs().await?; for tx in txs { - tracing::info!(id = tx, "Tx soft reorged"); + tracing::info!(tx_id = tx, "Transaction soft reorged"); } tokio::time::sleep(app.config.service.soft_reorg_interval).await; diff --git a/src/tasks/index.rs b/src/tasks/index.rs index eca129f..dcfe76c 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -12,6 +12,7 @@ use crate::app::App; use crate::broadcast_utils::gas_estimation::{ estimate_percentile_fees, FeesEstimate, }; +use crate::types::RelayerInfo; const BLOCK_FEE_HISTORY_SIZE: usize = 10; const FEE_PERCENTILES: [f64; 5] = [5.0, 25.0, 50.0, 75.0, 95.0]; @@ -48,7 +49,7 @@ pub async fn index_block( ) -> eyre::Result<()> { let block_number = block.number.context("Missing block number")?.as_u64(); - tracing::info!(block_number, "Indexing block"); + tracing::info!(chain_id, block_number, "Indexing block"); let block_timestamp_seconds = block.timestamp.as_u64(); let block_timestamp = @@ -75,17 +76,18 @@ pub async fn index_block( [("chain_id", chain_id.to_string())]; for tx in mined_txs { tracing::info!( - id = tx.0, - hash = ?tx.1, + tx_id = tx.0, + tx_hash = ?tx.1, "Tx mined" ); metrics::increment_counter!("tx_mined", &metric_labels); } - let relayer_addresses = app.db.get_relayer_addresses(chain_id).await?; + let relayers = app.db.get_relayers_by_chain_id(chain_id).await?; + + update_relayer_nonces(&relayers, &app, rpc, chain_id).await?; - update_relayer_nonces(relayer_addresses, &app, rpc, chain_id).await?; Ok(()) } @@ -138,14 +140,18 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { .await?; let Some(latest_block_number) = latest_block_number else { - tracing::info!("No blocks to estimate fees for"); + tracing::info!(chain_id, "No blocks to estimate fees for"); tokio::time::sleep(Duration::from_secs(2)).await; continue; }; - tracing::info!(block_number = latest_block_number, "Estimating fees"); + tracing::info!( + chain_id, + block_number = latest_block_number, + "Estimating fees" + ); let fee_estimates = get_block_fee_estimates(&rpc, latest_block_number) .await @@ -196,30 +202,31 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { } async fn update_relayer_nonces( - relayer_addresses: Vec, + relayers: &[RelayerInfo], app: &Arc, rpc: &Provider, chain_id: u64, ) -> Result<(), eyre::Error> { let mut futures = FuturesUnordered::new(); - for relayer_address in relayer_addresses { + for relayer in relayers { let app = app.clone(); futures.push(async move { let tx_count = - rpc.get_transaction_count(relayer_address, None).await?; + rpc.get_transaction_count(relayer.address.0, None).await?; tracing::info!( + relayer_id = relayer.id, nonce = ?tx_count, - ?relayer_address, + relayer_address = ?relayer.address.0, "Updating relayer nonce" ); app.db .update_relayer_nonce( chain_id, - relayer_address, + relayer.address.0, tx_count.as_u64(), ) .await?; From 9b358f3c93ca16b648e44c1e29ff94a32b0a046d Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 22 Jan 2024 08:32:46 -0800 Subject: [PATCH 121/135] instrument db txs (#19) --- src/db.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/db.rs b/src/db.rs index 43f9b9e..91b4507 100644 --- a/src/db.rs +++ b/src/db.rs @@ -7,6 +7,7 @@ use ethers::types::{Address, H256, U256}; use sqlx::migrate::{MigrateDatabase, Migrator}; use sqlx::types::{BigDecimal, Json}; use sqlx::{Pool, Postgres, Row}; +use tracing::instrument; use crate::broadcast_utils::gas_estimation::FeesEstimate; use crate::config::DatabaseConfig; @@ -39,6 +40,7 @@ impl Database { Ok(Self { pool }) } + #[instrument(skip(self), level = "debug")] pub async fn create_relayer( &self, id: &str, @@ -64,6 +66,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn update_relayer( &self, id: &str, @@ -188,6 +191,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn create_transaction( &self, tx_id: &str, @@ -257,6 +261,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn insert_tx_broadcast( &self, tx_id: &str, @@ -402,6 +407,7 @@ impl Database { Ok(row.try_get::(0)?) } + #[instrument(skip(self), level = "debug")] pub async fn save_block( &self, block_number: u64, @@ -464,6 +470,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn save_block_fees( &self, block_number: u64, @@ -490,6 +497,7 @@ impl Database { } /// Returns a list of soft reorged txs + #[instrument(skip(self), level = "debug", ret)] pub async fn handle_soft_reorgs(&self) -> eyre::Result> { let mut tx = self.pool.begin().await?; @@ -532,6 +540,7 @@ impl Database { } /// Returns a list of hard reorged txs + #[instrument(skip(self), level = "debug", ret)] pub async fn handle_hard_reorgs(&self) -> eyre::Result> { let mut tx = self.pool.begin().await?; @@ -591,6 +600,7 @@ impl Database { /// Marks txs as mined if the associated tx hash is present in a block /// /// returns the tx ids and hashes for all mined txs + #[instrument(skip(self), level = "debug", ret)] pub async fn mine_txs( &self, chain_id: u64, @@ -630,6 +640,7 @@ impl Database { .collect()) } + #[instrument(skip(self), level = "debug")] pub async fn finalize_txs( &self, finalization_timestmap: DateTime, @@ -700,6 +711,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn escalate_tx( &self, tx_id: &str, @@ -805,6 +817,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn update_relayer_nonce( &self, chain_id: u64, @@ -829,6 +842,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn prune_blocks( &self, timestamp: DateTime, @@ -846,6 +860,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn prune_txs( &self, timestamp: DateTime, @@ -868,6 +883,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn create_network( &self, chain_id: u64, @@ -943,6 +959,7 @@ impl Database { Ok(items.into_iter().map(|(x,)| x as u64).collect()) } + #[instrument(skip(self), level = "debug")] pub async fn create_api_key( &self, relayer_id: &str, @@ -1061,6 +1078,7 @@ impl Database { }) } + #[instrument(skip(self), level = "debug")] pub async fn purge_unsent_txs(&self, relayer_id: &str) -> eyre::Result<()> { sqlx::query( r#" From 18da0233149e6a0c547ca1dbb0786e3d67eb45c1 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 22 Jan 2024 08:33:50 -0800 Subject: [PATCH 122/135] Api Key Rework - PRO-467 (#17) * shortened api key and fixed predefined toml * dynamic length api key * Fix for security issue where api keys were being exposed in logs and traces --- config.toml | 12 ++- src/api_key.rs | 166 +++++++++++++++++++++++-------- src/app.rs | 5 +- src/client.rs | 11 +- src/server/routes/relayer.rs | 6 +- src/server/routes/transaction.rs | 12 +-- src/service.rs | 4 +- tests/rpc_access.rs | 7 +- 8 files changed, 161 insertions(+), 62 deletions(-) diff --git a/config.toml b/config.toml index 3977e73..a5c194f 100644 --- a/config.toml +++ b/config.toml @@ -3,16 +3,18 @@ escalation_interval = "1m" datadog_enabled = false statsd_enabled = false -[predefined.network] +[service.predefined.network] chain_id = 31337 -http_url = "http://127.0.0.1:8545" -ws_url = "ws://127.0.0.1:8545" +name = "predefined" +http_rpc = "http://127.0.0.1:8545" +ws_rpc = "ws://127.0.0.1:8545" -[predefined.relayer] +[service.predefined.relayer] id = "1b908a34-5dc1-4d2d-a146-5eb46e975830" +name = "predefined" chain_id = 31337 key_id = "d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa" -api_key = "G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH" +api_key = "G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsk=" [server] host = "127.0.0.1:3000" diff --git a/src/api_key.rs b/src/api_key.rs index 02d5576..06a1216 100644 --- a/src/api_key.rs +++ b/src/api_key.rs @@ -3,40 +3,79 @@ use std::str::FromStr; use base64::Engine; use rand::rngs::OsRng; -use rand::RngCore; +use rand::Rng; use serde::Serialize; use sha3::{Digest, Sha3_256}; -#[derive(Debug, Clone, PartialEq, Eq)] +const DEFAULT_SECRET_LEN: usize = 16; +const MIN_SECRET_LEN: usize = 16; +const MAX_SECRET_LEN: usize = 32; +const UUID_LEN: usize = 16; + +#[derive(Clone, Eq, PartialEq)] +struct ApiSecret(Vec); + +/// Derive Serialize manually to avoid leaking the secret. +impl Serialize for ApiSecret { + fn serialize( + &self, + serializer: S, + ) -> Result { + serializer.collect_str(&"***") + } +} + +/// Derive Debug manually to avoid leaking the secret. +impl std::fmt::Debug for ApiSecret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ApiSecret").field(&"***").finish() + } +} + +/// Zero out the secret when dropped. +impl Drop for ApiSecret { + fn drop(&mut self) { + self.0.iter_mut().for_each(|b| *b = 0); + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ApiKey { - pub relayer_id: String, - pub api_key: [u8; 32], + relayer_id: String, + secret: ApiSecret, } impl ApiKey { - pub fn new(relayer_id: impl ToString, key: [u8; 32]) -> Self { + pub fn new( + relayer_id: impl ToString, + secret: Vec, + ) -> eyre::Result { + if secret.len() < MIN_SECRET_LEN || secret.len() > MAX_SECRET_LEN { + eyre::bail!("invalid api key"); + } let relayer_id = relayer_id.to_string(); - Self { + Ok(Self { relayer_id, - api_key: key, - } + secret: ApiSecret(secret), + }) } pub fn random(relayer_id: impl ToString) -> Self { let relayer_id = relayer_id.to_string(); - let mut api_key = [0u8; 32]; - OsRng.fill_bytes(&mut api_key); - Self { relayer_id, - api_key, + secret: ApiSecret(OsRng.gen::<[u8; DEFAULT_SECRET_LEN]>().into()), } } - pub fn api_key_hash(&self) -> [u8; 32] { - Sha3_256::digest(self.api_key).into() + pub fn api_key_secret_hash(&self) -> [u8; 32] { + Sha3_256::digest(self.secret.0.clone()).into() + } + + pub fn relayer_id(&self) -> &str { + &self.relayer_id } } @@ -45,7 +84,8 @@ impl Serialize for ApiKey { &self, serializer: S, ) -> Result { - serializer.collect_str(self) + serializer + .serialize_str(&self.reveal().map_err(serde::ser::Error::custom)?) } } @@ -66,64 +106,112 @@ impl FromStr for ApiKey { fn from_str(s: &str) -> Result { let buffer = base64::prelude::BASE64_URL_SAFE.decode(s)?; - if buffer.len() != 48 { - return Err(eyre::eyre!("invalid api key")); + if buffer.len() < UUID_LEN + MIN_SECRET_LEN + || buffer.len() > UUID_LEN + MAX_SECRET_LEN + { + eyre::bail!("invalid api key"); } - let relayer_id = uuid::Uuid::from_slice(&buffer[..16])?; + let relayer_id = uuid::Uuid::from_slice(&buffer[..UUID_LEN])?; let relayer_id = relayer_id.to_string(); - let mut api_key = [0u8; 32]; - api_key.copy_from_slice(&buffer[16..]); + let secret = ApiSecret(buffer[UUID_LEN..].into()); - Ok(Self { - relayer_id, - api_key, - }) + Ok(Self { relayer_id, secret }) } } -impl std::fmt::Display for ApiKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut buffer = [0u8; 48]; - +impl ApiKey { + pub fn reveal(&self) -> eyre::Result { let relayer_id = uuid::Uuid::parse_str(&self.relayer_id) .map_err(|_| std::fmt::Error)?; - buffer[..16].copy_from_slice(relayer_id.as_bytes()); - buffer[16..].copy_from_slice(&self.api_key); + let bytes = relayer_id + .as_bytes() + .iter() + .cloned() + .chain(self.secret.0.iter().cloned()) + .collect::>(); - let encoded = base64::prelude::BASE64_URL_SAFE.encode(buffer); - - write!(f, "{}", encoded) + Ok(base64::prelude::BASE64_URL_SAFE.encode(bytes)) } } #[cfg(test)] mod tests { use rand::rngs::OsRng; - use rand::RngCore; use super::*; fn random_api_key() -> ApiKey { - let mut api_key = [0u8; 32]; - OsRng.fill_bytes(&mut api_key); + ApiKey::new( + uuid::Uuid::new_v4().to_string(), + OsRng.gen::<[u8; DEFAULT_SECRET_LEN]>().into(), + ) + .unwrap() + } + + fn invalid_short_api_key() -> ApiKey { + let mut buf = [0u8; MAX_SECRET_LEN + 1]; + OsRng.fill(&mut buf[..]); + ApiKey { + relayer_id: uuid::Uuid::new_v4().to_string(), + secret: ApiSecret(buf.into()), + } + } - ApiKey::new(uuid::Uuid::new_v4().to_string(), api_key) + fn invalid_long_api_key() -> ApiKey { + let mut buf = [0u8; MAX_SECRET_LEN + 1]; + OsRng.fill(&mut buf[..]); + ApiKey { + relayer_id: uuid::Uuid::new_v4().to_string(), + secret: ApiSecret(buf.into()), + } } #[test] fn from_to_str() { let api_key = random_api_key(); - let api_key_str = api_key.to_string(); + let api_key_str = api_key.reveal().unwrap(); println!("api_key_str = {api_key_str}"); let api_key_parsed = api_key_str.parse::().unwrap(); - assert_eq!(api_key, api_key_parsed); + assert_eq!(api_key.relayer_id, api_key_parsed.relayer_id); + assert_eq!(api_key.secret, api_key_parsed.secret); + } + + #[test] + fn assert_api_secret_debug() { + let api_secret = random_api_key().secret; + assert_eq!(&format!("{:?}", api_secret), "ApiSecret(\"***\")"); + } + + #[test] + fn assert_api_key_length_validation() { + let long_api_key = invalid_long_api_key(); + + let _ = ApiKey::new( + long_api_key.relayer_id.clone(), + long_api_key.secret.0.clone(), + ) + .expect_err("long api key should be invalid"); + + let _ = ApiKey::from_str(&long_api_key.reveal().unwrap()) + .expect_err("long api key should be invalid"); + + let short_api_key = invalid_short_api_key(); + + let _ = ApiKey::new( + short_api_key.relayer_id.clone(), + short_api_key.secret.0.clone(), + ) + .expect_err("short api key should be invalid"); + + let _ = ApiKey::from_str(&short_api_key.reveal().unwrap()) + .expect_err("short api key should be invalid"); } #[test] diff --git a/src/app.rs b/src/app.rs index 712423a..f3a1c49 100644 --- a/src/app.rs +++ b/src/app.rs @@ -81,7 +81,10 @@ impl App { api_token: &ApiKey, ) -> eyre::Result { self.db - .is_api_key_valid(&api_token.relayer_id, api_token.api_key_hash()) + .is_api_key_valid( + api_token.relayer_id(), + api_token.api_key_secret_hash(), + ) .await } } diff --git a/src/client.rs b/src/client.rs index 49aa7d1..9f5c108 100644 --- a/src/client.rs +++ b/src/client.rs @@ -86,8 +86,11 @@ impl TxSitterClient { api_key: &ApiKey, req: &SendTxRequest, ) -> eyre::Result { - self.json_post(&format!("{}/1/api/{api_key}/tx", self.url), req) - .await + self.json_post( + &format!("{}/1/api/{}/tx", self.url, api_key.reveal()?), + req, + ) + .await } pub async fn get_tx( @@ -96,9 +99,9 @@ impl TxSitterClient { tx_id: &str, ) -> eyre::Result { self.json_get(&format!( - "{}/1/api/{api_key}/tx/{tx_id}", + "{}/1/api/{}/tx/{tx_id}", self.url, - api_key = api_key, + api_key.reveal()?, tx_id = tx_id )) .await diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index 88d923e..da87148 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -121,7 +121,7 @@ pub async fn purge_unsent_txs( Ok(()) } -#[tracing::instrument(skip(app))] +#[tracing::instrument(skip(app, api_token))] pub async fn relayer_rpc( State(app): State>, Path(api_token): Path, @@ -131,7 +131,7 @@ pub async fn relayer_rpc( return Err(ApiError::Unauthorized); } - let relayer_info = app.db.get_relayer(&api_token.relayer_id).await?; + let relayer_info = app.db.get_relayer(api_token.relayer_id()).await?; // TODO: Cache? let http_provider = app.http_provider(relayer_info.chain_id).await?; @@ -161,7 +161,7 @@ pub async fn create_relayer_api_key( let api_key = ApiKey::random(&relayer_id); app.db - .create_api_key(&relayer_id, api_key.api_key_hash()) + .create_api_key(&relayer_id, api_key.api_key_secret_hash()) .await?; Ok(Json(CreateApiKeyResponse { api_key })) diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 75dceec..a74cf30 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -74,7 +74,7 @@ pub enum UnsentStatus { Unsent, } -#[tracing::instrument(skip(app))] +#[tracing::instrument(skip(app, api_token))] pub async fn send_tx( State(app): State>, Path(api_token): Path, @@ -98,7 +98,7 @@ pub async fn send_tx( req.value, req.gas_limit, req.priority, - &api_token.relayer_id, + api_token.relayer_id(), ) .await?; @@ -120,13 +120,13 @@ pub async fn get_txs( let txs = match query.status { Some(GetTxResponseStatus::TxStatus(status)) => { app.db - .read_txs(&api_token.relayer_id, Some(Some(status))) + .read_txs(api_token.relayer_id(), Some(Some(status))) .await? } Some(GetTxResponseStatus::Unsent(_)) => { - app.db.read_txs(&api_token.relayer_id, Some(None)).await? + app.db.read_txs(api_token.relayer_id(), Some(None)).await? } - None => app.db.read_txs(&api_token.relayer_id, None).await?, + None => app.db.read_txs(api_token.relayer_id(), None).await?, }; let txs = @@ -152,7 +152,7 @@ pub async fn get_txs( Ok(Json(txs)) } -#[tracing::instrument(skip(app))] +#[tracing::instrument(skip(app, api_token))] pub async fn get_tx( State(app): State>, Path((api_token, tx_id)): Path<(ApiKey, String)>, diff --git a/src/service.rs b/src/service.rs index f08b663..94260ef 100644 --- a/src/service.rs +++ b/src/service.rs @@ -125,8 +125,8 @@ async fn initialize_predefined_values( app.db .create_api_key( - &predefined.relayer.api_key.relayer_id, - predefined.relayer.api_key.api_key_hash(), + predefined.relayer.api_key.relayer_id(), + predefined.relayer.api_key.api_key_secret_hash(), ) .await?; diff --git a/tests/rpc_access.rs b/tests/rpc_access.rs index f646210..15eed04 100644 --- a/tests/rpc_access.rs +++ b/tests/rpc_access.rs @@ -15,8 +15,11 @@ async fn rpc_access() -> eyre::Result<()> { let CreateApiKeyResponse { api_key } = client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; - let rpc_url = - format!("http://{}/1/api/{api_key}/rpc", service.local_addr()); + let rpc_url = format!( + "http://{}/1/api/{}/rpc", + service.local_addr(), + api_key.reveal()? + ); let provider = Provider::new(Http::new(rpc_url.parse::()?)); From 8c5b2a437c803fb2882b0ac899019ae795197a81 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 23 Jan 2024 01:01:47 -0800 Subject: [PATCH 123/135] redact api key (#20) --- src/server.rs | 7 ++++++- src/server/trace_layer.rs | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/server/trace_layer.rs diff --git a/src/server.rs b/src/server.rs index 11803d1..c6b5502 100644 --- a/src/server.rs +++ b/src/server.rs @@ -13,10 +13,12 @@ use self::routes::relayer::{ purge_unsent_txs, relayer_rpc, update_relayer, }; use self::routes::transaction::{get_tx, get_txs, send_tx}; +use self::trace_layer::MatchedPathMakeSpan; use crate::app::App; mod middleware; pub mod routes; +mod trace_layer; #[derive(Debug, Error)] pub enum ApiError { @@ -99,7 +101,10 @@ pub async fn spawn_server( let router = Router::new() .nest("/1", v1_routes) .route("/health", get(routes::health)) - .layer(tower_http::trace::TraceLayer::new_for_http()) + .layer( + tower_http::trace::TraceLayer::new_for_http() + .make_span_with(MatchedPathMakeSpan), + ) .layer(axum::middleware::from_fn(middleware::log_response)); let server = axum::Server::bind(&app.config.server.host) diff --git a/src/server/trace_layer.rs b/src/server/trace_layer.rs new file mode 100644 index 0000000..85987b9 --- /dev/null +++ b/src/server/trace_layer.rs @@ -0,0 +1,25 @@ +use axum::extract::MatchedPath; +use hyper::Request; +use tower_http::trace::MakeSpan; +use tracing::{Level, Span}; + +/// MakeSpan to remove api keys from logs +#[derive(Clone)] +pub(crate) struct MatchedPathMakeSpan; + +impl MakeSpan for MatchedPathMakeSpan { + fn make_span(&mut self, request: &Request) -> Span { + let matched_path = request + .extensions() + .get::() + .map(MatchedPath::as_str); + + tracing::span!( + Level::DEBUG, + "request", + method = %request.method(), + matched_path, + version = ?request.version(), + ) + } +} From 97a46e712ce9f437f390146f7c09d7201babd101 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 25 Jan 2024 14:55:08 -0800 Subject: [PATCH 124/135] update tel batteries (#21) --- Cargo.lock | 287 +++++++++++++++++++++++++++++++++++++++++----------- Cargo.toml | 2 +- src/main.rs | 13 +-- 3 files changed, 233 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 698e24e..ef0d24a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -784,9 +784,9 @@ dependencies = [ [[package]] name = "cadence" -version = "0.29.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f39286bc075b023101dccdb79456a1334221c768b8faede0c2aff7ed29a9482d" +checksum = "eab51a759f502097abe855100b81b421d3a104b62a2c3209f751d90ce6dd2ea1" dependencies = [ "crossbeam-channel", ] @@ -1772,6 +1772,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -2011,6 +2026,15 @@ dependencies = [ "ahash 0.7.7", ] +[[package]] +name = "hashbrown" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +dependencies = [ + "ahash 0.8.6", +] + [[package]] name = "hashbrown" version = "0.14.2" @@ -2213,6 +2237,19 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -2600,14 +2637,43 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "metrics" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" +dependencies = [ + "ahash 0.8.6", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" +dependencies = [ + "base64 0.21.5", + "hyper", + "hyper-tls", + "indexmap 1.9.3", + "ipnet", + "metrics 0.22.0", + "metrics-util", + "quanta", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "metrics-exporter-statsd" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34a620eecf9e4321ebbef8f2f8e7cd22e098f11b65f2d987ce66faaa8918418" +checksum = "82bd7bb16e431f15d56a61b18ee34881cd9d427da7b4450d1a588c911c1d9ac3" dependencies = [ "cadence", - "metrics", + "metrics 0.22.0", "thiserror", ] @@ -2622,6 +2688,21 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "metrics-util" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2670b8badcc285d486261e2e9f1615b506baff91427b61bd336a472b65bbf5ed" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.13.1", + "metrics 0.22.0", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -2654,6 +2735,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -2810,36 +2909,81 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "openssl" +version = "0.10.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", + "futures-core", + "futures-sink", + "indexmap 2.1.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", ] [[package]] name = "opentelemetry-datadog" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5f4ecf595095d3b641dd2761a0c3d1f175d3d6c28f38e65418d8004ea3255dd" +checksum = "3e09667367cb509f10d7cf5960a83f9c4d96e93715f750b164b4b98d46c3cbf4" dependencies = [ "futures-core", "http", - "indexmap 1.9.3", - "itertools 0.10.5", + "indexmap 2.1.0", + "itertools 0.11.0", "once_cell", "opentelemetry", "opentelemetry-http", "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "reqwest", "rmp", "thiserror", @@ -2848,59 +2992,43 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +checksum = "7f51189ce8be654f9b5f7e70e49967ed894e84a06fc35c6c042e64ac1fc5399e" dependencies = [ "async-trait", "bytes", "http", - "opentelemetry_api", + "opentelemetry", "reqwest", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" dependencies = [ "opentelemetry", ] -[[package]] -name = "opentelemetry_api" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" -dependencies = [ - "futures-channel", - "futures-util", - "indexmap 1.9.3", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - [[package]] name = "opentelemetry_sdk" -version = "0.20.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", + "glob", "once_cell", - "opentelemetry_api", + "opentelemetry", "ordered-float", "percent-encoding", "rand", - "regex", "thiserror", "tokio", "tokio-stream", @@ -2914,9 +3042,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "3.9.2" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" dependencies = [ "num-traits", ] @@ -3367,6 +3495,21 @@ dependencies = [ "unarray", ] +[[package]] +name = "quanta" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.33" @@ -3421,6 +3564,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.4.1", +] + [[package]] name = "rayon" version = "1.8.0" @@ -4039,6 +4191,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "sketches-ddsketch" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" + [[package]] name = "slab" version = "0.4.9" @@ -4476,16 +4634,18 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "telemetry-batteries" version = "0.1.0" -source = "git+https://github.com/worldcoin/telemetry-batteries?branch=dzejkop/unnest-fields#4d64684669879cefae4b3c5ca047fdf4c2c62a7b" +source = "git+https://github.com/worldcoin/telemetry-batteries?rev=ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4#ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" dependencies = [ "chrono", "dirs", "http", - "metrics", + "metrics 0.22.0", + "metrics-exporter-prometheus", "metrics-exporter-statsd", "opentelemetry", "opentelemetry-datadog", "opentelemetry-http", + "opentelemetry_sdk", "serde", "serde_json", "thiserror", @@ -4669,6 +4829,16 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -4888,17 +5058,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tracing-log" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -4912,18 +5071,20 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" dependencies = [ + "js-sys", "once_cell", "opentelemetry", "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", - "tracing-log 0.1.4", + "tracing-log", "tracing-subscriber", + "web-time", ] [[package]] @@ -4953,7 +5114,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log 0.2.0", + "tracing-log", "tracing-serde", ] @@ -5012,7 +5173,7 @@ dependencies = [ "hyper", "indoc", "itertools 0.12.0", - "metrics", + "metrics 0.21.1", "num-bigint", "postgres-docker-utils", "rand", @@ -5286,6 +5447,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.24.0" diff --git a/Cargo.toml b/Cargo.toml index f9f6b47..8aee5cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ sqlx = { version = "0.7.2", features = [ strum = { version = "0.25.0", features = ["derive"] } # Company -telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", branch = "dzejkop/unnest-fields" } +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", rev = "ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" } thiserror = "1.0.50" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } toml = "0.8.8" diff --git a/src/main.rs b/src/main.rs index 18e8cca..9361396 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,8 +3,7 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; use telemetry_batteries::metrics::statsd::StatsdBattery; -use telemetry_batteries::metrics::MetricsBattery; -use telemetry_batteries::tracing::batteries::datadog::DatadogBattery; +use telemetry_batteries::tracing::datadog::DatadogBattery; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; @@ -59,11 +58,7 @@ async fn main() -> eyre::Result<()> { let config = settings.try_deserialize::()?; if config.service.datadog_enabled { - let datadog_battery = - DatadogBattery::new(None, "tx-sitter-monolith", None) - .with_location(); - - datadog_battery.init()?; + DatadogBattery::init(None, "tx-sitter-monolith", None, true); } else { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().pretty().compact()) @@ -72,15 +67,13 @@ async fn main() -> eyre::Result<()> { } if config.service.statsd_enabled { - let statsd_battery = StatsdBattery::new( + StatsdBattery::init( "localhost", 8125, 5000, 1024, Some("tx_sitter_monolith"), )?; - - statsd_battery.init()?; } let service = Service::new(config).await?; From 77add8f3c2ac774986fd8d13965785525f21cfb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Mon, 12 Feb 2024 13:06:41 +0100 Subject: [PATCH 125/135] Remove transaction simulation (#22) --- src/tasks/broadcast.rs | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 673c02f..3e6664b 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -109,33 +109,14 @@ async fn broadcast_relayer_txs( .fill_transaction(&mut typed_transaction, None) .await?; - tracing::debug!(tx_id = tx.id, "Simulating transaction"); - - // Simulate the transaction - match middleware.call(&typed_transaction, None).await { - Ok(_) => { - tracing::info!( - tx_id = tx.id, - "Transaction simulated successfully" - ); - } - Err(err) => { - tracing::error!(tx_id = tx.id, error = ?err, "Failed to simulate transaction"); - - // If we fail while broadcasting a tx with nonce `n`, - // it doesn't make sense to broadcast tx with nonce `n + 1` - return Ok(()); - } - }; - // Get the raw signed tx and derive the tx hash let raw_signed_tx = middleware .signer() .raw_signed_tx(&typed_transaction) .await?; - let tx_hash = H256::from(ethers::utils::keccak256(&raw_signed_tx)); + tracing::debug!(tx_id = tx.id, "Saving transaction"); app.db .insert_tx_broadcast( &tx.id, From 70031b171227bafacb808d3c7ccbe59ab8c508c3 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 14 Feb 2024 10:31:24 +0100 Subject: [PATCH 126/135] More instrumentation --- src/tasks/broadcast.rs | 159 ++++++++++++++++++----------------- src/tasks/escalate.rs | 184 +++++++++++++++++++++-------------------- src/tasks/index.rs | 55 ++++++------ 3 files changed, 211 insertions(+), 187 deletions(-) diff --git a/src/tasks/broadcast.rs b/src/tasks/broadcast.rs index 3e6664b..e2e97e8 100644 --- a/src/tasks/broadcast.rs +++ b/src/tasks/broadcast.rs @@ -69,87 +69,94 @@ async fn broadcast_relayer_txs( ); for tx in txs { - tracing::info!(tx_id = tx.id, nonce = tx.nonce, "Sending transaction"); - - let middleware = app - .signer_middleware(tx.chain_id, tx.key_id.clone()) - .await?; - - let fees = app - .db - .get_latest_block_fees_by_chain_id(tx.chain_id) - .await? - .context("Missing block fees")?; - - let max_base_fee_per_gas = fees.fee_estimates.base_fee_per_gas; - - let (max_fee_per_gas, max_priority_fee_per_gas) = - calculate_gas_fees_from_estimates( - &fees.fee_estimates, - tx.priority.to_percentile_index(), - max_base_fee_per_gas, - ); - - let mut typed_transaction = - TypedTransaction::Eip1559(Eip1559TransactionRequest { - from: None, - to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), - gas: Some(tx.gas_limit.0), - value: Some(tx.value.0), - data: Some(tx.data.into()), - nonce: Some(tx.nonce.into()), - access_list: AccessList::default(), - max_priority_fee_per_gas: Some(max_priority_fee_per_gas), - max_fee_per_gas: Some(max_fee_per_gas), - chain_id: Some(tx.chain_id.into()), - }); - - // Fill and simulate the transaction - middleware - .fill_transaction(&mut typed_transaction, None) - .await?; - - // Get the raw signed tx and derive the tx hash - let raw_signed_tx = middleware - .signer() - .raw_signed_tx(&typed_transaction) - .await?; - let tx_hash = H256::from(ethers::utils::keccak256(&raw_signed_tx)); - - tracing::debug!(tx_id = tx.id, "Saving transaction"); - app.db - .insert_tx_broadcast( - &tx.id, - tx_hash, - max_fee_per_gas, - max_priority_fee_per_gas, - ) - .await?; - - tracing::debug!(tx_id = tx.id, "Sending transaction"); - - let pending_tx = middleware.send_raw_transaction(raw_signed_tx).await; - - let pending_tx = match pending_tx { - Ok(pending_tx) => pending_tx, - Err(err) => { - tracing::error!(tx_id = tx.id, error = ?err, "Failed to send transaction"); - continue; - } - }; - - tracing::info!( - tx_id = tx.id, - tx_nonce = tx.nonce, - tx_hash = ?tx_hash, - ?pending_tx, - "Transaction broadcast" - ); + broadcast_relayer_tx(app, tx).await?; } Ok(()) } +#[tracing::instrument(skip(app, tx), fields(relayer_id = tx.relayer_id, tx_id = tx.id))] +async fn broadcast_relayer_tx(app: &App, tx: UnsentTx) -> eyre::Result<()> { + tracing::info!(tx_id = tx.id, nonce = tx.nonce, "Sending transaction"); + + let middleware = app + .signer_middleware(tx.chain_id, tx.key_id.clone()) + .await?; + + let fees = app + .db + .get_latest_block_fees_by_chain_id(tx.chain_id) + .await? + .context("Missing block fees")?; + + let max_base_fee_per_gas = fees.fee_estimates.base_fee_per_gas; + + let (max_fee_per_gas, max_priority_fee_per_gas) = + calculate_gas_fees_from_estimates( + &fees.fee_estimates, + tx.priority.to_percentile_index(), + max_base_fee_per_gas, + ); + + let mut typed_transaction = + TypedTransaction::Eip1559(Eip1559TransactionRequest { + from: None, + to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), + gas: Some(tx.gas_limit.0), + value: Some(tx.value.0), + data: Some(tx.data.into()), + nonce: Some(tx.nonce.into()), + access_list: AccessList::default(), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + max_fee_per_gas: Some(max_fee_per_gas), + chain_id: Some(tx.chain_id.into()), + }); + + // Fill and simulate the transaction + middleware + .fill_transaction(&mut typed_transaction, None) + .await?; + + // Get the raw signed tx and derive the tx hash + let raw_signed_tx = middleware + .signer() + .raw_signed_tx(&typed_transaction) + .await?; + let tx_hash = H256::from(ethers::utils::keccak256(&raw_signed_tx)); + + tracing::debug!(tx_id = tx.id, "Saving transaction"); + app.db + .insert_tx_broadcast( + &tx.id, + tx_hash, + max_fee_per_gas, + max_priority_fee_per_gas, + ) + .await?; + + tracing::debug!(tx_id = tx.id, "Sending transaction"); + + let pending_tx = middleware.send_raw_transaction(raw_signed_tx).await; + + let pending_tx = match pending_tx { + Ok(pending_tx) => pending_tx, + Err(err) => { + tracing::error!(tx_id = tx.id, error = ?err, "Failed to send transaction"); + return Ok(()); + } + }; + + tracing::info!( + tx_id = tx.id, + tx_nonce = tx.nonce, + tx_hash = ?tx_hash, + ?pending_tx, + "Transaction broadcast" + ); + + Ok(()) +} + fn sort_txs_by_relayer( mut txs: Vec, ) -> HashMap> { diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 76cf4da..cae22f0 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -12,6 +12,7 @@ use futures::StreamExt; use crate::app::App; use crate::broadcast_utils::should_send_relayer_transactions; use crate::db::TxForEscalation; +use crate::types::RelayerInfo; pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { loop { @@ -38,6 +39,7 @@ pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { } } +#[tracing::instrument(skip(app, txs))] async fn escalate_relayer_txs( app: &App, relayer_id: String, @@ -45,100 +47,106 @@ async fn escalate_relayer_txs( ) -> eyre::Result<()> { let relayer = app.db.get_relayer(&relayer_id).await?; - for tx in txs { - if !should_send_relayer_transactions(app, &relayer).await? { - tracing::warn!( - relayer_id = relayer.id, - "Skipping relayer escalations" - ); - - return Ok(()); - } - - tracing::info!( - tx_id = tx.id, - escalation_count = tx.escalation_count, - "Escalating transaction" - ); + if txs.is_empty() { + tracing::info!("No transactions to escalate"); + } - let escalation = tx.escalation_count + 1; + for tx in txs { + escalate_relayer_tx(app, &relayer, tx).await?; + } - let middleware = app - .signer_middleware(tx.chain_id, tx.key_id.clone()) - .await?; + Ok(()) +} - let fees = app - .db - .get_latest_block_fees_by_chain_id(tx.chain_id) - .await? - .context("Missing block")?; - - // Min increase of 20% on the priority fee required for a replacement tx - let factor = U256::from(100); - let increased_gas_price_percentage = - factor + U256::from(20 * (1 + escalation)); - - let initial_max_fee_per_gas = tx.initial_max_fee_per_gas.0; - - let max_fee_per_gas_increase = - initial_max_fee_per_gas * increased_gas_price_percentage / factor; - - let max_fee_per_gas = - initial_max_fee_per_gas + max_fee_per_gas_increase; - - let max_priority_fee_per_gas = - max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; - - let eip1559_tx = Eip1559TransactionRequest { - from: None, - to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), - gas: Some(tx.gas_limit.0), - value: Some(tx.value.0), - data: Some(tx.data.into()), - nonce: Some(tx.nonce.into()), - access_list: AccessList::default(), - max_priority_fee_per_gas: Some(max_priority_fee_per_gas), - max_fee_per_gas: Some(max_fee_per_gas), - chain_id: Some(tx.chain_id.into()), - }; - - let pending_tx = middleware - .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) - .await; - - let pending_tx = match pending_tx { - Ok(pending_tx) => pending_tx, - Err(err) => { - tracing::error!(tx_id = tx.id, error = ?err, "Failed to escalate transaction"); - continue; - } - }; - - let tx_hash = pending_tx.tx_hash(); - - tracing::info!( - tx_id = tx.id, - ?tx_hash, - ?initial_max_fee_per_gas, - ?max_fee_per_gas_increase, - ?max_fee_per_gas, - ?max_priority_fee_per_gas, - ?pending_tx, - "Escalated transaction" - ); - - app.db - .escalate_tx( - &tx.id, - tx_hash, - max_fee_per_gas, - max_priority_fee_per_gas, - ) - .await?; +#[tracing::instrument(skip(app, relayer, tx), fields(tx_id = tx.id))] +async fn escalate_relayer_tx( + app: &App, + relayer: &RelayerInfo, + tx: TxForEscalation, +) -> eyre::Result<()> { + if !should_send_relayer_transactions(app, relayer).await? { + tracing::warn!(relayer_id = relayer.id, "Skipping relayer escalations"); - tracing::info!(tx_id = tx.id, "Escalated transaction saved"); + return Ok(()); } + tracing::info!( + tx_id = tx.id, + escalation_count = tx.escalation_count, + "Escalating transaction" + ); + + let escalation = tx.escalation_count + 1; + + let middleware = app + .signer_middleware(tx.chain_id, tx.key_id.clone()) + .await?; + + let fees = app + .db + .get_latest_block_fees_by_chain_id(tx.chain_id) + .await? + .context("Missing block")?; + + // Min increase of 20% on the priority fee required for a replacement tx + let factor = U256::from(100); + let increased_gas_price_percentage = + factor + U256::from(20 * (1 + escalation)); + + let initial_max_fee_per_gas = tx.initial_max_fee_per_gas.0; + + let max_fee_per_gas_increase = + initial_max_fee_per_gas * increased_gas_price_percentage / factor; + + let max_fee_per_gas = initial_max_fee_per_gas + max_fee_per_gas_increase; + + let max_priority_fee_per_gas = + max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; + + let eip1559_tx = Eip1559TransactionRequest { + from: None, + to: Some(NameOrAddress::from(Address::from(tx.tx_to.0))), + gas: Some(tx.gas_limit.0), + value: Some(tx.value.0), + data: Some(tx.data.into()), + nonce: Some(tx.nonce.into()), + access_list: AccessList::default(), + max_priority_fee_per_gas: Some(max_priority_fee_per_gas), + max_fee_per_gas: Some(max_fee_per_gas), + chain_id: Some(tx.chain_id.into()), + }; + + let pending_tx = middleware + .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) + .await; + + let pending_tx = match pending_tx { + Ok(pending_tx) => pending_tx, + Err(err) => { + tracing::error!(tx_id = tx.id, error = ?err, "Failed to escalate transaction"); + return Ok(()); + } + }; + + let tx_hash = pending_tx.tx_hash(); + + tracing::info!( + tx_id = tx.id, + ?tx_hash, + ?initial_max_fee_per_gas, + ?max_fee_per_gas_increase, + ?max_fee_per_gas, + ?max_priority_fee_per_gas, + ?pending_tx, + "Escalated transaction" + ); + + app.db + .escalate_tx(&tx.id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) + .await?; + + tracing::info!(tx_id = tx.id, "Escalated transaction saved"); + Ok(()) } diff --git a/src/tasks/index.rs b/src/tasks/index.rs index dcfe76c..4edb23a 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -41,6 +41,7 @@ pub async fn index_chain(app: Arc, chain_id: u64) -> eyre::Result<()> { } } +#[tracing::instrument(skip(app, rpc, block))] pub async fn index_block( app: Arc, chain_id: u64, @@ -91,6 +92,7 @@ pub async fn index_block( Ok(()) } +#[tracing::instrument(skip(app, rpc, latest_block))] pub async fn backfill_to_block( app: Arc, chain_id: u64, @@ -203,41 +205,48 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { async fn update_relayer_nonces( relayers: &[RelayerInfo], - app: &Arc, + app: &App, rpc: &Provider, chain_id: u64, ) -> Result<(), eyre::Error> { let mut futures = FuturesUnordered::new(); for relayer in relayers { - let app = app.clone(); + futures.push(update_relayer_nonce(app, rpc, relayer, chain_id)); + } - futures.push(async move { - let tx_count = - rpc.get_transaction_count(relayer.address.0, None).await?; + while let Some(result) = futures.next().await { + result?; + } - tracing::info!( - relayer_id = relayer.id, - nonce = ?tx_count, - relayer_address = ?relayer.address.0, - "Updating relayer nonce" - ); + Ok(()) +} - app.db - .update_relayer_nonce( - chain_id, - relayer.address.0, - tx_count.as_u64(), - ) - .await?; +#[tracing::instrument(skip(app, rpc, relayer), fields(relayer_id = relayer.id))] +async fn update_relayer_nonce( + app: &App, + rpc: &Provider, + relayer: &RelayerInfo, + chain_id: u64, +) -> eyre::Result<()> { + let tx_count = rpc.get_transaction_count(relayer.address.0, None).await?; - Result::<(), eyre::Report>::Ok(()) - }) + if tx_count.as_u64() == relayer.current_nonce { + return Ok(()); } - while let Some(result) = futures.next().await { - result?; - } + tracing::info!( + relayer_id = relayer.id, + current_nonce = %relayer.current_nonce, + nonce = %relayer.nonce, + new_current_nonce = %tx_count.as_u64(), + relayer_address = ?relayer.address.0, + "Updating relayer nonce" + ); + + app.db + .update_relayer_nonce(chain_id, relayer.address.0, tx_count.as_u64()) + .await?; Ok(()) } From e8dd8c67d47cbdc9b2f52dd7f763e5df75a7dd8c Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 14 Feb 2024 15:13:59 +0100 Subject: [PATCH 127/135] Docker compose setup --- Dockerfile | 1 + compose.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 compose.yml diff --git a/Dockerfile b/Dockerfile index b1b104f..add7e48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ RUN mkdir -p ./src RUN mkdir -p ./crates/postgres-docker-utils/src # Copy only Cargo.toml for better caching +COPY .cargo/config.toml .cargo/config.toml COPY ./Cargo.toml ./Cargo.toml COPY ./Cargo.lock ./Cargo.lock COPY ./crates/postgres-docker-utils/Cargo.toml ./crates/postgres-docker-utils/Cargo.toml diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..7db4896 --- /dev/null +++ b/compose.yml @@ -0,0 +1,44 @@ +version: '3' +services: + tx-sitter: + build: + context: . + dockerfile: Dockerfile + depends_on: + - db + - blockchain + restart: always + ports: + - "3000:3000" + environment: + - TX_SITTER__SERVICE__ESCALATION_INTERVAL=1m + - TX_SITTER__DATABASE__KIND=connection_string + - TX_SITTER__DATABASE__CONNECTION_STRING=postgres://postgres:postgres@db:5432/tx-sitter?sslmode=disable + - TX_SITTER__KEYS__KIND=local + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__NAME=Anvil + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__HTTP_RPC=http://blockchain:8545 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__WS_RPC=ws://blockchain:8545 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__ID=1b908a34-5dc1-4d2d-a146-5eb46e975830 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__NAME=Relayer + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__KEY_ID=d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__API_KEY=G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH + - TX_SITTER__SERVER__HOST=0.0.0.0:3000 + - TX_SITTER__SERVER__DISABLE_AUTH=true + - RUST_LOG=info + + db: + hostname: db + image: postgres + ports: + - "5432:5432" + environment: + POSTGRES_HOST_AUTH_METHOD: trust + + blockchain: + hostname: blockchain + image: ghcr.io/foundry-rs/foundry:latest + ports: + - "8545:8545" + command: ["anvil --block-time 2"] From a3575968664cf9276a8027786d3102c2c4a3a7f9 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 14 Feb 2024 15:14:07 +0100 Subject: [PATCH 128/135] Misc --- src/db/data.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/db/data.rs b/src/db/data.rs index 058feb0..b9113be 100644 --- a/src/db/data.rs +++ b/src/db/data.rs @@ -113,6 +113,12 @@ where } } +impl From
for AddressWrapper { + fn from(value: Address) -> Self { + Self(value) + } +} + impl<'r, DB> sqlx::Decode<'r, DB> for U256Wrapper where DB: Database, @@ -158,6 +164,12 @@ where } } +impl From for U256Wrapper { + fn from(value: U256) -> Self { + Self(value) + } +} + impl<'r, DB> sqlx::Decode<'r, DB> for H256Wrapper where DB: Database, From 3a2aad57b0158f4ff048a8d2d58e9d3be630b2fa Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 14 Feb 2024 15:14:11 +0100 Subject: [PATCH 129/135] Additional logs --- src/tasks/escalate.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index cae22f0..c41e19a 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -16,11 +16,18 @@ use crate::types::RelayerInfo; pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { loop { + tracing::info!("Escalating transactions"); + let txs_for_escalation = app .db .get_txs_for_escalation(app.config.service.escalation_interval) .await?; + tracing::info!( + "Got {} transactions to escalate", + txs_for_escalation.len() + ); + let txs_for_escalation = split_txs_per_relayer(txs_for_escalation); let mut futures = FuturesUnordered::new(); From 29c90ed1fbbc2952439d3fe70484bc597970c358 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 14 Feb 2024 15:54:47 +0100 Subject: [PATCH 130/135] More logs --- src/tasks/escalate.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index c41e19a..25bc21a 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -89,12 +89,16 @@ async fn escalate_relayer_tx( .signer_middleware(tx.chain_id, tx.key_id.clone()) .await?; + tracing::info!("Escalating transaction - got middleware"); + let fees = app .db .get_latest_block_fees_by_chain_id(tx.chain_id) .await? .context("Missing block")?; + tracing::info!("Escalating transaction - got block fees"); + // Min increase of 20% on the priority fee required for a replacement tx let factor = U256::from(100); let increased_gas_price_percentage = @@ -123,10 +127,14 @@ async fn escalate_relayer_tx( chain_id: Some(tx.chain_id.into()), }; + tracing::info!("Escalating transaction - assembled tx"); + let pending_tx = middleware .send_transaction(TypedTransaction::Eip1559(eip1559_tx), None) .await; + tracing::info!("Escalating transaction - sent tx"); + let pending_tx = match pending_tx { Ok(pending_tx) => pending_tx, Err(err) => { @@ -135,6 +143,8 @@ async fn escalate_relayer_tx( } }; + tracing::info!("Escalating transaction - got pending tx"); + let tx_hash = pending_tx.tx_hash(); tracing::info!( From 1de92922a87a9fe72eba06e1b6cbc133a2f96b8a Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 15 Feb 2024 12:08:24 +0100 Subject: [PATCH 131/135] Abort on panic + better instrumentation --- Cargo.toml | 3 +++ src/service.rs | 2 +- src/tasks.rs | 2 +- src/tasks/escalate.rs | 46 +++++++++++++++++++++++-------------------- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8aee5cd..0c04a4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,3 +87,6 @@ test-case = "3.1.0" [features] default = ["default-config"] default-config = [] + +[profile.release] +panic = "abort" diff --git a/src/service.rs b/src/service.rs index 94260ef..e610e44 100644 --- a/src/service.rs +++ b/src/service.rs @@ -24,7 +24,7 @@ impl Service { let task_runner = TaskRunner::new(app.clone()); task_runner.add_task("Broadcast transactions", tasks::broadcast_txs); - task_runner.add_task("Escalate transactions", tasks::escalate_txs); + task_runner.add_task("Escalate transactions", tasks::escalate_txs_task); task_runner.add_task("Prune blocks", tasks::prune_blocks); task_runner.add_task("Prune transactions", tasks::prune_txs); task_runner.add_task("Finalize transactions", tasks::finalize_txs); diff --git a/src/tasks.rs b/src/tasks.rs index b86014b..7e13973 100644 --- a/src/tasks.rs +++ b/src/tasks.rs @@ -7,7 +7,7 @@ pub mod metrics; pub mod prune; pub use self::broadcast::broadcast_txs; -pub use self::escalate::escalate_txs; +pub use self::escalate::escalate_txs_task; pub use self::finalize::finalize_txs; pub use self::handle_reorgs::{handle_hard_reorgs, handle_soft_reorgs}; pub use self::index::index_chain; diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 25bc21a..1be0649 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -14,36 +14,40 @@ use crate::broadcast_utils::should_send_relayer_transactions; use crate::db::TxForEscalation; use crate::types::RelayerInfo; -pub async fn escalate_txs(app: Arc) -> eyre::Result<()> { +pub async fn escalate_txs_task(app: Arc) -> eyre::Result<()> { loop { - tracing::info!("Escalating transactions"); + escalate_txs(&app).await?; - let txs_for_escalation = app - .db - .get_txs_for_escalation(app.config.service.escalation_interval) - .await?; + tokio::time::sleep(app.config.service.escalation_interval).await; + } +} - tracing::info!( - "Got {} transactions to escalate", - txs_for_escalation.len() - ); +#[tracing::instrument(skip(app))] +async fn escalate_txs(app: &App) -> eyre::Result<()> { + tracing::info!("Escalating transactions"); - let txs_for_escalation = split_txs_per_relayer(txs_for_escalation); + let txs_for_escalation = app + .db + .get_txs_for_escalation(app.config.service.escalation_interval) + .await?; - let mut futures = FuturesUnordered::new(); + tracing::info!("Got {} transactions to escalate", txs_for_escalation.len()); - for (relayer_id, txs) in txs_for_escalation { - futures.push(escalate_relayer_txs(&app, relayer_id, txs)); - } + let txs_for_escalation = split_txs_per_relayer(txs_for_escalation); - while let Some(result) = futures.next().await { - if let Err(err) = result { - tracing::error!(error = ?err, "Failed escalating txs"); - } - } + let mut futures = FuturesUnordered::new(); - tokio::time::sleep(app.config.service.escalation_interval).await; + for (relayer_id, txs) in txs_for_escalation { + futures.push(escalate_relayer_txs(&app, relayer_id, txs)); + } + + while let Some(result) = futures.next().await { + if let Err(err) = result { + tracing::error!(error = ?err, "Failed escalating txs"); + } } + + Ok(()) } #[tracing::instrument(skip(app, txs))] From 9be5bfb8e44d9047ae2d111c4f624fb88dbaf05e Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Thu, 15 Feb 2024 12:58:16 +0100 Subject: [PATCH 132/135] Fix escalation fees calculation logic --- src/tasks/escalate.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 1be0649..1c5d9a8 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -108,15 +108,17 @@ async fn escalate_relayer_tx( let increased_gas_price_percentage = factor + U256::from(20 * (1 + escalation)); - let initial_max_fee_per_gas = tx.initial_max_fee_per_gas.0; + let initial_max_priority_fee_per_gas = + tx.initial_max_priority_fee_per_gas.0; - let max_fee_per_gas_increase = - initial_max_fee_per_gas * increased_gas_price_percentage / factor; + let initial_max_fee_per_gas = tx.initial_max_fee_per_gas.0; - let max_fee_per_gas = initial_max_fee_per_gas + max_fee_per_gas_increase; + let max_priority_fee_per_gas = initial_max_priority_fee_per_gas + * increased_gas_price_percentage + / factor; - let max_priority_fee_per_gas = - max_fee_per_gas - fees.fee_estimates.base_fee_per_gas; + let max_fee_per_gas = + max_priority_fee_per_gas + fees.fee_estimates.base_fee_per_gas; let eip1559_tx = Eip1559TransactionRequest { from: None, @@ -154,16 +156,16 @@ async fn escalate_relayer_tx( tracing::info!( tx_id = tx.id, ?tx_hash, + ?initial_max_priority_fee_per_gas, ?initial_max_fee_per_gas, - ?max_fee_per_gas_increase, - ?max_fee_per_gas, ?max_priority_fee_per_gas, + ?max_fee_per_gas, ?pending_tx, "Escalated transaction" ); app.db - .escalate_tx(&tx.id, tx_hash, max_fee_per_gas, max_priority_fee_per_gas) + .escalate_tx(&tx.id, tx_hash, max_fee_per_gas, max_fee_per_gas) .await?; tracing::info!(tx_id = tx.id, "Escalated transaction saved"); From 683c35cfd51a95604b67873ff8a13c0d8c1a6c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Thu, 29 Feb 2024 15:03:29 +0100 Subject: [PATCH 133/135] Bring back metrics (#24) * Config sanity check * Add max_queued_txs * Update metrics * Cleanup Cargo.toml * Add log to start service * Revert "Add max_queued_txs" This reverts commit 83c517d92b3fdaa5da5afde8df00ca944fce8135. * Add shutdown listening (non-graceful) * clippy + fmt --- Cargo.lock | 36 ++++++--------------------- Cargo.toml | 22 ++++++++--------- src/config.rs | 57 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 29 +++++----------------- src/shutdown.rs | 28 +++++++++++++++++++++ src/tasks/escalate.rs | 2 +- src/tasks/index.rs | 17 +++++-------- src/tasks/metrics.rs | 21 ++++++---------- 9 files changed, 124 insertions(+), 89 deletions(-) create mode 100644 src/shutdown.rs diff --git a/Cargo.lock b/Cargo.lock index ef0d24a..e1b98f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2628,20 +2628,9 @@ dependencies = [ [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" -dependencies = [ - "ahash 0.8.6", - "metrics-macros", - "portable-atomic", -] - -[[package]] -name = "metrics" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" +checksum = "cd71d9db2e4287c3407fa04378b8c2ee570aebe0854431562cdd89ca091854f4" dependencies = [ "ahash 0.8.6", "portable-atomic", @@ -2658,7 +2647,7 @@ dependencies = [ "hyper-tls", "indexmap 1.9.3", "ipnet", - "metrics 0.22.0", + "metrics", "metrics-util", "quanta", "thiserror", @@ -2673,21 +2662,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82bd7bb16e431f15d56a61b18ee34881cd9d427da7b4450d1a588c911c1d9ac3" dependencies = [ "cadence", - "metrics 0.22.0", + "metrics", "thiserror", ] -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "metrics-util" version = "0.16.0" @@ -2697,7 +2675,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.13.1", - "metrics 0.22.0", + "metrics", "num_cpus", "quanta", "sketches-ddsketch", @@ -4639,7 +4617,7 @@ dependencies = [ "chrono", "dirs", "http", - "metrics 0.22.0", + "metrics", "metrics-exporter-prometheus", "metrics-exporter-statsd", "opentelemetry", @@ -5173,7 +5151,7 @@ dependencies = [ "hyper", "indoc", "itertools 0.12.0", - "metrics 0.21.1", + "metrics", "num-bigint", "postgres-docker-utils", "rand", diff --git a/Cargo.toml b/Cargo.toml index 0c04a4a..c0433f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,6 @@ default-run = "tx-sitter" members = ["crates/*"] [dependencies] -async-trait = "0.1.74" - ## AWS aws-config = { version = "1.0.1" } aws-credential-types = { version = "1.0.1", features = [ @@ -19,6 +17,15 @@ aws-sdk-kms = "1.3.0" aws-smithy-runtime-api = "1.0.2" aws-smithy-types = "1.0.2" aws-types = "1.0.1" + +# Internal +postgres-docker-utils = { path = "crates/postgres-docker-utils" } + +# Company +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", rev = "ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" } + +## External +async-trait = "0.1.74" axum = { version = "0.6.20", features = ["headers"] } base64 = "0.21.5" bigdecimal = "0.4.2" @@ -36,18 +43,12 @@ humantime = "2.1.0" humantime-serde = "1.1.1" hyper = "0.14.27" itertools = "0.12.0" -metrics = "0.21.1" +metrics = "0.22.1" num-bigint = "0.4.4" -# telemetry-batteries = { path = "../telemetry-batteries" } - -# Internal -postgres-docker-utils = { path = "crates/postgres-docker-utils" } rand = "0.8.5" reqwest = { version = "0.11.13", default-features = false, features = [ "rustls-tls", ] } - -## Other serde = "1.0.136" serde_json = "1.0.91" sha3 = "0.10.8" @@ -62,9 +63,6 @@ sqlx = { version = "0.7.2", features = [ "bigdecimal", ] } strum = { version = "0.25.0", features = ["derive"] } - -# Company -telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", rev = "ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" } thiserror = "1.0.50" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } toml = "0.8.8" diff --git a/src/config.rs b/src/config.rs index 6d2ed1c..35037bb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,10 +1,40 @@ use std::net::SocketAddr; +use std::path::Path; use std::time::Duration; +use config::FileFormat; use serde::{Deserialize, Serialize}; use crate::api_key::ApiKey; +pub fn load_config<'a>( + config_files: impl Iterator, +) -> eyre::Result { + let mut settings = config::Config::builder(); + + for config_file in config_files { + settings = settings.add_source( + config::File::from(config_file).format(FileFormat::Toml), + ); + } + + let settings = settings + .add_source( + config::Environment::with_prefix("TX_SITTER").separator("__"), + ) + .add_source( + config::Environment::with_prefix("TX_SITTER_EXT") + .separator("__") + .try_parsing(true) + .list_separator(","), + ) + .build()?; + + let config = settings.try_deserialize::()?; + + Ok(config) +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub struct Config { @@ -262,4 +292,31 @@ mod tests { assert_eq!(toml, WITH_DB_PARTS); } + + #[test] + fn env_config_test() { + std::env::set_var("TX_SITTER__DATABASE__KIND", "parts"); + std::env::set_var("TX_SITTER__DATABASE__HOST", "dbHost"); + std::env::set_var("TX_SITTER__DATABASE__PORT", "dbPort"); + std::env::set_var("TX_SITTER__DATABASE__DATABASE", "dbName"); + std::env::set_var("TX_SITTER__DATABASE__USERNAME", "dbUsername"); + std::env::set_var("TX_SITTER__DATABASE__PASSWORD", "dbPassword"); + std::env::set_var("TX_SITTER__SERVICE__ESCALATION_INTERVAL", "1m"); + std::env::set_var("TX_SITTER__SERVICE__DATADOG_ENABLED", "true"); + std::env::set_var("TX_SITTER__SERVICE__STATSD_ENABLED", "true"); + std::env::set_var("TX_SITTER__SERVER__HOST", "0.0.0.0:8080"); + std::env::set_var("TX_SITTER__SERVER__USERNAME", "authUsername"); + std::env::set_var("TX_SITTER__SERVER__PASSWORD", "authPassword"); + std::env::set_var("TX_SITTER__KEYS__KIND", "kms"); + + let config = load_config(std::iter::empty()).unwrap(); + + assert!(config.service.statsd_enabled); + assert!(config.service.datadog_enabled); + assert_eq!(config.service.escalation_interval, Duration::from_secs(60)); + assert_eq!( + config.database.to_connection_string(), + "postgres://dbUsername:dbPassword@dbHost:dbPort/dbName" + ); + } } diff --git a/src/lib.rs b/src/lib.rs index 70b3af8..abb94b0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod keys; pub mod serde_utils; pub mod server; pub mod service; +pub mod shutdown; pub mod task_runner; pub mod tasks; pub mod types; diff --git a/src/main.rs b/src/main.rs index 9361396..42031e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,14 @@ use std::path::PathBuf; use clap::Parser; -use config::FileFormat; use telemetry_batteries::metrics::statsd::StatsdBattery; use telemetry_batteries::tracing::datadog::DatadogBattery; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; -use tx_sitter::config::Config; +use tx_sitter::config::load_config; use tx_sitter::service::Service; +use tx_sitter::shutdown::spawn_await_shutdown_task; #[derive(Parser)] #[command(author, version, about)] @@ -35,27 +35,7 @@ async fn main() -> eyre::Result<()> { dotenv::from_path(path)?; } - let mut settings = config::Config::builder(); - - for arg in &args.config { - settings = settings.add_source( - config::File::from(arg.as_ref()).format(FileFormat::Toml), - ); - } - - let settings = settings - .add_source( - config::Environment::with_prefix("TX_SITTER").separator("__"), - ) - .add_source( - config::Environment::with_prefix("TX_SITTER_EXT") - .separator("__") - .try_parsing(true) - .list_separator(","), - ) - .build()?; - - let config = settings.try_deserialize::()?; + let config = load_config(args.config.iter().map(PathBuf::as_ref))?; if config.service.datadog_enabled { DatadogBattery::init(None, "tx-sitter-monolith", None, true); @@ -76,6 +56,9 @@ async fn main() -> eyre::Result<()> { )?; } + spawn_await_shutdown_task(); + + tracing::info!(?config, "Starting service"); let service = Service::new(config).await?; service.wait().await?; diff --git a/src/shutdown.rs b/src/shutdown.rs new file mode 100644 index 0000000..fa35a31 --- /dev/null +++ b/src/shutdown.rs @@ -0,0 +1,28 @@ +use core::panic; + +use tokio::signal::unix::{signal, SignalKind}; + +pub fn spawn_await_shutdown_task() { + tokio::spawn(async { + let result = await_shutdown_signal().await; + if let Err(err) = result { + tracing::error!("Error while waiting for shutdown signal: {}", err); + panic!("Error while waiting for shutdown signal: {}", err); + } + + tracing::info!("Shutdown complete"); + std::process::exit(0); + }); +} + +pub async fn await_shutdown_signal() -> eyre::Result<()> { + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigterm = signal(SignalKind::terminate())?; + + tokio::select! { + _ = sigint.recv() => { tracing::info!("SIGINT received, shutting down"); } + _ = sigterm.recv() => { tracing::info!("SIGTERM received, shutting down"); } + }; + + Ok(()) +} diff --git a/src/tasks/escalate.rs b/src/tasks/escalate.rs index 1c5d9a8..55367f6 100644 --- a/src/tasks/escalate.rs +++ b/src/tasks/escalate.rs @@ -38,7 +38,7 @@ async fn escalate_txs(app: &App) -> eyre::Result<()> { let mut futures = FuturesUnordered::new(); for (relayer_id, txs) in txs_for_escalation { - futures.push(escalate_relayer_txs(&app, relayer_id, txs)); + futures.push(escalate_relayer_txs(app, relayer_id, txs)); } while let Some(result) = futures.next().await { diff --git a/src/tasks/index.rs b/src/tasks/index.rs index 4edb23a..49eaa8b 100644 --- a/src/tasks/index.rs +++ b/src/tasks/index.rs @@ -82,7 +82,7 @@ pub async fn index_block( "Tx mined" ); - metrics::increment_counter!("tx_mined", &metric_labels); + metrics::counter!("tx_mined", &metric_labels).increment(1); } let relayers = app.db.get_relayers_by_chain_id(chain_id).await?; @@ -171,16 +171,11 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { .await?; let labels = [("chain_id", chain_id.to_string())]; - metrics::gauge!( - "gas_price", - gas_price.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR, - &labels - ); - metrics::gauge!( - "base_fee_per_gas", + metrics::gauge!("gas_price", &labels) + .set(gas_price.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR); + metrics::gauge!("base_fee_per_gas", &labels).set( fee_estimates.base_fee_per_gas.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR, - &labels ); for (i, percentile) in FEE_PERCENTILES.iter().enumerate() { @@ -188,12 +183,12 @@ pub async fn estimate_gas(app: Arc, chain_id: u64) -> eyre::Result<()> { metrics::gauge!( "percentile_fee", - percentile_fee.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR, &[ ("chain_id", chain_id.to_string()), ("percentile", percentile.to_string()), ] - ); + ) + .set(percentile_fee.as_u64() as f64 * GAS_PRICE_FOR_METRICS_FACTOR); } tokio::time::sleep(Duration::from_secs( diff --git a/src/tasks/metrics.rs b/src/tasks/metrics.rs index fb945b1..f8e989b 100644 --- a/src/tasks/metrics.rs +++ b/src/tasks/metrics.rs @@ -15,19 +15,14 @@ pub async fn emit_metrics(app: Arc) -> eyre::Result<()> { // TODO: Add labels for env, etc. let labels = [("chain_id", chain_id.to_string())]; - metrics::gauge!("pending_txs", stats.pending_txs as f64, &labels); - metrics::gauge!("mined_txs", stats.mined_txs as f64, &labels); - metrics::gauge!( - "finalized_txs", - stats.finalized_txs as f64, - &labels - ); - metrics::gauge!( - "total_indexed_blocks", - stats.total_indexed_blocks as f64, - &labels - ); - metrics::gauge!("block_txs", stats.block_txs as f64, &labels); + metrics::gauge!("pending_txs", &labels) + .set(stats.pending_txs as f64); + metrics::gauge!("mined_txs", &labels).set(stats.mined_txs as f64); + metrics::gauge!("finalized_txs", &labels) + .set(stats.finalized_txs as f64); + metrics::gauge!("total_indexed_blocks", &labels) + .set(stats.total_indexed_blocks as f64); + metrics::gauge!("block_txs", &labels).set(stats.block_txs as f64); } tokio::time::sleep(EMIT_METRICS_INTERVAL).await; From bc538aaabcaec005bc7a6f6a2ced1fa55e9a4548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Tue, 19 Mar 2024 14:31:30 +0100 Subject: [PATCH 134/135] Fix telemetry issues (#27) * Fix telemetry issues * Update dep --- Cargo.lock | 11 ++++++++--- Cargo.toml | 4 ++-- src/main.rs | 11 ++++++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1b98f1..ace9c7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3643,9 +3643,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64 0.21.5", "bytes", @@ -3657,10 +3657,12 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "log", "mime", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -3669,8 +3671,10 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", + "tokio-native-tls", "tokio-rustls", "tower-service", "url", @@ -4612,7 +4616,7 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "telemetry-batteries" version = "0.1.0" -source = "git+https://github.com/worldcoin/telemetry-batteries?rev=ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4#ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" +source = "git+https://github.com/worldcoin/telemetry-batteries?rev=e0891328b29d9f85df037633feccca2f74a291a6#e0891328b29d9f85df037633feccca2f74a291a6" dependencies = [ "chrono", "dirs", @@ -4624,6 +4628,7 @@ dependencies = [ "opentelemetry-datadog", "opentelemetry-http", "opentelemetry_sdk", + "reqwest", "serde", "serde_json", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index c0433f8..0ab0993 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ aws-types = "1.0.1" postgres-docker-utils = { path = "crates/postgres-docker-utils" } # Company -telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", rev = "ec8ba6d4da45fdb98f900d8d4c8e1a09186894b4" } +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries", rev = "e0891328b29d9f85df037633feccca2f74a291a6" } ## External async-trait = "0.1.74" @@ -46,7 +46,7 @@ itertools = "0.12.0" metrics = "0.22.1" num-bigint = "0.4.4" rand = "0.8.5" -reqwest = { version = "0.11.13", default-features = false, features = [ +reqwest = { version = "0.11.24", default-features = false, features = [ "rustls-tls", ] } serde = "1.0.136" diff --git a/src/main.rs b/src/main.rs index 42031e8..412f807 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,14 +37,19 @@ async fn main() -> eyre::Result<()> { let config = load_config(args.config.iter().map(PathBuf::as_ref))?; - if config.service.datadog_enabled { - DatadogBattery::init(None, "tx-sitter-monolith", None, true); + let _shutdown_handle = if config.service.datadog_enabled { + let shutdown_handle = + DatadogBattery::init(None, "tx-sitter-monolith", None, true); + + Some(shutdown_handle) } else { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer().pretty().compact()) .with(EnvFilter::from_default_env()) .init(); - } + + None + }; if config.service.statsd_enabled { StatsdBattery::init( From 3d78aa1b8eecb0a85e68d7eb0dbfb61c6369a0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Tue, 19 Mar 2024 14:47:50 +0100 Subject: [PATCH 135/135] Add max_queued_txs column + minor client improvement (#25) * Add max_queued_txs column * Implement logic (WIP) * Serialize errors * Update errors in client + update logic * Instrument remaining db methods + fix * Fix in code condition * Reenable test * Fix and refactor tests --- db/migrations/003_relayers_tx_limits.sql | 6 ++ src/client.rs | 68 ++++++++++--- src/db.rs | 88 +++++++++++++++- src/server.rs | 41 +------- src/server/error.rs | 123 +++++++++++++++++++++++ src/server/routes/relayer.rs | 8 +- src/server/routes/transaction.rs | 18 ++++ src/types.rs | 36 +++++++ tests/common/mod.rs | 23 ++++- tests/disabled_relayer.rs | 53 ++++++++++ tests/send_many_txs.rs | 14 +-- tests/send_too_many_txs.rs | 117 +++++++++++++++++++++ 12 files changed, 520 insertions(+), 75 deletions(-) create mode 100644 db/migrations/003_relayers_tx_limits.sql create mode 100644 src/server/error.rs create mode 100644 tests/disabled_relayer.rs create mode 100644 tests/send_too_many_txs.rs diff --git a/db/migrations/003_relayers_tx_limits.sql b/db/migrations/003_relayers_tx_limits.sql new file mode 100644 index 0000000..2edf370 --- /dev/null +++ b/db/migrations/003_relayers_tx_limits.sql @@ -0,0 +1,6 @@ +ALTER TABLE relayers +ADD COLUMN max_queued_txs BIGINT NOT NULL DEFAULT 20, +ADD CONSTRAINT check_max_queued_txs CHECK (max_queued_txs > max_inflight_txs); + +UPDATE relayers +SET max_queued_txs = GREATEST(max_inflight_txs, 20); diff --git a/src/client.rs b/src/client.rs index 9f5c108..c92f639 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,4 +1,5 @@ use reqwest::Response; +use thiserror::Error; use crate::api_key::ApiKey; use crate::server::routes::network::NewNetworkInfo; @@ -8,12 +9,29 @@ use crate::server::routes::relayer::{ use crate::server::routes::transaction::{ GetTxResponse, SendTxRequest, SendTxResponse, }; +use crate::server::ApiError; +use crate::types::RelayerUpdate; pub struct TxSitterClient { client: reqwest::Client, url: String, } +#[derive(Debug, Error)] +pub enum ClientError { + #[error("Reqwest error: {0}")] + Reqwest(#[from] reqwest::Error), + + #[error("Serialization error: {0}")] + Serde(#[from] serde_json::Error), + + #[error("API error: {0}")] + TxSitter(#[from] ApiError), + + #[error("Invalid API key: {0}")] + InvalidApiKey(eyre::Error), +} + impl TxSitterClient { pub fn new(url: impl ToString) -> Self { Self { @@ -22,7 +40,7 @@ impl TxSitterClient { } } - async fn post(&self, url: &str) -> eyre::Result + async fn post(&self, url: &str) -> Result where R: serde::de::DeserializeOwned, { @@ -33,7 +51,11 @@ impl TxSitterClient { Ok(response.json().await?) } - async fn json_post(&self, url: &str, body: T) -> eyre::Result + async fn json_post( + &self, + url: &str, + body: T, + ) -> Result where T: serde::Serialize, R: serde::de::DeserializeOwned, @@ -45,7 +67,7 @@ impl TxSitterClient { Ok(response.json().await?) } - async fn json_get(&self, url: &str) -> eyre::Result + async fn json_get(&self, url: &str) -> Result where R: serde::de::DeserializeOwned, { @@ -56,19 +78,21 @@ impl TxSitterClient { Ok(response.json().await?) } - async fn validate_response(response: Response) -> eyre::Result { + async fn validate_response( + response: Response, + ) -> Result { if !response.status().is_success() { - let body = response.text().await?; - - return Err(eyre::eyre!("{body}")); + let body: ApiError = response.json().await?; + return Err(ClientError::TxSitter(body)); } Ok(response) } + pub async fn create_relayer( &self, req: &CreateRelayerRequest, - ) -> eyre::Result { + ) -> Result { self.json_post(&format!("{}/1/admin/relayer", self.url), req) .await } @@ -76,18 +100,34 @@ impl TxSitterClient { pub async fn create_relayer_api_key( &self, relayer_id: &str, - ) -> eyre::Result { + ) -> Result { self.post(&format!("{}/1/admin/relayer/{relayer_id}/key", self.url,)) .await } + pub async fn update_relayer( + &self, + relayer_id: &str, + relayer_update: RelayerUpdate, + ) -> Result<(), ClientError> { + self.json_post( + &format!("{}/1/admin/relayer/{relayer_id}", self.url), + relayer_update, + ) + .await + } + pub async fn send_tx( &self, api_key: &ApiKey, req: &SendTxRequest, - ) -> eyre::Result { + ) -> Result { self.json_post( - &format!("{}/1/api/{}/tx", self.url, api_key.reveal()?), + &format!( + "{}/1/api/{}/tx", + self.url, + api_key.reveal().map_err(ClientError::InvalidApiKey)? + ), req, ) .await @@ -97,11 +137,11 @@ impl TxSitterClient { &self, api_key: &ApiKey, tx_id: &str, - ) -> eyre::Result { + ) -> Result { self.json_get(&format!( "{}/1/api/{}/tx/{tx_id}", self.url, - api_key.reveal()?, + api_key.reveal().map_err(ClientError::InvalidApiKey)?, tx_id = tx_id )) .await @@ -111,7 +151,7 @@ impl TxSitterClient { &self, chain_id: u64, req: &NewNetworkInfo, - ) -> eyre::Result<()> { + ) -> Result<(), ClientError> { let response = self .client .post(&format!("{}/1/admin/network/{}", self.url, chain_id)) diff --git a/src/db.rs b/src/db.rs index 91b4507..ec2b221 100644 --- a/src/db.rs +++ b/src/db.rs @@ -74,7 +74,15 @@ impl Database { ) -> eyre::Result<()> { let mut tx = self.pool.begin().await?; - if let Some(name) = &update.relayer_name { + let RelayerUpdate { + relayer_name, + max_inflight_txs, + max_queued_txs, + gas_price_limits, + enabled, + } = update; + + if let Some(name) = relayer_name { sqlx::query( r#" UPDATE relayers @@ -88,7 +96,7 @@ impl Database { .await?; } - if let Some(max_inflight_txs) = update.max_inflight_txs { + if let Some(max_inflight_txs) = max_inflight_txs { sqlx::query( r#" UPDATE relayers @@ -97,12 +105,26 @@ impl Database { "#, ) .bind(id) - .bind(max_inflight_txs as i64) + .bind(*max_inflight_txs as i64) + .execute(tx.as_mut()) + .await?; + } + + if let Some(max_queued_txs) = max_queued_txs { + sqlx::query( + r#" + UPDATE relayers + SET max_queued_txs = $2 + WHERE id = $1 + "#, + ) + .bind(id) + .bind(*max_queued_txs as i64) .execute(tx.as_mut()) .await?; } - if let Some(gas_price_limits) = &update.gas_price_limits { + if let Some(gas_price_limits) = gas_price_limits { sqlx::query( r#" UPDATE relayers @@ -116,11 +138,26 @@ impl Database { .await?; } + if let Some(enabled) = enabled { + sqlx::query( + r#" + UPDATE relayers + SET enabled = $2 + WHERE id = $1 + "#, + ) + .bind(id) + .bind(*enabled) + .execute(tx.as_mut()) + .await?; + } + tx.commit().await?; Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn get_relayers(&self) -> eyre::Result> { Ok(sqlx::query_as( r#" @@ -142,6 +179,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn get_relayers_by_chain_id( &self, chain_id: u64, @@ -157,6 +195,7 @@ impl Database { nonce, current_nonce, max_inflight_txs, + max_queued_txs, gas_price_limits, enabled FROM relayers @@ -168,6 +207,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn get_relayer(&self, id: &str) -> eyre::Result { Ok(sqlx::query_as( r#" @@ -180,6 +220,7 @@ impl Database { nonce, current_nonce, max_inflight_txs, + max_queued_txs, gas_price_limits, enabled FROM relayers @@ -191,6 +232,28 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] + pub async fn get_relayer_pending_txs( + &self, + relayer_id: &str, + ) -> eyre::Result { + let (tx_count,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(1) + FROM transactions t + LEFT JOIN sent_transactions s ON (t.id = s.tx_id) + WHERE t.relayer_id = $1 + AND (s.tx_id IS NULL OR s.status = $2) + "#, + ) + .bind(relayer_id) + .bind(TxStatus::Pending) + .fetch_one(&self.pool) + .await?; + + Ok(tx_count as usize) + } + #[instrument(skip(self), level = "debug")] pub async fn create_transaction( &self, @@ -245,6 +308,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn get_unsent_txs(&self) -> eyre::Result> { Ok(sqlx::query_as( r#" @@ -309,6 +373,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn get_latest_block_number_without_fee_estimates( &self, chain_id: u64, @@ -334,6 +399,7 @@ impl Database { Ok(block_number.map(|(n,)| n as u64)) } + #[instrument(skip(self), level = "debug")] pub async fn get_latest_block_number( &self, chain_id: u64, @@ -354,6 +420,7 @@ impl Database { Ok(block_number.map(|(n,)| n as u64)) } + #[instrument(skip(self), level = "debug")] pub async fn get_latest_block_fees_by_chain_id( &self, chain_id: u64, @@ -387,6 +454,7 @@ impl Database { })) } + #[instrument(skip(self), level = "debug")] pub async fn has_blocks_for_chain( &self, chain_id: u64, @@ -687,6 +755,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn get_txs_for_escalation( &self, escalation_interval: Duration, @@ -770,6 +839,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn read_tx( &self, tx_id: &str, @@ -789,6 +859,7 @@ impl Database { .await?) } + #[instrument(skip(self), level = "debug")] pub async fn read_txs( &self, relayer_id: &str, @@ -925,6 +996,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn get_network_rpc( &self, chain_id: u64, @@ -946,6 +1018,7 @@ impl Database { Ok(row.0) } + #[instrument(skip(self), level = "debug")] pub async fn get_network_chain_ids(&self) -> eyre::Result> { let items: Vec<(i64,)> = sqlx::query_as( r#" @@ -979,6 +1052,7 @@ impl Database { Ok(()) } + #[instrument(skip(self), level = "debug")] pub async fn is_api_key_valid( &self, relayer_id: &str, @@ -1002,6 +1076,7 @@ impl Database { Ok(is_valid) } + #[instrument(skip(self), level = "debug")] pub async fn get_stats(&self, chain_id: u64) -> eyre::Result { let (pending_txs,): (i64,) = sqlx::query_as( r#" @@ -1128,7 +1203,8 @@ mod tests { match Database::new(&DatabaseConfig::connection_string(&url)).await { Ok(db) => return Ok((db, db_container)), - Err(_) => { + Err(err) => { + eprintln!("Failed to connect to the database: {err:?}"); tokio::time::sleep(Duration::from_secs(1)).await; } } @@ -1263,6 +1339,7 @@ mod tests { &RelayerUpdate { relayer_name: None, max_inflight_txs: Some(10), + max_queued_txs: Some(20), gas_price_limits: Some(vec![RelayerGasPriceLimit { chain_id: 1, value: U256Wrapper(U256::from(10_123u64)), @@ -1282,6 +1359,7 @@ mod tests { assert_eq!(relayer.nonce, 0); assert_eq!(relayer.current_nonce, 0); assert_eq!(relayer.max_inflight_txs, 10); + assert_eq!(relayer.max_queued_txs, 20); assert_eq!( relayer.gas_price_limits.0, vec![RelayerGasPriceLimit { diff --git a/src/server.rs b/src/server.rs index c6b5502..3a9b9ac 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,11 +1,8 @@ use std::sync::Arc; -use axum::http::StatusCode; -use axum::response::IntoResponse; use axum::routing::{get, post, IntoMakeService}; use axum::Router; use hyper::server::conn::AddrIncoming; -use thiserror::Error; use tower_http::validate_request::ValidateRequestHeaderLayer; use self::routes::relayer::{ @@ -16,46 +13,12 @@ use self::routes::transaction::{get_tx, get_txs, send_tx}; use self::trace_layer::MatchedPathMakeSpan; use crate::app::App; +mod error; mod middleware; pub mod routes; mod trace_layer; -#[derive(Debug, Error)] -pub enum ApiError { - #[error("Invalid key encoding")] - KeyEncoding, - - #[error("Invalid key length")] - KeyLength, - - #[error("Unauthorized")] - Unauthorized, - - #[error("Invalid format")] - InvalidFormat, - - #[error("Missing tx")] - MissingTx, - - #[error("Internal error {0}")] - Eyre(#[from] eyre::Report), -} - -impl IntoResponse for ApiError { - fn into_response(self) -> axum::response::Response { - let status_code = match self { - Self::KeyLength | Self::KeyEncoding => StatusCode::BAD_REQUEST, - Self::Unauthorized => StatusCode::UNAUTHORIZED, - Self::Eyre(_) => StatusCode::INTERNAL_SERVER_ERROR, - Self::InvalidFormat => StatusCode::BAD_REQUEST, - Self::MissingTx => StatusCode::NOT_FOUND, - }; - - let message = self.to_string(); - - (status_code, message).into_response() - } -} +pub use self::error::ApiError; pub async fn serve(app: Arc) -> eyre::Result<()> { let server = spawn_server(app).await?; diff --git a/src/server/error.rs b/src/server/error.rs new file mode 100644 index 0000000..2b2456b --- /dev/null +++ b/src/server/error.rs @@ -0,0 +1,123 @@ +use axum::response::IntoResponse; +use hyper::StatusCode; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Debug, Error, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ApiError { + #[error("Invalid key encoding")] + KeyEncoding, + + #[error("Invalid key length")] + KeyLength, + + #[error("Unauthorized")] + Unauthorized, + + #[error("Invalid format")] + InvalidFormat, + + #[error("Missing tx")] + MissingTx, + + #[error("Relayer is disabled")] + RelayerDisabled, + + #[error("Too many queued transactions, max: {max}, current: {current}")] + TooManyTransactions { max: usize, current: usize }, + + #[error("Internal error {0}")] + #[serde(with = "serde_eyre")] + Other(#[from] eyre::Report), +} + +impl IntoResponse for ApiError { + fn into_response(self) -> axum::response::Response { + let status_code = match self { + Self::KeyLength | Self::KeyEncoding => StatusCode::BAD_REQUEST, + Self::Unauthorized => StatusCode::UNAUTHORIZED, + Self::Other(_) => StatusCode::INTERNAL_SERVER_ERROR, + Self::InvalidFormat => StatusCode::BAD_REQUEST, + Self::MissingTx => StatusCode::NOT_FOUND, + Self::RelayerDisabled => StatusCode::FORBIDDEN, + Self::TooManyTransactions { .. } => StatusCode::TOO_MANY_REQUESTS, + }; + + let message = serde_json::to_string(&self) + .expect("Failed to serialize error message"); + + (status_code, message).into_response() + } +} + +// Mostly used for tests +impl PartialEq for ApiError { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::TooManyTransactions { + max: l_max, + current: l_current, + }, + Self::TooManyTransactions { + max: r_max, + current: r_current, + }, + ) => l_max == r_max && l_current == r_current, + (Self::Other(l0), Self::Other(r0)) => { + l0.to_string() == r0.to_string() + } + _ => { + core::mem::discriminant(self) == core::mem::discriminant(other) + } + } + } +} + +mod serde_eyre { + use std::borrow::Cow; + + use serde::Deserialize; + + pub fn serialize( + error: &eyre::Report, + serializer: S, + ) -> Result + where + S: serde::Serializer, + { + let error = error.to_string(); + serializer.serialize_str(&error) + } + + pub fn deserialize<'de, D>( + deserializer: D, + ) -> Result + where + D: serde::Deserializer<'de>, + { + let error = Cow::<'static, str>::deserialize(deserializer)?; + Ok(eyre::eyre!(error)) + } +} + +#[cfg(test)] +mod tests { + use test_case::test_case; + + use super::*; + + #[test_case(ApiError::KeyLength, r#""keyLength""# ; "Key length")] + #[test_case(ApiError::Other(eyre::eyre!("Test error")), r#"{"other":"Test error"}"# ; "Other error")] + #[test_case(ApiError::TooManyTransactions { max: 10, current: 20 }, r#"{"tooManyTransactions":{"max":10,"current":20}}"# ; "Too many transactions")] + fn serialization(error: ApiError, expected: &str) { + let serialized = serde_json::to_string(&error).unwrap(); + + assert_eq!(serialized, expected); + + let deserialized = serde_json::from_str::(expected).unwrap(); + + assert_eq!(error, deserialized); + } +} diff --git a/src/server/routes/relayer.rs b/src/server/routes/relayer.rs index da87148..066f4ab 100644 --- a/src/server/routes/relayer.rs +++ b/src/server/routes/relayer.rs @@ -83,10 +83,10 @@ pub async fn update_relayer( State(app): State>, Path(relayer_id): Path, Json(req): Json, -) -> Result<(), ApiError> { +) -> Result, ApiError> { app.db.update_relayer(&relayer_id, &req).await?; - Ok(()) + Ok(Json(())) } #[tracing::instrument(skip(app))] @@ -115,10 +115,10 @@ pub async fn get_relayer( pub async fn purge_unsent_txs( State(app): State>, Path(relayer_id): Path, -) -> Result<(), ApiError> { +) -> Result, ApiError> { app.db.purge_unsent_txs(&relayer_id).await?; - Ok(()) + Ok(Json(())) } #[tracing::instrument(skip(app, api_token))] diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index a74cf30..45e16d5 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -90,6 +90,24 @@ pub async fn send_tx( uuid::Uuid::new_v4().to_string() }; + let relayer = app.db.get_relayer(api_token.relayer_id()).await?; + + if !relayer.enabled { + return Err(ApiError::RelayerDisabled); + } + + let relayer_queued_tx_count = app + .db + .get_relayer_pending_txs(api_token.relayer_id()) + .await?; + + if relayer_queued_tx_count > relayer.max_queued_txs as usize { + return Err(ApiError::TooManyTransactions { + max: relayer.max_queued_txs as usize, + current: relayer_queued_tx_count, + }); + } + app.db .create_transaction( &tx_id, diff --git a/src/types.rs b/src/types.rs index b349d08..657e245 100644 --- a/src/types.rs +++ b/src/types.rs @@ -42,6 +42,8 @@ pub struct RelayerInfo { pub current_nonce: u64, #[sqlx(try_from = "i64")] pub max_inflight_txs: u64, + #[sqlx(try_from = "i64")] + pub max_queued_txs: u64, pub gas_price_limits: Json>, pub enabled: bool, } @@ -54,6 +56,8 @@ pub struct RelayerUpdate { #[serde(default)] pub max_inflight_txs: Option, #[serde(default)] + pub max_queued_txs: Option, + #[serde(default)] pub gas_price_limits: Option>, #[serde(default)] pub enabled: Option, @@ -66,6 +70,36 @@ pub struct RelayerGasPriceLimit { pub chain_id: i64, } +impl RelayerUpdate { + pub fn with_relayer_name(mut self, relayer_name: String) -> Self { + self.relayer_name = Some(relayer_name); + self + } + + pub fn with_max_inflight_txs(mut self, max_inflight_txs: u64) -> Self { + self.max_inflight_txs = Some(max_inflight_txs); + self + } + + pub fn with_max_queued_txs(mut self, max_queued_txs: u64) -> Self { + self.max_queued_txs = Some(max_queued_txs); + self + } + + pub fn with_gas_price_limits( + mut self, + gas_price_limits: Vec, + ) -> Self { + self.gas_price_limits = Some(gas_price_limits); + self + } + + pub fn with_enabled(mut self, enabled: bool) -> Self { + self.enabled = Some(enabled); + self + } +} + #[cfg(test)] mod tests { use ethers::types::{Address, U256}; @@ -83,6 +117,7 @@ mod tests { nonce: 0, current_nonce: 0, max_inflight_txs: 0, + max_queued_txs: 0, gas_price_limits: Json(vec![RelayerGasPriceLimit { value: U256Wrapper(U256::zero()), chain_id: 1, @@ -102,6 +137,7 @@ mod tests { "nonce": 0, "currentNonce": 0, "maxInflightTxs": 0, + "maxQueuedTxs": 0, "gasPriceLimits": [ { "value": "0x0", diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 3d5f121..c514749 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,12 +1,13 @@ #![allow(dead_code)] // Needed because this module is imported as module by many test crates use std::sync::Arc; +use std::time::Duration; use ethers::core::k256::ecdsa::SigningKey; use ethers::middleware::SignerMiddleware; use ethers::providers::{Http, Middleware, Provider}; use ethers::signers::{LocalWallet, Signer}; -use ethers::types::{Address, H160}; +use ethers::types::{Address, H160, U256}; use postgres_docker_utils::DockerContainerGuard; use tracing::level_filters::LevelFilter; use tracing_subscriber::layer::SubscriberExt; @@ -104,3 +105,23 @@ pub async fn setup_provider( Ok(provider) } + +pub async fn await_balance( + provider: &Provider, + expected_balance: U256, + address: Address, +) -> eyre::Result<()> { + for _ in 0..50 { + let balance = provider.get_balance(address, None).await?; + + tracing::info!(?balance, ?expected_balance, "Checking balance"); + + if balance >= expected_balance { + return Ok(()); + } else { + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + + eyre::bail!("Balance did not reach expected value"); +} diff --git a/tests/disabled_relayer.rs b/tests/disabled_relayer.rs new file mode 100644 index 0000000..20a1953 --- /dev/null +++ b/tests/disabled_relayer.rs @@ -0,0 +1,53 @@ +mod common; + +use tx_sitter::types::RelayerUpdate; + +use crate::common::prelude::*; + +#[tokio::test] +async fn disabled_relayer() -> eyre::Result<()> { + setup_tracing(); + + let (db_url, _db_container) = setup_db().await?; + let anvil = AnvilBuilder::default().spawn().await?; + + let (_service, client) = + ServiceBuilder::default().build(&anvil, &db_url).await?; + + tracing::info!("Creating relayer"); + let CreateRelayerResponse { relayer_id, .. } = client + .create_relayer(&CreateRelayerRequest { + name: "Test relayer".to_string(), + chain_id: DEFAULT_ANVIL_CHAIN_ID, + }) + .await?; + + tracing::info!("Creating API key"); + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(&relayer_id).await?; + + tracing::info!("Disabling relayer"); + client + .update_relayer( + &relayer_id, + RelayerUpdate::default().with_enabled(false), + ) + .await?; + + let value: U256 = parse_units("1", "ether")?.into(); + let response = client + .send_tx( + &api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + gas_limit: U256::from(21_000), + ..Default::default() + }, + ) + .await; + + assert!(response.is_err()); + + Ok(()) +} diff --git a/tests/send_many_txs.rs b/tests/send_many_txs.rs index 4325423..bb273f6 100644 --- a/tests/send_many_txs.rs +++ b/tests/send_many_txs.rs @@ -47,17 +47,7 @@ async fn send_many_txs() -> eyre::Result<()> { } let expected_balance = value * num_transfers; - for _ in 0..50 { - let balance = provider.get_balance(ARBITRARY_ADDRESS, None).await?; + await_balance(&provider, expected_balance, ARBITRARY_ADDRESS).await?; - tracing::info!(?balance, ?expected_balance, "Checking balance"); - - if balance == expected_balance { - return Ok(()); - } else { - tokio::time::sleep(Duration::from_secs(5)).await; - } - } - - panic!("Transactions were not sent") + Ok(()) } diff --git a/tests/send_too_many_txs.rs b/tests/send_too_many_txs.rs new file mode 100644 index 0000000..226cc43 --- /dev/null +++ b/tests/send_too_many_txs.rs @@ -0,0 +1,117 @@ +mod common; + +use tx_sitter::client::ClientError; +use tx_sitter::server::ApiError; +use tx_sitter::types::{RelayerUpdate, TransactionPriority}; + +use crate::common::prelude::*; + +const MAX_QUEUED_TXS: usize = 20; + +#[tokio::test] +async fn send_too_many_txs() -> eyre::Result<()> { + setup_tracing(); + + let (db_url, _db_container) = setup_db().await?; + let anvil = AnvilBuilder::default().spawn().await?; + + let (_service, client) = + ServiceBuilder::default().build(&anvil, &db_url).await?; + + let CreateApiKeyResponse { api_key } = + client.create_relayer_api_key(DEFAULT_RELAYER_ID).await?; + + let CreateRelayerResponse { + relayer_id: secondary_relayer_id, + address: secondary_relayer_address, + } = client + .create_relayer(&CreateRelayerRequest { + name: "Secondary Relayer".to_string(), + chain_id: DEFAULT_ANVIL_CHAIN_ID, + }) + .await?; + + let CreateApiKeyResponse { + api_key: secondary_api_key, + } = client.create_relayer_api_key(&secondary_relayer_id).await?; + + // Set max queued txs + client + .update_relayer( + &secondary_relayer_id, + RelayerUpdate::default().with_max_queued_txs(MAX_QUEUED_TXS as u64), + ) + .await?; + + let provider = setup_provider(anvil.endpoint()).await?; + + // Send a transaction + let value: U256 = parse_units("0.01", "ether")?.into(); + + for _ in 0..=MAX_QUEUED_TXS { + client + .send_tx( + &secondary_api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + data: None, + gas_limit: U256::from(21_000), + priority: TransactionPriority::Regular, + tx_id: None, + }, + ) + .await?; + } + + // Sending one more tx should fail + let result = client + .send_tx( + &secondary_api_key, + &SendTxRequest { + to: ARBITRARY_ADDRESS, + value, + data: None, + gas_limit: U256::from(21_000), + priority: TransactionPriority::Regular, + tx_id: None, + }, + ) + .await; + + assert!( + matches!( + result, + Err(ClientError::TxSitter(ApiError::TooManyTransactions { .. })) + ), + "Result {:?} should be too many transactions", + result + ); + + // Accumulate total value + gas budget + let send_value = value * (MAX_QUEUED_TXS + 1); + let total_required_value = send_value + parse_units("1", "ether")?; + + client + .send_tx( + &api_key, + &SendTxRequest { + to: secondary_relayer_address, + value: total_required_value, + data: None, + gas_limit: U256::from(21_000), + priority: TransactionPriority::Regular, + tx_id: None, + }, + ) + .await?; + + tracing::info!("Waiting for secondary relayer balance"); + await_balance(&provider, total_required_value, secondary_relayer_address) + .await?; + + tracing::info!("Waiting for queued up txs to be processed"); + await_balance(&provider, send_value, ARBITRARY_ADDRESS).await?; + + Ok(()) +}