diff --git a/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/down.sql b/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/down.sql new file mode 100644 index 00000000000000..b8bbded70bf476 --- /dev/null +++ b/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sum_packages; diff --git a/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/up.sql b/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/up.sql new file mode 100644 index 00000000000000..64a9f75b9d6fc0 --- /dev/null +++ b/crates/sui-indexer-alt/migrations/2024-10-31-000319_sum_packages/up.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS sum_packages +( + package_id BYTEA PRIMARY KEY, + original_id BYTEA NOT NULL, + package_version BIGINT NOT NULL, + move_package BYTEA NOT NULL, + cp_sequence_number BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS sum_packages_cp_id_version +ON sum_packages (cp_sequence_number, original_id, package_version); + +CREATE INDEX IF NOT EXISTS sum_packages_id_version_cp +ON sum_packages (original_id, package_version, cp_sequence_number); diff --git a/crates/sui-indexer-alt/src/handlers/mod.rs b/crates/sui-indexer-alt/src/handlers/mod.rs index 055ceb870d8a94..78d614c6896e74 100644 --- a/crates/sui-indexer-alt/src/handlers/mod.rs +++ b/crates/sui-indexer-alt/src/handlers/mod.rs @@ -9,6 +9,7 @@ pub mod kv_transactions; pub mod obj_versions; pub mod sum_coin_balances; pub mod sum_obj_types; +pub mod sum_packages; pub mod tx_affected_objects; pub mod tx_balance_changes; pub mod wal_coin_balances; diff --git a/crates/sui-indexer-alt/src/handlers/sum_packages.rs b/crates/sui-indexer-alt/src/handlers/sum_packages.rs new file mode 100644 index 00000000000000..6bf0176fd8ca8c --- /dev/null +++ b/crates/sui-indexer-alt/src/handlers/sum_packages.rs @@ -0,0 +1,82 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; + +use anyhow::{anyhow, Result}; +use diesel::{upsert::excluded, ExpressionMethods}; +use diesel_async::RunQueryDsl; +use futures::future::try_join_all; +use sui_types::full_checkpoint_content::CheckpointData; + +use crate::{ + db, + models::packages::StoredPackage, + pipeline::{sequential::Handler, Processor}, + schema::sum_packages, +}; + +const CHUNK_ROWS: usize = i16::MAX as usize / 5; + +pub struct SumPackages; + +impl Processor for SumPackages { + const NAME: &'static str = "sum_packages"; + + type Value = StoredPackage; + + fn process(checkpoint: &Arc) -> Result> { + let CheckpointData { + checkpoint_summary, + transactions, + .. + } = checkpoint.as_ref(); + + let cp_sequence_number = checkpoint_summary.sequence_number as i64; + let mut values = vec![]; + for tx in transactions { + for obj in &tx.output_objects { + let Some(package) = obj.data.try_as_package() else { + continue; + }; + + values.push(StoredPackage { + package_id: obj.id().to_vec(), + original_id: package.original_package_id().to_vec(), + package_version: obj.version().value() as i64, + move_package: bcs::to_bytes(package) + .map_err(|e| anyhow!("Error serializing package {}: {e}", obj.id()))?, + cp_sequence_number, + }); + } + } + + Ok(values) + } +} + +#[async_trait::async_trait] +impl Handler for SumPackages { + type Batch = Vec; + + fn batch(batch: &mut Self::Batch, values: Vec) { + batch.extend(values); + } + + async fn commit(values: &Self::Batch, conn: &mut db::Connection<'_>) -> Result { + let updates = values.chunks(CHUNK_ROWS).map(|chunk| { + diesel::insert_into(sum_packages::table) + .values(chunk) + .on_conflict(sum_packages::package_id) + .do_update() + .set(( + sum_packages::package_version.eq(excluded(sum_packages::package_version)), + sum_packages::move_package.eq(excluded(sum_packages::move_package)), + sum_packages::cp_sequence_number.eq(excluded(sum_packages::cp_sequence_number)), + )) + .execute(conn) + }); + + Ok(try_join_all(updates).await?.into_iter().sum()) + } +} diff --git a/crates/sui-indexer-alt/src/main.rs b/crates/sui-indexer-alt/src/main.rs index fd8e30cc8bee52..f593e58f0db052 100644 --- a/crates/sui-indexer-alt/src/main.rs +++ b/crates/sui-indexer-alt/src/main.rs @@ -10,7 +10,7 @@ use sui_indexer_alt::{ handlers::{ ev_emit_mod::EvEmitMod, ev_struct_inst::EvStructInst, kv_checkpoints::KvCheckpoints, kv_objects::KvObjects, kv_transactions::KvTransactions, obj_versions::ObjVersions, - sum_coin_balances::SumCoinBalances, sum_obj_types::SumObjTypes, + sum_coin_balances::SumCoinBalances, sum_obj_types::SumObjTypes, sum_packages::SumPackages, tx_affected_objects::TxAffectedObjects, tx_balance_changes::TxBalanceChanges, wal_coin_balances::WalCoinBalances, wal_obj_types::WalObjTypes, }, @@ -48,6 +48,7 @@ async fn main() -> Result<()> { indexer.concurrent_pipeline::().await?; indexer.sequential_pipeline::(lag).await?; indexer.sequential_pipeline::(lag).await?; + indexer.sequential_pipeline::(None).await?; let h_indexer = indexer.run().await.context("Failed to start indexer")?; diff --git a/crates/sui-indexer-alt/src/models/mod.rs b/crates/sui-indexer-alt/src/models/mod.rs index b20e260b291761..bc190532c38c8c 100644 --- a/crates/sui-indexer-alt/src/models/mod.rs +++ b/crates/sui-indexer-alt/src/models/mod.rs @@ -4,5 +4,6 @@ pub mod checkpoints; pub mod events; pub mod objects; +pub mod packages; pub mod transactions; pub mod watermarks; diff --git a/crates/sui-indexer-alt/src/models/packages.rs b/crates/sui-indexer-alt/src/models/packages.rs new file mode 100644 index 00000000000000..920419dd89e772 --- /dev/null +++ b/crates/sui-indexer-alt/src/models/packages.rs @@ -0,0 +1,16 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use diesel::prelude::*; + +use crate::schema::sum_packages; + +#[derive(Insertable, Debug, Clone)] +#[diesel(table_name = sum_packages, primary_key(package_id))] +pub struct StoredPackage { + pub package_id: Vec, + pub original_id: Vec, + pub package_version: i64, + pub move_package: Vec, + pub cp_sequence_number: i64, +} diff --git a/crates/sui-indexer-alt/src/schema.rs b/crates/sui-indexer-alt/src/schema.rs index 7b492f46d97530..8ae2642b00f3de 100644 --- a/crates/sui-indexer-alt/src/schema.rs +++ b/crates/sui-indexer-alt/src/schema.rs @@ -81,6 +81,16 @@ diesel::table! { } } +diesel::table! { + sum_packages (package_id) { + package_id -> Bytea, + original_id -> Bytea, + package_version -> Int8, + move_package -> Bytea, + cp_sequence_number -> Int8, + } +} + diesel::table! { tx_affected_objects (affected, tx_sequence_number) { tx_sequence_number -> Int8, @@ -144,6 +154,7 @@ diesel::allow_tables_to_appear_in_same_query!( obj_versions, sum_coin_balances, sum_obj_types, + sum_packages, tx_affected_objects, tx_balance_changes, wal_coin_balances,