diff --git a/Cargo.toml b/Cargo.toml index dc1aa70..cca1d61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,15 +7,16 @@ edition = "2021" [dependencies] sovereign-sdk = { git = "https://github.com/Sovereign-Labs/sovereign.git", rev = "5e43c3ee9b5785abdca33b21c86fd38dbd9285e0" } -tendermint = "0.27" +tendermint = "0.32" +tendermint-proto = "0.32" prost = "0.11" prost-types = "0.11" borsh = { version = "0.10.3", features = ["bytes"] } # Convenience -reqwest = { version = "0.11.13", features = ["blocking"] } -tokio = { version = "1", features = ["full"] } +reqwest = { version = "0.11.13", features = ["blocking"], optional = true } +tokio = { version = "1", features = ["full"], optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_cbor = "0.11.2" @@ -23,17 +24,24 @@ hex = { version = "0.4.3", features = ["serde"] } hex-literal = "0.3.4" base64 = "0.13.1" anyhow = "1.0.62" -jsonrpsee = { version = "0.16.2", features = ["http-client"] } +jsonrpsee = { version = "0.16.2", features = ["http-client"], optional = true } tracing = "0.1.37" -tracing-subscriber = "0.3.16" +nmt-rs = { git = "https://github.com/Sovereign-Labs/nmt-rs.git", rev = "aec2dcdc279b381162537f5b20ce43d1d46dc42f", features = ["serde", "borsh"] } -#nmt-rs = { path = "../nmt-rs", features = ["serde", "borsh"] } -nmt-rs = { git = "https://github.com/Sovereign-Labs/nmt-rs.git", rev = "36bffad64cf257264069e3b04679d945d0a0af36", features = ["serde", "borsh"] } +[dev-dependencies] +postcard = { version = "1", features = ["use-std"]} [build-dependencies] prost-build = { version = "0.11" } -#[patch.'https://github.com/Sovereign-Labs/sovereign.git'] -#sovereign-sdk = { path = "../sovereign/sdk" } +[patch.crates-io] +# Patch tendermint until the "0.32" release lands. We need 0.32 to avoid a lot of unnecessary data copying during header verification +tendermint = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" } +tendermint-proto = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" } + +[features] +default = ["native"] +native = ["dep:tokio", "dep:reqwest", "dep:jsonrpsee"] +verifier = [] diff --git a/src/celestia.rs b/src/celestia.rs index 7f6b83c..2993690 100644 --- a/src/celestia.rs +++ b/src/celestia.rs @@ -1,4 +1,4 @@ -use std::ops::Range; +use std::{cell::RefCell, ops::Range}; use borsh::{BorshDeserialize, BorshSerialize}; use nmt_rs::NamespacedHash; @@ -7,16 +7,21 @@ use serde::{Deserialize, Serialize}; use sovereign_sdk::core::traits::{ AddressTrait as Address, BlockHeaderTrait as BlockHeader, CanonicalHash, }; +pub use tendermint::block::Header as TendermintHeader; +use tendermint::{crypto::default::Sha256, merkle::simple_hash_from_byte_vectors, Hash}; +use tendermint_proto::Protobuf; use tracing::debug; +pub use tendermint_proto::v0_34 as celestia_tm_version; + const NAMESPACED_HASH_LEN: usize = 48; use crate::{ - da_app::{address::CelestiaAddress, TmHash}, - da_service::PFB_NAMESPACE, pfb::{BlobTx, MsgPayForBlobs, Tx}, shares::{read_varint, Blob, BlobRefIterator, NamespaceGroup}, utils::BoxError, + verifier::PFB_NAMESPACE, + verifier::{address::CelestiaAddress, TmHash}, }; #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] @@ -25,6 +30,158 @@ pub struct MarshalledDataAvailabilityHeader { pub column_roots: Vec, } +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, BorshDeserialize, BorshSerialize)] +pub struct PartialBlockId { + pub hash: ProtobufHash, + pub part_set_header: Vec, +} + +/// A partially serialized tendermint header. Only fields which are actually inspected by +/// Jupiter are included in their raw form. Other fields are pre-encoded as protobufs. +/// +/// This type was first introduced as a way to circumvent a bug in tendermint-rs which prevents +/// a tendermint::block::Header from being deserialized in most formats except JSON. However +/// it also provides a significant efficiency benefit over the standard tendermint type, which +/// performs a complete protobuf serialization every time `.hash()` is called. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] +pub struct CompactHeader { + /// Header version + pub version: Vec, + + /// Chain ID + pub chain_id: Vec, + + /// Current block height + pub height: Vec, + + /// Current timestamp + pub time: Vec, + + /// Previous block info + pub last_block_id: Vec, + + /// Commit from validators from the last block + pub last_commit_hash: Vec, + + /// Merkle root of transaction hashes + pub data_hash: Option, + + /// Validators for the current block + pub validators_hash: Vec, + + /// Validators for the next block + pub next_validators_hash: Vec, + + /// Consensus params for the current block + pub consensus_hash: Vec, + + /// State after txs from the previous block + pub app_hash: Vec, + + /// Root hash of all results from the txs from the previous block + pub last_results_hash: Vec, + + /// Hash of evidence included in the block + pub evidence_hash: Vec, + + /// Original proposer of the block + pub proposer_address: Vec, +} + +trait EncodeTm34 { + fn encode_to_tm34_protobuf(&self) -> Result, BoxError>; +} + +impl From for CompactHeader { + fn from(value: TendermintHeader) -> Self { + let data_hash = if let Some(h) = value.data_hash { + match h { + Hash::Sha256(value) => Some(ProtobufHash(value)), + Hash::None => None, + } + } else { + None + }; + Self { + version: Protobuf::::encode_vec( + &value.version, + ) + .unwrap(), + chain_id: value.chain_id.encode_vec().unwrap(), + height: value.height.encode_vec().unwrap(), + time: value.time.encode_vec().unwrap(), + last_block_id: Protobuf::::encode_vec( + &value.last_block_id.unwrap_or_default(), + ) + .unwrap(), + last_commit_hash: value + .last_commit_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + data_hash, + validators_hash: value.validators_hash.encode_vec().unwrap(), + next_validators_hash: value.next_validators_hash.encode_vec().unwrap(), + consensus_hash: value.consensus_hash.encode_vec().unwrap(), + app_hash: value.app_hash.encode_vec().unwrap(), + last_results_hash: value + .last_results_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + evidence_hash: value + .evidence_hash + .unwrap_or_default() + .encode_vec() + .unwrap(), + proposer_address: value.proposer_address.encode_vec().unwrap(), + } + } +} + +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize, BorshDeserialize, BorshSerialize)] +pub struct ProtobufHash(pub [u8; 32]); + +pub fn protobuf_encode(hash: &Option) -> Vec { + match hash { + Some(ProtobufHash(value)) => prost::Message::encode_to_vec(&value.to_vec()), + None => prost::Message::encode_to_vec(&vec![]), + } +} + +impl CompactHeader { + /// Hash this header + // TODO: this function can be made even more efficient. Rather than computing the block hash, + // we could provide the hash as a non-deterministic input and simply verify the correctness of the + // fields that we care about. + pub fn hash(&self) -> Hash { + // Note that if there is an encoding problem this will + // panic (as the golang code would): + // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/block.go#L393 + // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/encoding_helper.go#L9:6 + + let encoded_data_hash = protobuf_encode(&self.data_hash); + let fields_bytes = vec![ + &self.version, + &self.chain_id, + &self.height, + &self.time, + &self.last_block_id, + &self.last_commit_hash, + &encoded_data_hash, + &self.validators_hash, + &self.next_validators_hash, + &self.consensus_hash, + &self.app_hash, + &self.last_results_hash, + &self.evidence_hash, + &self.proposer_address, + ]; + + Hash::Sha256(simple_hash_from_byte_vectors::(&fields_bytes)) + } +} + #[derive( PartialEq, Debug, Clone, Deserialize, serde::Serialize, BorshDeserialize, BorshSerialize, )] @@ -59,6 +216,8 @@ impl TryFrom for DataAvailabilityHeader { } } +/// The response from the celestia `/header` endpoint. Must be converted to a +/// [`CelestiaHeader`] before use. #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] pub struct CelestiaHeaderResponse { pub header: tendermint::block::Header, @@ -74,10 +233,20 @@ pub struct NamespacedSharesResponse { #[derive(Debug, PartialEq, Clone, Deserialize, serde::Serialize)] pub struct CelestiaHeader { pub dah: DataAvailabilityHeader, - pub header: tendermint::block::Header, + pub header: CompactHeader, + #[serde(skip)] + cached_prev_hash: RefCell>, } impl CelestiaHeader { + pub fn new(dah: DataAvailabilityHeader, header: CompactHeader) -> Self { + Self { + dah, + header, + cached_prev_hash: RefCell::new(None), + } + } + pub fn square_size(&self) -> usize { self.dah.row_roots.len() } @@ -100,13 +269,22 @@ pub struct BlobWithSender { impl BlockHeader for CelestiaHeader { type Hash = TmHash; - fn prev_hash(&self) -> &Self::Hash { - self.header - .last_block_id - .as_ref() + fn prev_hash(&self) -> Self::Hash { + // Try to return the cached value + if let Some(hash) = self.cached_prev_hash.borrow().as_ref() { + return hash.clone(); + } + // If we reach this point, we know that the cach is empty - so there can't be any outstanding references to its value. + // That means its safe to borrow the cache mutably and populate it. + let mut cached_hash = self.cached_prev_hash.borrow_mut(); + let hash = + >::decode( + self.header.last_block_id.as_ref(), + ) .expect("must not call prev_hash on block with no predecessor") - .hash - .as_ref() + .hash; + *cached_hash = Some(TmHash(hash.clone())); + TmHash(hash) } } @@ -216,3 +394,33 @@ fn next_pfb(mut data: &mut BlobRefIterator) -> Result<(MsgPayForBlobs, TxPositio }, )) } + +#[cfg(test)] +mod tests { + use crate::{CelestiaHeaderResponse, CompactHeader}; + + const HEADER_RESPONSE_JSON: &[u8] = include_bytes!("./header_response.json"); + + #[test] + fn test_compact_header_serde() { + let original_header: CelestiaHeaderResponse = + serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap(); + + let header: CompactHeader = original_header.header.into(); + + let serialized_header = postcard::to_stdvec(&header).unwrap(); + let deserialized_header: CompactHeader = postcard::from_bytes(&serialized_header).unwrap(); + assert_eq!(deserialized_header, header) + } + + #[test] + fn test_compact_header_hash() { + let original_header: CelestiaHeaderResponse = + serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap(); + + let tm_header = original_header.header.clone(); + let compact_header: CompactHeader = original_header.header.into(); + + assert_eq!(tm_header.hash(), compact_header.hash()); + } +} diff --git a/src/da_service.rs b/src/da_service.rs index 244b086..bf06120 100644 --- a/src/da_service.rs +++ b/src/da_service.rs @@ -1,36 +1,25 @@ use std::{collections::HashMap, future::Future, pin::Pin}; -use borsh::{BorshDeserialize, BorshSerialize}; use jsonrpsee::{core::client::ClientT, http_client::HttpClient}; -use nmt_rs::{CelestiaNmt, NamespaceId, NamespacedHash}; -use serde::Deserialize; -use sovereign_sdk::{ - serial::{Decode, DecodeBorrowed, Encode}, - services::da::{DaService, SlotData}, - Bytes, -}; -use tendermint::merkle; +use nmt_rs::NamespaceId; +use sovereign_sdk::services::da::DaService; use tracing::{debug, info, span, Level}; // 0x736f762d74657374 = b"sov-test" // pub const ROLLUP_NAMESPACE: NamespaceId = NamespaceId(b"sov-test"); -pub const ROLLUP_NAMESPACE: NamespaceId = NamespaceId([115, 111, 118, 45, 116, 101, 115, 116]); -pub const PFB_NAMESPACE: NamespaceId = NamespaceId(hex_literal::hex!("0000000000000004")); -pub const PARITY_SHARES_NAMESPACE: NamespaceId = NamespaceId(hex_literal::hex!("ffffffffffffffff")); use crate::{ - da_app::{ - address::CelestiaAddress, - proofs::{CompletenessProof, CorrectnessProof}, - CelestiaSpec, - }, parse_pfb_namespace, - pfb::MsgPayForBlobs, share_commit::recreate_commitment, shares::{NamespaceGroup, Share}, - types::{ExtendedDataSquare, RpcNamespacedSharesResponse}, + types::{ExtendedDataSquare, FilteredCelestiaBlock, Row, RpcNamespacedSharesResponse}, utils::BoxError, - BlobWithSender, CelestiaHeader, CelestiaHeaderResponse, DataAvailabilityHeader, TxPosition, + verifier::{ + address::CelestiaAddress, + proofs::{CompletenessProof, CorrectnessProof}, + CelestiaSpec, PFB_NAMESPACE, ROLLUP_NAMESPACE, + }, + BlobWithSender, CelestiaHeader, CelestiaHeaderResponse, DataAvailabilityHeader, }; #[derive(Debug, Clone)] @@ -44,100 +33,6 @@ impl CelestiaService { } } -#[derive(Debug, Clone, PartialEq, Deserialize, serde::Serialize)] -pub struct FilteredCelestiaBlock { - pub header: CelestiaHeader, - pub rollup_data: NamespaceGroup, - /// A mapping from blob commitment to the PFB containing that commitment - /// for each blob addressed to the rollup namespace - pub relevant_pfbs: HashMap, - /// All rows in the extended data square which contain rollup data - pub rollup_rows: Vec, - /// All rows in the extended data square which contain pfb data - pub pfb_rows: Vec, -} - -impl Encode for FilteredCelestiaBlock { - fn encode(&self, target: &mut impl std::io::Write) { - serde_cbor::ser::to_writer(target, self).expect("serializing to writer should not fail"); - } -} - -impl Decode for FilteredCelestiaBlock { - type Error = anyhow::Error; - - fn decode(target: &mut R) -> Result::Error> { - Ok(serde_cbor::de::from_reader(target)?) - } -} - -impl<'de> DecodeBorrowed<'de> for FilteredCelestiaBlock { - type Error = anyhow::Error; - - fn decode_from_slice(target: &'de [u8]) -> Result { - Ok(serde_cbor::de::from_slice(target)?) - } -} - -impl SlotData for FilteredCelestiaBlock { - fn hash(&self) -> [u8; 32] { - match self.header.header.hash() { - tendermint::Hash::Sha256(h) => h, - tendermint::Hash::None => unreachable!("tendermint::Hash::None should not be possible"), - } - } -} -impl FilteredCelestiaBlock { - pub fn square_size(&self) -> usize { - self.header.square_size() - } - - pub fn get_row_number(&self, share_idx: usize) -> usize { - share_idx / self.square_size() - } - pub fn get_col_number(&self, share_idx: usize) -> usize { - share_idx % self.square_size() - } - - pub fn row_root_for_share(&self, share_idx: usize) -> &NamespacedHash { - &self.header.dah.row_roots[self.get_row_number(share_idx)] - } - - pub fn col_root_for_share(&self, share_idx: usize) -> &NamespacedHash { - &self.header.dah.column_roots[self.get_col_number(share_idx)] - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum ValidationError { - MissingDataHash, - InvalidDataRoot, - InvalidEtxProof(&'static str), - MissingTx, - InvalidRowProof, - InvalidSigner, -} - -impl CelestiaHeader { - pub fn validate_dah(&self) -> Result<(), ValidationError> { - let rows_iter = self.dah.row_roots.iter(); - let cols_iter = self.dah.column_roots.iter(); - let byte_vecs = rows_iter - .chain(cols_iter) - .map(|hash| hash.0.to_vec()) - .collect(); - let root = merkle::simple_hash_from_byte_vectors(byte_vecs); - let data_hash = self - .header - .data_hash - .ok_or(ValidationError::MissingDataHash)?; - if &root != >::as_ref(&data_hash) { - return Err(ValidationError::InvalidDataRoot); - } - Ok(()) - } -} - impl CelestiaService {} /// Fetch the rollup namespace shares and etx data. Returns a tuple `(rollup_shares, etx_shares)` @@ -245,10 +140,7 @@ impl DaService for CelestiaService { } let filtered_block = FilteredCelestiaBlock { - header: CelestiaHeader { - header: unmarshalled_header.header, - dah, - }, + header: CelestiaHeader::new(dah, unmarshalled_header.header.into()), rollup_data: rollup_shares, relevant_pfbs: pfd_map, rollup_rows, @@ -304,33 +196,6 @@ impl DaService for CelestiaService { } } -#[derive( - Debug, Clone, PartialEq, serde::Serialize, Deserialize, BorshDeserialize, BorshSerialize, -)] -pub struct Row { - pub shares: Vec, - pub root: NamespacedHash, -} - -impl Row { - pub fn merklized(&self) -> CelestiaNmt { - let mut nmt = CelestiaNmt::new(); - for (idx, share) in self.shares.iter().enumerate() { - // Shares in the two left-hand quadrants are prefixed with their namespace, while parity - // shares (in the right-hand) quadrants always have the PARITY_SHARES_NAMESPACE - let namespace = if idx < self.shares.len() / 2 { - share.namespace() - } else { - PARITY_SHARES_NAMESPACE - }; - nmt.push_leaf(share.as_serialized(), namespace) - .expect("shares are pushed in order"); - } - assert_eq!(&nmt.root(), &self.root); - nmt - } -} - async fn get_rows_containing_namespace( nid: NamespaceId, dah: &DataAvailabilityHeader, diff --git a/src/lib.rs b/src/lib.rs index 0402de6..efb60d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,9 +4,10 @@ pub mod celestia; pub mod shares; pub use celestia::*; -pub mod da_app; +#[cfg(feature = "native")] pub mod da_service; pub mod pfb; pub mod share_commit; pub mod types; mod utils; +pub mod verifier; diff --git a/src/share_commit.rs b/src/share_commit.rs index ef3bc28..0cc49e6 100644 --- a/src/share_commit.rs +++ b/src/share_commit.rs @@ -1,6 +1,6 @@ use crate::shares::{self, Share}; -use tendermint::merkle::simple_hash_from_byte_vectors; +use tendermint::{crypto::default::Sha256, merkle::simple_hash_from_byte_vectors}; // /// Calculates the size of the smallest square that could be used to commit // /// to this message, following Celestia's "non-interactive default rules" @@ -54,12 +54,7 @@ pub fn recreate_commitment( } subtree_roots.push(tree.root()); } - let h = simple_hash_from_byte_vectors( - subtree_roots - .into_iter() - .map(|x| x.as_ref().to_vec()) - .collect(), - ); + let h = simple_hash_from_byte_vectors::(&subtree_roots); Ok(h) } diff --git a/src/shares.rs b/src/shares.rs index ce282c4..40bb123 100644 --- a/src/shares.rs +++ b/src/shares.rs @@ -15,7 +15,7 @@ use sovereign_sdk::{ }; use tracing::error; -use crate::da_service::PFB_NAMESPACE; +use crate::verifier::PFB_NAMESPACE; /// The length of the "reserved bytes" field in a compact share pub const RESERVED_BYTES_LEN: usize = 4; diff --git a/src/types.rs b/src/types.rs index 791401e..f9877d7 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,7 +1,22 @@ +use std::collections::HashMap; + use anyhow::ensure; +use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use sovereign_sdk::{ + serial::{Decode, DecodeBorrowed, Encode}, + services::da::SlotData, + Bytes, +}; +use tendermint::{crypto::default::Sha256, merkle}; -use crate::{shares::Share, utils::BoxError}; +use crate::{ + pfb::MsgPayForBlobs, + shares::{NamespaceGroup, Share}, + utils::BoxError, + verifier::PARITY_SHARES_NAMESPACE, + CelestiaHeader, TxPosition, +}; #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] pub struct RpcNamespacedShares { #[serde(rename = "Proof")] @@ -51,6 +66,125 @@ impl ExtendedDataSquare { } } +#[derive(Debug, Clone, PartialEq, Deserialize, serde::Serialize)] +pub struct FilteredCelestiaBlock { + pub header: CelestiaHeader, + pub rollup_data: NamespaceGroup, + /// A mapping from blob commitment to the PFB containing that commitment + /// for each blob addressed to the rollup namespace + pub relevant_pfbs: HashMap, + /// All rows in the extended data square which contain rollup data + pub rollup_rows: Vec, + /// All rows in the extended data square which contain pfb data + pub pfb_rows: Vec, +} + +impl Encode for FilteredCelestiaBlock { + fn encode(&self, target: &mut impl std::io::Write) { + serde_cbor::ser::to_writer(target, self).expect("serializing to writer should not fail"); + } +} + +impl Decode for FilteredCelestiaBlock { + type Error = anyhow::Error; + + fn decode(target: &mut R) -> Result::Error> { + Ok(serde_cbor::de::from_reader(target)?) + } +} + +impl<'de> DecodeBorrowed<'de> for FilteredCelestiaBlock { + type Error = anyhow::Error; + + fn decode_from_slice(target: &'de [u8]) -> Result { + Ok(serde_cbor::de::from_slice(target)?) + } +} + +impl SlotData for FilteredCelestiaBlock { + fn hash(&self) -> [u8; 32] { + match self.header.header.hash() { + tendermint::Hash::Sha256(h) => h, + tendermint::Hash::None => unreachable!("tendermint::Hash::None should not be possible"), + } + } +} +impl FilteredCelestiaBlock { + pub fn square_size(&self) -> usize { + self.header.square_size() + } + + pub fn get_row_number(&self, share_idx: usize) -> usize { + share_idx / self.square_size() + } + pub fn get_col_number(&self, share_idx: usize) -> usize { + share_idx % self.square_size() + } + + pub fn row_root_for_share(&self, share_idx: usize) -> &NamespacedHash { + &self.header.dah.row_roots[self.get_row_number(share_idx)] + } + + pub fn col_root_for_share(&self, share_idx: usize) -> &NamespacedHash { + &self.header.dah.column_roots[self.get_col_number(share_idx)] + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum ValidationError { + MissingDataHash, + InvalidDataRoot, + InvalidEtxProof(&'static str), + MissingTx, + InvalidRowProof, + InvalidSigner, +} + +impl CelestiaHeader { + pub fn validate_dah(&self) -> Result<(), ValidationError> { + let rows_iter = self.dah.row_roots.iter(); + let cols_iter = self.dah.column_roots.iter(); + let byte_vecs: Vec<&NamespacedHash> = rows_iter.chain(cols_iter).collect(); + let root = merkle::simple_hash_from_byte_vectors::(&byte_vecs); + let data_hash = self + .header + .data_hash + .as_ref() + .ok_or(ValidationError::MissingDataHash)?; + if &root != &data_hash.0 { + return Err(ValidationError::InvalidDataRoot); + } + Ok(()) + } +} + +#[derive( + Debug, Clone, PartialEq, serde::Serialize, Deserialize, BorshDeserialize, BorshSerialize, +)] +pub struct Row { + pub shares: Vec, + pub root: NamespacedHash, +} + +impl Row { + pub fn merklized(&self) -> CelestiaNmt { + let mut nmt = CelestiaNmt::new(); + for (idx, share) in self.shares.iter().enumerate() { + // Shares in the two left-hand quadrants are prefixed with their namespace, while parity + // shares (in the right-hand) quadrants always have the PARITY_SHARES_NAMESPACE + let namespace = if idx < self.shares.len() / 2 { + share.namespace() + } else { + PARITY_SHARES_NAMESPACE + }; + nmt.push_leaf(share.as_serialized(), namespace) + .expect("shares are pushed in order"); + } + assert_eq!(&nmt.root(), &self.root); + nmt + } +} + #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)] pub struct StringWrapper { #[serde(rename = "/")] @@ -61,7 +195,7 @@ pub struct StringWrapper { pub struct RpcNamespacedSharesResponse(pub Option>); use nmt_rs::{ - simple_merkle::proof::Proof, NamespaceProof, NamespacedHash, NamespacedSha2Hasher, + simple_merkle::proof::Proof, CelestiaNmt, NamespaceProof, NamespacedHash, NamespacedSha2Hasher, NAMESPACED_HASH_LEN, }; @@ -94,7 +228,7 @@ mod tests { use nmt_rs::{NamespaceProof, NamespacedSha2Hasher}; - use crate::da_service::ROLLUP_NAMESPACE; + use crate::verifier::ROLLUP_NAMESPACE; use super::{ns_hash_from_b64, RpcNamespacedSharesResponse}; diff --git a/src/verifier/address.rs b/src/verifier/address.rs new file mode 100644 index 0000000..ab87ec0 --- /dev/null +++ b/src/verifier/address.rs @@ -0,0 +1,20 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use sovereign_sdk::core::traits::AddressTrait as Address; + +#[derive(Debug, PartialEq, Clone, Eq, BorshDeserialize, BorshSerialize)] +pub struct CelestiaAddress(pub Vec); + +impl AsRef<[u8]> for CelestiaAddress { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} +impl Address for CelestiaAddress {} + +impl<'a> TryFrom<&'a [u8]> for CelestiaAddress { + type Error = anyhow::Error; + + fn try_from(value: &'a [u8]) -> Result { + Ok(Self(value.to_vec())) + } +} diff --git a/src/verifier/mod.rs b/src/verifier/mod.rs new file mode 100644 index 0000000..bbe08b4 --- /dev/null +++ b/src/verifier/mod.rs @@ -0,0 +1,252 @@ +use nmt_rs::NamespaceId; +use serde::Deserialize; +use sovereign_sdk::{ + da::{self, BlobTransactionTrait, BlockHashTrait as BlockHash, DaSpec}, + serial::{Decode, DecodeBorrowed, DeserializationError, Encode}, + Bytes, +}; + +pub mod address; +pub mod proofs; + +use crate::{ + pfb_from_iter, + share_commit::recreate_commitment, + shares::{read_varint, BlobIterator, NamespaceGroup, Share}, + types::ValidationError, + BlobWithSender, CelestiaHeader, DataAvailabilityHeader, +}; +use proofs::*; + +use self::address::CelestiaAddress; + +pub struct CelestiaVerifier; + +pub const ROLLUP_NAMESPACE: NamespaceId = NamespaceId([115, 111, 118, 45, 116, 101, 115, 116]); +pub const PFB_NAMESPACE: NamespaceId = NamespaceId(hex_literal::hex!("0000000000000004")); +pub const PARITY_SHARES_NAMESPACE: NamespaceId = NamespaceId(hex_literal::hex!("ffffffffffffffff")); + +impl BlobTransactionTrait for BlobWithSender { + type Data = BlobIterator; + fn sender(&self) -> CelestiaAddress { + self.sender.clone() + } + + fn data(&self) -> Self::Data { + self.blob.clone().into_iter() + } +} +#[derive(Debug, PartialEq, Clone, Eq, Hash, serde::Serialize, Deserialize)] + +pub struct TmHash(pub tendermint::Hash); + +impl AsRef<[u8]> for TmHash { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl TmHash { + pub fn inner(&self) -> &[u8; 32] { + match self.0 { + tendermint::Hash::Sha256(ref h) => h, + tendermint::Hash::None => unreachable!("tendermint::Hash::None should not be possible"), + } + } +} + +impl AsRef for tendermint::Hash { + fn as_ref(&self) -> &TmHash { + unsafe { std::mem::transmute(self) } + } +} + +impl BlockHash for TmHash {} + +impl Decode for TmHash { + type Error = sovereign_sdk::serial::DeserializationError; + + fn decode(target: &mut R) -> Result::Error> { + // TODO: make this reasonable + let mut out = [0u8; 32]; + target + .read_exact(&mut out) + .map_err(|_| DeserializationError::DataTooShort { + expected: 32, + got: 1, + })?; + Ok(TmHash(tendermint::Hash::Sha256(out))) + } +} + +impl<'de> DecodeBorrowed<'de> for TmHash { + type Error = sovereign_sdk::serial::DeserializationError; + + fn decode_from_slice(target: &'de [u8]) -> Result { + let mut out = [0u8; 32]; + out.copy_from_slice(&target[..32]); + Ok(TmHash(tendermint::Hash::Sha256(out))) + } +} + +impl Encode for TmHash { + fn encode(&self, target: &mut impl std::io::Write) { + // TODO: make this reasonable + target + .write_all(self.as_ref()) + .expect("Serialization should not fail") + } +} + +pub struct CelestiaSpec; + +impl DaSpec for CelestiaSpec { + type SlotHash = TmHash; + + type Address = CelestiaAddress; + + type BlockHeader = CelestiaHeader; + + type BlobTransaction = BlobWithSender; + + type InclusionMultiProof = Vec; + + type CompletenessProof = Vec; +} + +impl da::DaVerifier for CelestiaVerifier { + type Spec = CelestiaSpec; + + type Error = ValidationError; + + fn verify_relevant_tx_list( + &self, + block_header: &::BlockHeader, + txs: &[::BlobTransaction], + inclusion_proof: ::InclusionMultiProof, + completeness_proof: ::CompletenessProof, + ) -> Result<(), Self::Error> { + // Validate that the provided DAH is well-formed + block_header.validate_dah()?; + + // Check the validity and completeness of the rollup row proofs, against the DAH. + // Extract the data from the row proofs and build a namespace_group from it + let rollup_shares_u8 = Self::verify_row_proofs(completeness_proof, &block_header.dah)?; + if rollup_shares_u8.is_empty() { + if txs.is_empty() { + return Ok(()); + } + return Err(ValidationError::MissingTx); + } + let namespace = NamespaceGroup::from_shares_unchecked(rollup_shares_u8); + + // Check the e-tx proofs... + // TODO(@preston-evans98): Remove this logic if Celestia adds blob.sender metadata directly into blob + let mut tx_iter = txs.iter(); + let square_size = block_header.dah.row_roots.len(); + for (blob, tx_proof) in namespace.blobs().zip(inclusion_proof.into_iter()) { + // Force the row number to be monotonically increasing + let start_offset = tx_proof.proof[0].start_offset; + + // Verify each sub-proof and flatten the shares back into a sequential array + // First, enforce that the sub-proofs cover a contiguous range of shares + for [l, r] in tx_proof.proof.array_windows::<2>() { + assert_eq!(l.start_share_idx + l.shares.len(), r.start_share_idx) + } + let mut tx_shares = Vec::new(); + // Then, verify the sub proofs + for sub_proof in tx_proof.proof.into_iter() { + let row_num = sub_proof.start_share_idx / square_size; + let root = &block_header.dah.row_roots[row_num]; + sub_proof + .proof + .verify_range(root, &sub_proof.shares, PFB_NAMESPACE) + .map_err(|_| ValidationError::InvalidEtxProof("invalid sub proof"))?; + tx_shares.extend( + sub_proof + .shares + .into_iter() + .map(|share_vec| Share::new(share_vec.into())), + ) + } + + // Next, ensure that the start_index is valid + if !tx_shares[0].is_valid_tx_start(start_offset) { + return Err(ValidationError::InvalidEtxProof("invalid start index")); + } + + // Collect all of the shares data into a single array + let trailing_shares = tx_shares[1..] + .iter() + .map(|share| share.data_ref().iter()) + .flatten(); + let tx_data: Vec = tx_shares[0].data_ref()[start_offset..] + .iter() + .chain(trailing_shares) + .map(|x| *x) + .collect(); + + // Deserialize the pfb transaction + let (len, len_of_len) = { + let cursor = std::io::Cursor::new(&tx_data); + read_varint(cursor).expect("tx must be length prefixed") + }; + let mut cursor = std::io::Cursor::new(&tx_data[len_of_len..len as usize + len_of_len]); + + let pfb = pfb_from_iter(&mut cursor, len as usize) + .map_err(|_| ValidationError::InvalidEtxProof("invalid pfb"))?; + + // Verify the sender and data of each blob which was sent into this namespace + for (blob_idx, nid) in pfb.namespace_ids.iter().enumerate() { + if nid != &ROLLUP_NAMESPACE.0[..] { + continue; + } + let tx = tx_iter.next().ok_or(ValidationError::MissingTx)?; + if tx.sender.as_ref() != pfb.signer.as_bytes() { + return Err(ValidationError::InvalidSigner); + } + + let blob_ref = blob.clone(); + let blob_data: Bytes = blob.clone().data().collect(); + let tx_data: Bytes = tx.data().collect(); + assert_eq!(blob_data, tx_data); + + // Link blob commitment to e-tx commitment + let expected_commitment = + recreate_commitment(square_size, blob_ref).map_err(|_| { + ValidationError::InvalidEtxProof("failed to recreate commitment") + })?; + + assert_eq!(&pfb.share_commitments[blob_idx][..], &expected_commitment); + } + } + + Ok(()) + } +} + +impl CelestiaVerifier { + pub fn verify_row_proofs( + row_proofs: Vec, + dah: &DataAvailabilityHeader, + ) -> Result>, ValidationError> { + let mut row_proofs = row_proofs.into_iter(); + // Check the validity and completeness of the rollup share proofs + let mut rollup_shares_u8: Vec> = Vec::new(); + for row_root in dah.row_roots.iter() { + // TODO: short circuit this loop at the first row after the rollup namespace + if row_root.contains(ROLLUP_NAMESPACE) { + let row_proof = row_proofs.next().ok_or(ValidationError::InvalidRowProof)?; + row_proof + .proof + .verify_complete_namespace(row_root, &row_proof.leaves, ROLLUP_NAMESPACE) + .expect("Proofs must be valid"); + + for leaf in row_proof.leaves { + rollup_shares_u8.push(leaf) + } + } + } + Ok(rollup_shares_u8) + } +} diff --git a/src/verifier/proofs.rs b/src/verifier/proofs.rs new file mode 100644 index 0000000..20f495b --- /dev/null +++ b/src/verifier/proofs.rs @@ -0,0 +1,112 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use nmt_rs::{NamespaceProof, NamespacedSha2Hasher}; + +use crate::{ + share_commit::recreate_commitment, shares::BlobRef, types::FilteredCelestiaBlock, + BlobWithSender, +}; + +use super::ROLLUP_NAMESPACE; + +#[derive(Debug, PartialEq, Clone, BorshDeserialize, BorshSerialize)] +pub struct EtxProof { + pub proof: Vec, +} + +#[derive(Debug, PartialEq, Clone, BorshDeserialize, BorshSerialize)] +pub struct EtxRangeProof { + pub shares: Vec>, + pub proof: NamespaceProof, + pub start_share_idx: usize, + pub start_offset: usize, +} + +#[derive(Debug, PartialEq, Clone, BorshDeserialize, BorshSerialize)] + +pub struct RelevantRowProof { + pub leaves: Vec>, + pub proof: NamespaceProof, +} + +#[derive(Debug, PartialEq, Clone)] +pub struct CompletenessProof(pub Vec); + +impl CompletenessProof { + pub fn from_filtered_block(block: &FilteredCelestiaBlock) -> Self { + let mut row_proofs = Vec::new(); + for row in block.rollup_rows.iter() { + let mut nmt = row.merklized(); + let (leaves, proof) = nmt.get_namespace_with_proof(ROLLUP_NAMESPACE); + let row_proof = RelevantRowProof { leaves, proof }; + row_proofs.push(row_proof) + } + Self(row_proofs) + } +} + +pub struct CorrectnessProof(pub Vec); + +impl CorrectnessProof { + pub fn for_block(block: &FilteredCelestiaBlock, relevant_txs: &Vec) -> Self { + let mut needed_tx_shares = Vec::new(); + + // Extract (and clone) the position of each transaction + for tx in relevant_txs.iter() { + let commitment = recreate_commitment(block.square_size(), BlobRef::with(&tx.blob.0)) + .expect("commitment is valid"); + + let (_, position) = block + .relevant_pfbs + .get(&commitment[..]) + .expect("commitment must exist in map"); + needed_tx_shares.push(position.clone()); + } + + let mut needed_tx_shares = needed_tx_shares.into_iter().peekable(); + let mut current_tx_proof: EtxProof = EtxProof { proof: Vec::new() }; + let mut tx_proofs: Vec = Vec::with_capacity(relevant_txs.len()); + + for (row_idx, row) in block.pfb_rows.iter().enumerate() { + let mut nmt = row.merklized(); + while let Some(next_needed_share) = needed_tx_shares.peek_mut() { + // If the next needed share falls in this row + let row_start_idx = block.square_size() * row_idx; + let start_column_number = next_needed_share.share_range.start - row_start_idx; + if start_column_number < block.square_size() { + let end_column_number = next_needed_share.share_range.end - row_start_idx; + if end_column_number <= block.square_size() { + let (shares, proof) = + nmt.get_range_with_proof(start_column_number..end_column_number); + + current_tx_proof.proof.push(EtxRangeProof { + shares, + proof, + start_offset: next_needed_share.start_offset, + start_share_idx: next_needed_share.share_range.start, + }); + tx_proofs.push(current_tx_proof); + current_tx_proof = EtxProof { proof: Vec::new() }; + let _ = needed_tx_shares.next(); + } else { + let (shares, proof) = + nmt.get_range_with_proof(start_column_number..block.square_size()); + + current_tx_proof.proof.push(EtxRangeProof { + shares, + proof, + start_offset: next_needed_share.start_offset, + start_share_idx: next_needed_share.share_range.start, + }); + next_needed_share.share_range.start = block.square_size() * (row_idx + 1); + next_needed_share.start_offset = 0; + + break; + } + } else { + break; + } + } + } + Self(tx_proofs) + } +}