chore: make proto crates publishable on crates.io

Cargo.toml

@@ -10,7 +10,7 @@ members = [
   "crates/utils",
   "crates/test-macro",
 ]
-default-members = ["bin/node", "bin/faucet"]
+default-members = ["bin/faucet", "bin/node", "crates/proto", "crates/rpc-proto"]
 
 resolver = "2"
 

Makefile

@@ -4,9 +4,10 @@
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-# -- variables --------------------------------------------------------------------------------------
+# -- variables ------------------------------------------------------------------------------------
 
 WARNINGS=RUSTDOCFLAGS="-D warnings"
+BUILD_PROTO=BUILD_PROTO=1
 
 # -- linting --------------------------------------------------------------------------------------
 
@@ -54,27 +55,33 @@ test:  ## Runs all tests
 
 .PHONY: check
 check: ## Check all targets and features for errors without code generation
-	cargo check --all-features --all-targets
+	${BUILD_PROTO} cargo check --all-features --all-targets
 
-# --- installing --------------------------------------------------------------------------------
+# --- building ------------------------------------------------------------------------------------
+
+.PHONY: build
+build: ## Builds all crates and re-builds ptotobuf bindings for proto crates
+	${BUILD_PROTO} cargo build
+
+# --- installing ----------------------------------------------------------------------------------
 
 .PHONY: install-node
 install-node: ## Installs node
-	cargo install --path bin/node
+	${BUILD_PROTO} cargo install --path bin/node
 
 .PHONY: install-faucet
 install-faucet: ## Installs faucet
-	cargo install --path bin/faucet
+	${BUILD_PROTO} cargo install --path bin/faucet
 
 .PHONY: install-node-testing
 install-node-testing: ## Installs node with testing feature enabled
-	cargo install --features testing --path bin/node
+	${BUILD_PROTO} cargo install --features testing --path bin/node
 
 .PHONY: install-faucet-testing
 install-faucet-testing: ## Installs faucet with testing feature enabled
-	cargo install --features testing --path bin/faucet
+	${BUILD_PROTO} cargo install --features testing --path bin/faucet
 
-# --- docker --------------------------------------------------------------------------------
+# --- docker --------------------------------------------------------------------------------------
 
 .PHONY: docker-build-node
 docker-build-node: ## Builds the Miden node using Docker

crates/proto/build.rs

@@ -3,7 +3,19 @@ use std::{env, fs, path::PathBuf};
 use miette::IntoDiagnostic;
 use prost::Message;
 
+/// Generates Rust protobuf bindings from .proto files in the root directory.
+/// 
+/// This is done only if BUILD_PROTO environment variable is set to `1` to avoid running the script
+/// on crates.io where repo-level .proto files are not available.
 fn main() -> miette::Result<()> {
+    println!("cargo:rerun-if-changed=generated");
+    println!("cargo:rerun-if-changed=../../proto");
+
+    // skip this build script in BUILD_PROTO environment variable is not set to `1`
+    if env::var("BUILD_PROTO").unwrap_or("0".to_string()) == "0" {
+        return Ok(())
+    }
+
     // Compute the directory of the `proto` definitions
     let cwd: PathBuf = env::current_dir().into_diagnostic()?;
 

crates/rpc-proto/build.rs

@@ -1,24 +1,47 @@
 use std::{
     env,
-    fs::File,
+    fs::{self, File},
     io::{self, Read, Write},
-    path::Path,
+    path::{Path, PathBuf},
 };
 
+// CONSTANTS
+// ================================================================================================
+
+const REPO_PROTO_DIR: &str = "../../proto";
+const CRATE_PROTO_DIR: &str = "proto";
+
 const DOC_COMMENT: &str =
     "A list of tuples containing the names and contents of various protobuf files.";
 
+// BUILD SCRIPT
+// ================================================================================================
+
+/// Copies .proto files to the local directory and re-builds src/proto_files.rs file.
+/// 
+/// This is done only if BUILD_PROTO environment variable is set to `1` to avoid running the script
+/// on crates.io where repo-level .proto files are not available.
 fn main() -> io::Result<()> {
+    println!("cargo:rerun-if-changed=proto");
+    println!("cargo:rerun-if-changed=../../proto");
+
+    // skip this build script in BUILD_PROTO environment variable is not set to `1`
+    if env::var("BUILD_PROTO").unwrap_or("0".to_string()) == "0" {
+        return Ok(())
+    }
+
+    // copy all .proto files into this crate. all these files need to be local to the crate to
+    // publish the crate to crates.io
+    copy_proto_files()?;
+
     let out_dir = env::current_dir().expect("Error getting cwd");
     let dest_path = Path::new(&out_dir).join("./src/proto_files.rs");
     let mut file = File::create(dest_path)?;
 
-    let proto_dir = Path::new("../../proto");
-
     writeln!(file, "/// {DOC_COMMENT}")?;
     writeln!(file, "pub const PROTO_FILES: &[(&str, &str)] = &[")?;
 
-    for entry in std::fs::read_dir(proto_dir)? {
+    for entry in std::fs::read_dir(CRATE_PROTO_DIR)? {
         let entry = entry?;
         let path = entry.path();
         if path.is_file() {
@@ -29,7 +52,7 @@ fn main() -> io::Result<()> {
             File::open(&path)?.read_to_string(&mut file_content)?;
             writeln!(
                 file,
-                "    (\"{}\", include_str!(\"../../../proto/{}\")),",
+                "    (\"{}\", include_str!(\"../{CRATE_PROTO_DIR}/{}\")),",
                 file_name, file_name
             )?;
         }
@@ -39,3 +62,23 @@ fn main() -> io::Result<()> {
 
     Ok(())
 }
+
+// HELPER FUNCTIONS
+// ================================================================================================
+
+/// Copies all .proto files from the root proto directory to the proto directory of this crate.
+fn copy_proto_files() -> io::Result<()> {
+    let dest_dir: PathBuf = CRATE_PROTO_DIR.into();
+
+    fs::create_dir_all(dest_dir.clone())?;
+    for entry in fs::read_dir(REPO_PROTO_DIR)? {
+        let entry = entry?;
+        println!("{entry:?}");
+        let ty = entry.file_type()?;
+        if !ty.is_dir() {
+            fs::copy(entry.path(), dest_dir.join(entry.file_name()))?;
+        }
+    }
+
+    Ok(())
+}

crates/rpc-proto/proto/account.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+package account;
+
+import "digest.proto";
+
+message AccountId {
+    // A miden account is defined with a little bit of proof-of-work, the id itself is defined as
+    // the first word of a hash digest. For this reason account ids can be considered as random
+    // values, because of that the encoding bellow uses fixed 64 bits, instead of zig-zag encoding.
+    fixed64 id = 1;
+}
+
+message AccountSummary {
+    AccountId account_id = 1;
+    digest.Digest account_hash = 2;
+    uint32 block_num = 3;
+}
+
+message AccountInfo {
+    AccountSummary summary = 1;
+    optional bytes details = 2;
+}

crates/rpc-proto/proto/block_header.proto

@@ -0,0 +1,27 @@
+syntax = "proto3";
+package block_header;
+
+import "digest.proto";
+
+message BlockHeader {
+    // specifies the version of the protocol.
+    uint32 version = 1;
+    // the hash of the previous blocks header.
+    digest.Digest prev_hash = 2;
+    // a unique sequential number of the current block.
+    fixed32 block_num = 3;
+    // a commitment to an MMR of the entire chain where each block is a leaf.
+    digest.Digest chain_root = 4;
+    // a commitment to account database.
+    digest.Digest account_root = 5;
+    // a commitment to the nullifier database.
+    digest.Digest nullifier_root = 6;
+    // a commitment to all notes created in the current block.
+    digest.Digest note_root = 7;
+    // a commitment to a set of IDs of transactions which affected accounts in this block.
+    digest.Digest tx_hash = 8;
+    // a hash of a STARK proof attesting to the correct state transition.
+    digest.Digest proof_hash = 9;
+    // the time when the block was created.
+    fixed32 timestamp = 10;
+}

crates/rpc-proto/proto/block_producer.proto

@@ -0,0 +1,11 @@
+// Specification of the user facing gRPC API.
+syntax = "proto3";
+package block_producer;
+
+import "requests.proto";
+import "responses.proto";
+
+service Api {
+    rpc SubmitProvenTransaction(requests.SubmitProvenTransactionRequest) returns (responses.SubmitProvenTransactionResponse) {}
+}
+

crates/rpc-proto/proto/digest.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+package digest;
+
+// A hash digest, the result of a hash function.
+message Digest {
+    fixed64 d0 = 1;
+    fixed64 d1 = 2;
+    fixed64 d2 = 3;
+    fixed64 d3 = 4;
+}

crates/rpc-proto/proto/merkle.proto

@@ -0,0 +1,8 @@
+syntax = "proto3";
+package merkle;
+
+import "digest.proto";
+
+message MerklePath {
+    repeated digest.Digest siblings = 1;
+}

crates/rpc-proto/proto/mmr.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+package mmr;
+
+import "digest.proto";
+
+message MmrDelta {
+    uint64 forest = 1;
+    repeated digest.Digest data = 2;
+}

crates/rpc-proto/proto/note.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+package note;
+
+import "digest.proto";
+import "merkle.proto";
+import "account.proto";
+
+message NoteMetadata {
+    account.AccountId sender = 1;
+    uint32 note_type = 2;
+    fixed32 tag = 3;
+    fixed64 aux = 4;
+}
+
+message Note {
+    fixed32 block_num = 1;
+    uint32 note_index = 2;
+    digest.Digest note_id = 3;
+    NoteMetadata metadata = 4;
+    merkle.MerklePath merkle_path = 5;
+    // This field will be present when the note is on-chain.
+    // details contain the `Note` in a serialized format.
+    optional bytes details = 6;
+}
+
+message NoteSyncRecord {
+    uint32 note_index = 1;
+    digest.Digest note_id = 2;
+    NoteMetadata metadata = 3;
+    merkle.MerklePath merkle_path = 4;
+}

crates/rpc-proto/proto/requests.proto

@@ -0,0 +1,102 @@
+syntax = "proto3";
+package requests;
+
+import "account.proto";
+import "block_header.proto";
+import "digest.proto";
+import "note.proto";
+
+message ApplyBlockRequest {
+    bytes block = 1;
+}
+
+message CheckNullifiersRequest {
+    repeated digest.Digest nullifiers = 1;
+}
+
+// Returns the block header corresponding to the requested block number, as well as the merkle
+// path and current forest which validate the block's inclusion in the chain.
+//
+// The Merkle path is an MMR proof for the block's leaf, based on the current chain length.
+message GetBlockHeaderByNumberRequest {
+    // The block number of the target block.
+    //
+    // If not provided, means latest know block.
+    optional uint32 block_num = 1;
+    // Whether or not to return authentication data for the block header.
+    optional bool include_mmr_proof = 2;
+}
+
+// State synchronization request.
+//
+// Specifies state updates the client is intersted in. The server will return the first block which
+// contains a note matching `note_tags` or the chain tip. And the corresponding updates to
+// `nullifiers` and `account_ids` for that block range.
+message SyncStateRequest {
+    // Last block known by the client. The response will contain data starting from the next block,
+    // until the first block which contains a note of matching the requested tag, or the chain tip
+    // if there are no notes.
+    fixed32 block_num = 1;
+
+    // Accounts' hash to include in the response.
+    //
+    // An account hash will be included if-and-only-if it is the latest update. Meaning it is
+    // possible there was an update to the account for the given range, but if it is not the latest,
+    // it won't be included in the response.
+    repeated account.AccountId account_ids = 2;
+
+    // Determines the tags which the client is interested in. These are only the 16high bits of the
+    // note's complete tag.
+    //
+    // The above means it is not possible to request an specific note, but only a "note family",
+    // this is done to increase the privacy of the client, by hiding the note's the client is
+    // intereted on.
+    repeated uint32 note_tags = 3;
+
+    // Determines the nullifiers the client is interested in.
+    //
+    // Similarly to the note_tags, this determins only the 16high bits of the target nullifier.
+    repeated uint32 nullifiers = 4;
+}
+
+message GetBlockInputsRequest {
+    // ID of the account against which a transaction is executed.
+    repeated account.AccountId account_ids = 1;
+    // Array of nullifiers for all notes consumed by a transaction.
+    repeated digest.Digest nullifiers = 2;
+    // Array of note IDs to be checked for existence in the database.
+    repeated digest.Digest unauthenticated_notes = 3;
+}
+
+message GetTransactionInputsRequest {
+    account.AccountId account_id = 1;
+    repeated digest.Digest nullifiers = 2;
+    repeated digest.Digest unauthenticated_notes = 3;
+}
+
+message SubmitProvenTransactionRequest {
+    // Transaction encoded using miden's native format
+    bytes transaction = 1;
+}
+
+message GetNotesByIdRequest {
+    // List of NoteId's to be queried from the database
+    repeated digest.Digest note_ids = 1;
+}
+
+message ListNullifiersRequest {}
+
+message ListAccountsRequest {}
+
+message ListNotesRequest {}
+
+// Returns the latest state of an account with the specified ID.
+message GetAccountDetailsRequest {
+    // Account ID to get details.
+    account.AccountId account_id = 1;
+}
+
+message GetBlockByNumberRequest {
+    // The block number of the target block.
+    fixed32 block_num = 1;
+}