From 05543270246d3a96017719be4968d8d58daaacfb Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Tue, 16 Jan 2024 16:31:54 +0530 Subject: [PATCH 01/10] update Cargo.toml files to reference local crates --- host/Cargo.toml | 18 +++++++++--------- host/chain/Cargo.toml | 12 ++++++------ host/collection/Cargo.toml | 10 +++++----- host/fs/Cargo.toml | 14 +++++++------- host/fs/src/txn/mod.rs | 1 - host/generic/Cargo.toml | 2 +- host/scalar/Cargo.toml | 8 ++++---- host/state/Cargo.toml | 18 +++++++++--------- host/transact/Cargo.toml | 6 +++--- host/value/Cargo.toml | 4 ++-- 10 files changed, 46 insertions(+), 47 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index fd435747..add5ba4d 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -41,14 +41,14 @@ safecast = "0.2" serde = { version = "1.0", features = [] } serde_json = { version = "1.0" } tbon = "0.5" -tc-chain = "0.3" -tc-collection = "0.3" -tc-error = "0.10" -tc-fs = "0.3" -tc-scalar = "0.3" -tc-state = "0.3" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-chain = { path = "chain" } +tc-collection = { path = "collection" } +tc-error = { path = "error" } +tc-fs = { path = "fs" } +tc-scalar = { path = "scalar" } +tc-state = { path = "state" } +tc-transact = { path = "transact" } +tc-value = { path = "value" } +tcgeneric = { path = "generic" } tokio = { version = "1.35", features = ["rt-multi-thread", "signal"] } url = { version = "2.5" } diff --git a/host/chain/Cargo.toml b/host/chain/Cargo.toml index cf79c6d1..ffd64b45 100644 --- a/host/chain/Cargo.toml +++ b/host/chain/Cargo.toml @@ -21,9 +21,9 @@ log = { version = "0.4" } num_cpus = "1.16" safecast = "0.2" tbon = "0.5" -tc-collection = "0.3" -tc-error = "0.10" -tc-scalar = "0.3" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-collection = { path = "../collection" } +tc-error = { path = "../error" } +tc-scalar = { path = "../scalar" } +tc-transact = { path = "../transact" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } diff --git a/host/collection/Cargo.toml b/host/collection/Cargo.toml index 2eded07f..9580d0d2 100644 --- a/host/collection/Cargo.toml +++ b/host/collection/Cargo.toml @@ -32,9 +32,9 @@ smallvec = "1.11" pin-project = "1.1" rayon = "1.8" safecast = "0.2" -tc-error = "0.10" -tc-scalar = "0.3" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-error = { path = "../error" } +tc-scalar = { path = "../scalar" } +tc-transact = { path = "../transact" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } tokio = { version = "1.35", features = ["sync"] } diff --git a/host/fs/Cargo.toml b/host/fs/Cargo.toml index f630db27..cc68cfd7 100644 --- a/host/fs/Cargo.toml +++ b/host/fs/Cargo.toml @@ -25,12 +25,12 @@ num_cpus = "1.16" rjwt = "0.4" safecast = "0.2" tbon = "0.5" -tc-chain = "0.3" -tc-collection = "0.3" -tc-error = "0.10" -tc-scalar = "0.3" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-chain = { path = "../chain" } +tc-collection = { path = "../collection" } +tc-error = { path = "../error" } +tc-scalar = { path = "../scalar" } +tc-transact = { path = "../transact" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } tokio = { version = "1.35", features = ["rt-multi-thread", "signal"] } tokio-util = { version = "0.7", features = ["io"] } diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index 215a6deb..eeeb131e 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -21,7 +21,6 @@ use tcgeneric::{ use crate::block::CacheBlock; -pub use hypothetical::Hypothetical; pub use request::*; pub use server::*; pub use tc_transact::TxnId; diff --git a/host/generic/Cargo.toml b/host/generic/Cargo.toml index a081eeaf..58192eed 100644 --- a/host/generic/Cargo.toml +++ b/host/generic/Cargo.toml @@ -23,4 +23,4 @@ safecast = "0.2" serde = { version = "1.0", features = [] } sha2 = "0.10" smallvec = "1.11" -tc-error = "0.10" +tc-error = { path = "../error" } diff --git a/host/scalar/Cargo.toml b/host/scalar/Cargo.toml index 1627d629..cbb4892f 100644 --- a/host/scalar/Cargo.toml +++ b/host/scalar/Cargo.toml @@ -24,7 +24,7 @@ num_cpus = "1.13" safecast = "0.2" serde = { version = "1.0", features = [] } smallvec = "1.11" -tc-error = "0.10" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-error = { path = "../error" } +tc-transact = { path = "../transact" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } diff --git a/host/state/Cargo.toml b/host/state/Cargo.toml index 911d9ce9..906134fd 100644 --- a/host/state/Cargo.toml +++ b/host/state/Cargo.toml @@ -20,13 +20,13 @@ futures = "0.3" get-size = "0.1" get-size-derive = "0.1" log = "0.4" -num_cpus = "1.13" +num_cpus = "1.16" safecast = "0.2" -tc-chain = "0.3" -tc-collection = "0.3" -tc-error = "0.10" -tc-fs = "0.3" -tc-scalar = "0.3" -tc-transact = "0.19" -tc-value = "0.11" -tcgeneric = "0.9" +tc-chain = { path = "../chain" } +tc-collection = { path = "../collection" } +tc-error = { path = "../error" } +tc-fs = { path = "../fs" } +tc-scalar = { path = "../scalar" } +tc-transact = { path = "../transact" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } diff --git a/host/transact/Cargo.toml b/host/transact/Cargo.toml index 3bf4c6b6..134f5a49 100644 --- a/host/transact/Cargo.toml +++ b/host/transact/Cargo.toml @@ -24,9 +24,9 @@ num_cpus = "1.16" rand = "0.8" safecast = "0.2" sha2 = "0.10" -tc-error = "0.10" -tc-value = "0.11" -tcgeneric = "0.9" +tc-error = { path = "../error" } +tc-value = { path = "../value" } +tcgeneric = { path = "../generic" } tokio = { version = "1.35", features = ["sync"] } txfs = { version = "0.2", features = ["logging"] } txn_lock = { version = "0.8", features = ["logging"] } diff --git a/host/value/Cargo.toml b/host/value/Cargo.toml index 1b4ba94e..b26857d6 100644 --- a/host/value/Cargo.toml +++ b/host/value/Cargo.toml @@ -31,6 +31,6 @@ safecast = "0.2" serde = { version = "1.0", features = [] } serde_json = { version = "1.0", features = [] } smallvec = { version = "1.11", features = ["serde"] } -tcgeneric = "0.9" -tc-error = "0.10" +tc-error = { path = "../error" } +tcgeneric = { path = "../generic" } uuid = { version = "1.6", features = ["v4"] } From c0bc71de319088e0dcc53c095554f8473f38caa7 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Wed, 17 Jan 2024 10:43:38 +0530 Subject: [PATCH 02/10] remove unnecessary memory allocation from Gateway::new_txn --- host/fs/src/txn/mod.rs | 8 ++++---- host/fs/src/txn/request.rs | 8 ++------ host/fs/src/txn/server.rs | 2 +- host/src/gateway.rs | 9 +++------ host/src/lib.rs | 1 - 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index eeeb131e..5db59ffc 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -112,7 +112,7 @@ impl Active { #[derive(Clone)] pub struct Txn { active: Arc, - gateway: Arc>>, + gateway: Arc>, request: Arc, dir: DirLock, } @@ -120,7 +120,7 @@ pub struct Txn { impl Txn { fn new( active: Arc, - gateway: Arc>>, + gateway: Arc>, request: Request, ) -> Self { let request = Arc::new(request); @@ -166,7 +166,7 @@ where use rjwt::Resolve; let host = self.gateway.link(cluster_path); - let resolver = Resolver::new(&self.gateway, &host, self.request.txn_id()); + let resolver = Resolver::new(&*self.gateway, &host, self.request.txn_id()); debug!( "granting scopes {} to {}", @@ -387,7 +387,7 @@ where } } -impl Hash for Txn { +impl Hash for Txn { fn hash(&self, state: &mut H) { self.request.txn_id().hash(state) } diff --git a/host/fs/src/txn/request.rs b/host/fs/src/txn/request.rs index 9820c74b..ca7e4bec 100644 --- a/host/fs/src/txn/request.rs +++ b/host/fs/src/txn/request.rs @@ -72,18 +72,14 @@ impl Request { /// Struct responsible for resolving JWT auth identities (cf. the [`rjwt`] crate). pub struct Resolver<'a, State> { - gateway: &'a Box>, + gateway: &'a dyn Gateway, host: &'a Link, txn_id: &'a TxnId, } impl<'a, State> Resolver<'a, State> { /// Construct a new `Resolver`. - pub fn new( - gateway: &'a Box>, - host: &'a Link, - txn_id: &'a TxnId, - ) -> Self { + pub fn new(gateway: &'a dyn Gateway, host: &'a Link, txn_id: &'a TxnId) -> Self { Self { gateway, host, diff --git a/host/fs/src/txn/server.rs b/host/fs/src/txn/server.rs index b9dff9cb..3127b58a 100644 --- a/host/fs/src/txn/server.rs +++ b/host/fs/src/txn/server.rs @@ -36,7 +36,7 @@ impl TxnServer { /// Return the active `Txn` with the given [`TxnId`], or initiate a new [`Txn`]. pub async fn new_txn( &self, - gateway: Arc>>, + gateway: Arc>, txn_id: TxnId, token: (String, Claims), ) -> TCResult> { diff --git a/host/src/gateway.rs b/host/src/gateway.rs index e3c9d426..08d603c3 100644 --- a/host/src/gateway.rs +++ b/host/src/gateway.rs @@ -140,8 +140,7 @@ impl Gateway { let token = if let Some(token) = token { use rjwt::Resolve; - let gateway: Box> = Box::new(self.clone()); - Resolver::new(&gateway, &self.host().clone().into(), &txn_id) + Resolver::new(&self, &self.host().clone().into(), &txn_id) .consume_and_sign(&self.inner.actor, vec![], token, txn_id.time().into()) .map_err(|cause| unauthorized!("credential error").consume(cause)) .await? @@ -151,15 +150,13 @@ impl Gateway { let txn_server = self.inner.txn_server.clone(); - txn_server - .new_txn(Arc::new(Box::new(self)), txn_id, token) - .await + txn_server.new_txn(Arc::new(self), txn_id, token).await } /// Start this `Gateway`'s server pub fn listen(self) -> Pin> + 'static>> { let port = self.inner.config.http_port; - let server = crate::http::HTTPServer::new(self); + let server = http::HTTPServer::new(self); let listener = server.listen(port).map_err(|e| { let e: TokioError = Box::new(e); e diff --git a/host/src/lib.rs b/host/src/lib.rs index a54dfebb..bdaf0bb8 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -166,7 +166,6 @@ impl Builder { let txn_id = tc_transact::TxnId::new(gateway::Gateway::time()); let token = gateway.new_token(&txn_id).expect("token"); - let gateway: Box> = Box::new(gateway); let txn = txn_server .new_txn(Arc::new(gateway), txn_id, token) .await From 46cc9037b780462da99003d3438baa238ce601dd Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Wed, 17 Jan 2024 11:11:23 +0530 Subject: [PATCH 03/10] resolve a TODO on the method signature of Transation::subcontext --- host/chain/src/block.rs | 2 +- host/chain/src/data/history.rs | 4 ++-- host/fs/src/txn/mod.rs | 13 +++++-------- host/src/cluster/service.rs | 2 +- host/src/http/client.rs | 2 +- host/state/src/lib.rs | 2 +- host/transact/src/lib.rs | 6 +++--- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/host/chain/src/block.rs b/host/chain/src/block.rs index bd04101a..0cb33144 100644 --- a/host/chain/src/block.rs +++ b/host/chain/src/block.rs @@ -326,7 +326,7 @@ where let txn = self .txn - .subcontext(HISTORY.into()) + .subcontext(HISTORY) .map_err(de::Error::custom) .await?; diff --git a/host/chain/src/data/history.rs b/host/chain/src/data/history.rs index f9da7f00..c7814bd2 100644 --- a/host/chain/src/data/history.rs +++ b/host/chain/src/data/history.rs @@ -24,7 +24,7 @@ use tc_transact::lock::TxnLock; use tc_transact::public::{Public, Route, StateInstance}; use tc_transact::{AsyncHash, IntoView, Transact, Transaction, TxnId}; use tc_value::Value; -use tcgeneric::{label, Label, Map, TCBoxStream, TCBoxTryStream, ThreadSafe, Tuple}; +use tcgeneric::{label, Id, Label, Map, TCBoxStream, TCBoxTryStream, ThreadSafe, Tuple}; use crate::{null_hash, BLOCK_SIZE, CHAIN}; @@ -757,7 +757,7 @@ where .create_file(WRITE_AHEAD.into(), block, size_hint) .map_err(de::Error::custom)?; - let subcontext = |i: u64| self.txn.subcontext(i.into()).map_err(de::Error::custom); + let subcontext = |i: u64| self.txn.subcontext(Id::from(i)).map_err(de::Error::custom); let mut i = 0u64; let mut last_hash = null_hash.clone(); diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index 5db59ffc..ea8c791b 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -307,20 +307,17 @@ where #[async_trait] impl Transaction for Txn { #[inline] - fn id(&'_ self) -> &'_ TxnId { + fn id(&self) -> &TxnId { self.request.txn_id() } - fn context(&'_ self) -> &'_ tc_transact::fs::Inner { + fn context(&self) -> &tc_transact::fs::Inner { &self.dir } - // TODO: accept a ToOwned - async fn subcontext(&self, id: Id) -> TCResult { - let dir = { - let mut dir = self.dir.write().await; - dir.create_dir(id.to_string())? - }; + async fn subcontext + Send>(&self, id: I) -> TCResult { + let mut dir = self.dir.write().await; + let dir = dir.create_dir(id.into().to_string())?; Ok(Txn { active: self.active.clone(), diff --git a/host/src/cluster/service.rs b/host/src/cluster/service.rs index 6d4885c9..df657a4a 100644 --- a/host/src/cluster/service.rs +++ b/host/src/cluster/service.rs @@ -328,7 +328,7 @@ impl Replica for Service { let source = source.clone().append(number); // TODO: parallelize - let txn = txn.subcontext(number.into()).await?; + let txn = txn.subcontext(number).await?; version.replicate(&txn, source).await?; } diff --git a/host/src/http/client.rs b/host/src/http/client.rs index 4643936a..5b23a18b 100644 --- a/host/src/http/client.rs +++ b/host/src/http/client.rs @@ -145,7 +145,7 @@ impl crate::gateway::Client for Client { .header(hyper::header::CONTENT_TYPE, Encoding::Tbon.as_str()); let txn = txn.subcontext_unique().await?; - let subcontext = txn.subcontext(label("_params").into()).await?; + let subcontext = txn.subcontext(label("_params")).await?; let params_view = params.clone().into_view(subcontext).await?; let body = tbon::en::encode(params_view) .map_err(|cause| bad_request!("unable to encode stream").consume(cause))?; diff --git a/host/state/src/lib.rs b/host/state/src/lib.rs index 4bad7c38..57e87f00 100644 --- a/host/state/src/lib.rs +++ b/host/state/src/lib.rs @@ -1551,7 +1551,7 @@ impl<'a> de::Visitor for StateVisitor { loop { let txn = self .txn - .subcontext(i.into()) + .subcontext(i) .map_err(de::Error::custom) .await?; diff --git a/host/transact/src/lib.rs b/host/transact/src/lib.rs index 1969a0b1..f65bf602 100644 --- a/host/transact/src/lib.rs +++ b/host/transact/src/lib.rs @@ -87,14 +87,14 @@ pub trait Transact { #[async_trait] pub trait Transaction: Clone + Sized + Send + Sync + 'static { /// The [`TxnId`] of this transaction context. - fn id(&'_ self) -> &'_ TxnId; + fn id(&self) -> &TxnId; /// Allows locking the filesystem directory of this transaction context, /// e.g. to cache un-committed state or to compute an intermediate result. - fn context(&'_ self) -> &'_ DirLock; + fn context(&self) -> &DirLock; /// Create a new transaction context with the given `id`. - async fn subcontext(&self, id: Id) -> TCResult; + async fn subcontext + Send>(&self, id: I) -> TCResult; /// Create a new transaction subcontext with its own unique [`Dir`]. async fn subcontext_unique(&self) -> TCResult; From 253b4f965b665a614fffa80c3049d2546d13e308 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Wed, 17 Jan 2024 15:56:17 +0530 Subject: [PATCH 04/10] only create a transaction workspace directory when needed (don't lock the root workspace for every transaction) (#276) --- host/chain/src/block.rs | 6 +- host/chain/src/data/history.rs | 25 +++-- host/chain/src/sync.rs | 20 ++-- host/collection/src/btree/file.rs | 11 ++- host/collection/src/btree/public.rs | 10 +- host/collection/src/table/file.rs | 11 ++- host/collection/src/table/public.rs | 15 +-- host/collection/src/tensor/dense/base.rs | 7 +- host/collection/src/tensor/public.rs | 6 +- host/collection/src/tensor/sparse/base.rs | 5 +- host/collection/src/tensor/sparse/mod.rs | 2 +- host/fs/src/txn/mod.rs | 109 +++++++++++----------- host/fs/src/txn/server.rs | 27 ++---- host/src/cluster/service.rs | 4 +- host/src/http/client.rs | 8 +- host/src/http/server.rs | 4 +- host/state/src/lib.rs | 26 +----- host/transact/src/lib.rs | 6 +- tests/tctest/host/test_tensor.py | 1 + 19 files changed, 145 insertions(+), 158 deletions(-) diff --git a/host/chain/src/block.rs b/host/chain/src/block.rs index 0cb33144..593bece5 100644 --- a/host/chain/src/block.rs +++ b/host/chain/src/block.rs @@ -324,11 +324,7 @@ where let subject = seq.next_element::(self.txn.clone()).await?; let subject = subject.ok_or_else(|| de::Error::invalid_length(0, "a BlockChain schema"))?; - let txn = self - .txn - .subcontext(HISTORY) - .map_err(de::Error::custom) - .await?; + let txn = self.txn.subcontext(HISTORY); let history = seq.next_element::>(txn).await?; let history = diff --git a/host/chain/src/data/history.rs b/host/chain/src/data/history.rs index c7814bd2..1024ca50 100644 --- a/host/chain/src/data/history.rs +++ b/host/chain/src/data/history.rs @@ -24,7 +24,7 @@ use tc_transact::lock::TxnLock; use tc_transact::public::{Public, Route, StateInstance}; use tc_transact::{AsyncHash, IntoView, Transact, Transaction, TxnId}; use tc_value::Value; -use tcgeneric::{label, Id, Label, Map, TCBoxStream, TCBoxTryStream, ThreadSafe, Tuple}; +use tcgeneric::{label, Label, Map, TCBoxStream, TCBoxTryStream, ThreadSafe, Tuple}; use crate::{null_hash, BLOCK_SIZE, CHAIN}; @@ -720,14 +720,14 @@ where let null_hash = null_hash(); let txn_id = *self.txn.id(); + let cxt = self.txn.context().map_err(de::Error::custom).await?; + let store = { - let dir = self - .txn - .context() - .write() - .await - .create_dir(STORE.to_string()) - .map_err(de::Error::custom)?; + let dir = { + let mut cxt = cxt.write().await; + cxt.create_dir(STORE.to_string()) + .map_err(de::Error::custom)? + }; let dir = fs::Dir::load(txn_id, dir) .map_err(de::Error::custom) @@ -737,9 +737,8 @@ where }; let file = { - let mut dir = self.txn.context().write().await; - - dir.create_dir(CHAIN.to_string()) + let mut cxt = cxt.write().await; + cxt.create_dir(CHAIN.to_string()) .map_err(de::Error::custom)? }; @@ -757,12 +756,10 @@ where .create_file(WRITE_AHEAD.into(), block, size_hint) .map_err(de::Error::custom)?; - let subcontext = |i: u64| self.txn.subcontext(Id::from(i)).map_err(de::Error::custom); - let mut i = 0u64; let mut last_hash = null_hash.clone(); - while let Some(state) = seq.next_element::(subcontext(i).await?).await? { + while let Some(state) = seq.next_element::(self.txn.subcontext(i)).await? { let (hash, block_data): (Bytes, Map>) = state .try_cast_into(|s| de::Error::invalid_type(format!("{s:?}"), "a chain block"))?; diff --git a/host/chain/src/sync.rs b/host/chain/src/sync.rs index 6d6d89fe..d8913aa5 100644 --- a/host/chain/src/sync.rs +++ b/host/chain/src/sync.rs @@ -360,12 +360,15 @@ where ) -> Result { let subject = T::from_stream(txn.clone(), decoder).await?; - let mut context = txn.context().write().await; + let cxt = txn.context().map_err(de::Error::custom).await?; let store = { - let dir = context - .create_dir(STORE.to_string()) - .map_err(de::Error::custom)?; + let dir = { + let mut cxt = cxt.write().await; + + cxt.create_dir(STORE.to_string()) + .map_err(de::Error::custom)? + }; fs::Dir::load(*txn.id(), dir) .map_ok(super::data::Store::new) @@ -374,9 +377,12 @@ where }; let mut blocks_dir = { - let file = context - .create_dir(BLOCKS.to_string()) - .map_err(de::Error::custom)?; + let file = { + let mut cxt = cxt.write().await; + + cxt.create_dir(BLOCKS.to_string()) + .map_err(de::Error::custom)? + }; file.write_owned().await }; diff --git a/host/collection/src/btree/file.rs b/host/collection/src/btree/file.rs index bc27fd77..3a63f975 100644 --- a/host/collection/src/btree/file.rs +++ b/host/collection/src/btree/file.rs @@ -936,13 +936,14 @@ where let (canon, versions) = { trace!("lock txn context dir to create canon and versions directories..."); - let mut dir = self.txn.context().write().await; + let cxt = self.txn.context().map_err(de::Error::custom).await?; + let mut cxt = cxt.write().await; - let canon = dir + let canon = cxt .create_dir(CANON.to_string()) .map_err(de::Error::custom)?; - let versions = dir + let versions = cxt .create_dir(VERSIONS.to_string()) .map_err(de::Error::custom)?; @@ -1003,8 +1004,10 @@ where trace!("decoded BTreeFile"); + let dir = self.txn.context().map_err(de::Error::custom).await?; + Ok(BTreeFile { - dir: self.txn.context().clone(), + dir, state: Arc::new(RwLock::new(State { commits: OrdHashSet::with_capacity(0), deltas: OrdHashMap::with_capacity(0), diff --git a/host/collection/src/btree/public.rs b/host/collection/src/btree/public.rs index 9930e94e..94325a3b 100644 --- a/host/collection/src/btree/public.rs +++ b/host/collection/src/btree/public.rs @@ -63,8 +63,9 @@ where // let txn_id = *txn.id(); // // let store = { - // let mut dir = txn.context().write().await; - // let (_, cache) = dir.create_dir_unique()?; + // let cxt = txn.context().await?; + // let mut cxt = cxt.write().await; + // let (_, cache) = cxt.create_dir_unique()?; // Dir::load(*txn.id(), cache).await? // }; @@ -92,8 +93,9 @@ where Box::pin(async move { let schema = cast_into_schema(value)?; let store = { - let mut dir = txn.context().write().await; - let (_, cache) = dir.create_dir_unique()?; + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; + let (_, cache) = cxt.create_dir_unique()?; Dir::load(*txn.id(), cache).await? }; diff --git a/host/collection/src/table/file.rs b/host/collection/src/table/file.rs index f1706c38..16c01210 100644 --- a/host/collection/src/table/file.rs +++ b/host/collection/src/table/file.rs @@ -1102,13 +1102,14 @@ where trace!("decoded table schema: {:?}", schema); let (canon, versions) = { - let mut dir = self.txn.context().write().await; + let cxt = self.txn.context().map_err(de::Error::custom).await?; + let mut cxt = cxt.write().await; - let canon = dir + let canon = cxt .create_dir(CANON.to_string()) .map_err(de::Error::custom)?; - let versions = dir + let versions = cxt .create_dir(VERSIONS.to_string()) .map_err(de::Error::custom)?; @@ -1166,8 +1167,10 @@ where let collator = canon.collator().inner().clone(); let semaphore = Semaphore::with_reservation(txn_id, collator, Range::default()); + let dir = self.txn.context().map_err(de::Error::custom).await?; + Ok(TableFile { - dir: self.txn.context().clone(), + dir, state: Arc::new(RwLock::new(State { commits: OrdHashSet::with_capacity(0), deltas: OrdHashMap::with_capacity(0), diff --git a/host/collection/src/table/public.rs b/host/collection/src/table/public.rs index d1975207..f9c0b67b 100644 --- a/host/collection/src/table/public.rs +++ b/host/collection/src/table/public.rs @@ -64,8 +64,9 @@ where // params.expect_empty()?; // // let _store = { - // let mut context = txn.context().write().await; - // let (_, dir) = context.create_dir_unique()?; + // let cxt = txn.context().await?; + // let mut cxt = cxt.write().await; + // let (_, dir) = cxt.create_dir_unique()?; // Dir::load(*txn.id(), dir).await? // }; @@ -91,8 +92,9 @@ where let schema = TableSchema::try_cast_from_value(value)?; let store = { - let mut context = txn.context().write().await; - let (_, dir) = context.create_dir_unique()?; + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; + let (_, dir) = cxt.create_dir_unique()?; Dir::load(*txn.id(), dir).await? }; @@ -283,8 +285,9 @@ where txn: &Txn, ) -> TCResult> { let (_, tmp) = { - let mut context = txn.context().write().await; - context.create_dir_unique()? + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; + cxt.create_dir_unique()? }; b_tree::BTreeLock::create( diff --git a/host/collection/src/tensor/dense/base.rs b/host/collection/src/tensor/dense/base.rs index 83961b4f..8e22cc71 100644 --- a/host/collection/src/tensor/dense/base.rs +++ b/host/collection/src/tensor/dense/base.rs @@ -7,7 +7,7 @@ use collate::Collator; use destream::de; use ds_ext::{OrdHashMap, OrdHashSet}; use freqfs::{DirLock, FileWriteGuardOwned}; -use futures::{join, try_join}; +use futures::{join, try_join, TryFutureExt}; use ha_ndarray::{AccessBuf, Accessor, Array, Buffer, CType, Convert, PlatformInstance}; use log::{debug, trace, warn}; use rayon::prelude::*; @@ -680,7 +680,7 @@ where ) -> Result { let (txn, shape) = cxt; - let dir = txn.context().clone(); + let dir = txn.context().map_err(de::Error::custom).await?; let (canon, versions) = { let mut guard = dir.write().await; @@ -768,7 +768,8 @@ macro_rules! impl_from_stream_complex { let (re, im) = { let dir = { - let mut cxt = txn.context().write().await; + let cxt = txn.context().map_err(de::Error::custom).await?; + let mut cxt = cxt.write().await; let (_dir_name, dir) = cxt.create_dir_unique().map_err(de::Error::custom)?; dir diff --git a/host/collection/src/tensor/public.rs b/host/collection/src/tensor/public.rs index 9273a59b..7e4017e7 100644 --- a/host/collection/src/tensor/public.rs +++ b/host/collection/src/tensor/public.rs @@ -1868,7 +1868,8 @@ where }; let (_name, store) = { - let mut cxt = txn.context().write().await; + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; cxt.create_dir_unique()? }; @@ -1900,7 +1901,8 @@ where Txn: Transaction, FE: Clone + ThreadSafe, { - let mut cxt = txn.context().write().await; + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; let (_dir_name, dir) = cxt.create_dir_unique()?; fs::Dir::load(*txn.id(), dir).await } diff --git a/host/collection/src/tensor/sparse/base.rs b/host/collection/src/tensor/sparse/base.rs index 94095af5..ef23ef55 100644 --- a/host/collection/src/tensor/sparse/base.rs +++ b/host/collection/src/tensor/sparse/base.rs @@ -724,7 +724,7 @@ where ) -> Result { let (txn, shape) = cxt; - let dir = txn.context().clone(); + let dir = txn.context().map_err(de::Error::custom).await?; let (canon, versions) = { let mut dir = dir.write().await; @@ -761,7 +761,8 @@ where async fn new(txn: Txn, shape: Shape) -> TCResult { let (re, im) = { let dir = { - let mut cxt = txn.context().write().await; + let cxt = txn.context().await?; + let mut cxt = cxt.write().await; let (_dir_name, dir) = cxt.create_dir_unique()?; dir }; diff --git a/host/collection/src/tensor/sparse/mod.rs b/host/collection/src/tensor/sparse/mod.rs index bd0e948f..d949a339 100644 --- a/host/collection/src/tensor/sparse/mod.rs +++ b/host/collection/src/tensor/sparse/mod.rs @@ -1746,7 +1746,7 @@ where dtype: NumberType, shape: Shape, ) -> Result, E> { - let store = self.txn.context().clone(); + let store = self.txn.context().map_err(de::Error::custom).await?; let txn_id = *self.txn.id(); let store = fs::Dir::load(txn_id, store) diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index ea8c791b..e1e3f16f 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -3,21 +3,21 @@ use std::hash::{Hash, Hasher}; use std::iter::FromIterator; use std::ops::Deref; +use std::pin::Pin; use std::sync::Arc; use async_trait::async_trait; use freqfs::DirLock; -use futures::future::TryFutureExt; +use futures::future::{Future, TryFutureExt}; use log::debug; use safecast::CastInto; use tc_error::*; use tc_transact::public::StateInstance; use tc_transact::Transaction; +use tc_value::uuid::Uuid; use tc_value::{Host, Link, ToUrl, Value}; -use tcgeneric::{ - Id, NetworkTime, PathSegment, TCBoxFuture, TCBoxTryFuture, TCPathBuf, ThreadSafe, Tuple, -}; +use tcgeneric::{Id, PathSegment, TCBoxFuture, TCBoxTryFuture, TCPathBuf, ThreadSafe, Tuple}; use crate::block::CacheBlock; @@ -82,55 +82,67 @@ pub trait Gateway: ThreadSafe { fn finalize(&self, txn_id: TxnId) -> TCBoxFuture<()>; } -struct Active { - workspace: DirLock, - expires: NetworkTime, - scope: Scope, +#[derive(Clone)] +enum LazyDir { + Workspace(DirLock), + Lazy(Arc, Id), } -impl Active { - fn new(txn_id: &TxnId, workspace: DirLock, expires: NetworkTime) -> Self { - let scope = TCPathBuf::from(txn_id.to_id()); +impl LazyDir { + fn get_or_create<'a>( + &'a self, + txn_id: &'a TxnId, + ) -> Pin>> + Send + 'a>> { + Box::pin(async move { + match self { + Self::Workspace(workspace) => { + let mut parent = workspace.write().await; + parent.get_or_create_dir(txn_id.to_string()).map_err(TCError::from) + } + Self::Lazy(parent, name) => { + let parent = parent.get_or_create(txn_id).await?; + let mut parent = parent.write().await; - Self { - workspace, - expires, - scope, - } + parent + .get_or_create_dir(name.to_string()) + .map_err(TCError::from) + } + } + }) } - fn expires(&self) -> &NetworkTime { - &self.expires + fn create_dir(self, name: Id) -> Self { + Self::Lazy(Arc::new(self), name) } - fn scope(&self) -> &Scope { - &self.scope + fn create_dir_unique(self) -> Self { + Self::Lazy(Arc::new(self), Uuid::new_v4().into()) } } /// A transaction context. #[derive(Clone)] pub struct Txn { - active: Arc, gateway: Arc>, request: Arc, - dir: DirLock, + scope: TCPathBuf, + dir: LazyDir, } impl Txn { fn new( - active: Arc, + workspace: DirLock, gateway: Arc>, request: Request, ) -> Self { + let scope = request.txn_id().to_id().into(); let request = Arc::new(request); - let dir = active.workspace.clone(); Self { - active, gateway, request, - dir, + scope, + dir: LazyDir::Workspace(workspace), } } } @@ -180,10 +192,10 @@ where .await?; Ok(Self { - active: self.active.clone(), gateway: self.gateway.clone(), - dir: self.dir.clone(), request: Arc::new(Request::new(*txn_id, token, claims)), + scope: self.scope.clone(), + dir: self.dir.clone(), }) } @@ -200,7 +212,7 @@ where } if self.owner().is_none() { - self.grant(actor, cluster_path, vec![self.active.scope().clone()]) + self.grant(actor, cluster_path, vec![self.scope.clone()]) .await } else { Err(forbidden!( @@ -228,14 +240,12 @@ where /// Return the owner of this transaction, if there is one. pub fn owner(&self) -> Option<&Link> { - let active_scope = self.active.scope(); - self.request .scopes() .iter() .filter(|(_, actor_id, _)| *actor_id == &Value::None) .filter_map(|(host, _actor_id, scopes)| { - if scopes.contains(active_scope) { + if scopes.contains(&self.scope) { Some(host) } else { None @@ -280,21 +290,19 @@ where leader )) } else { - let scopes = vec![self.active.scope().clone()]; + let scopes = vec![self.scope.clone()]; self.grant(actor, cluster_path, scopes).await } } /// Return the leader of this transaction for the given cluster, if there is one. pub fn leader(&self, cluster_path: &[PathSegment]) -> Option<&Link> { - let active_scope = self.active.scope(); - self.request .scopes() .iter() .filter(|(_, actor_id, _)| *actor_id == &Value::None) .filter_map(|(host, _actor_id, scopes)| { - if scopes.contains(active_scope) && cluster_path.starts_with(host.path()) { + if scopes.contains(&self.scope) && cluster_path.starts_with(host.path()) { Some(host) } else { None @@ -311,31 +319,26 @@ impl Transaction for Txn { self.request.txn_id() } - fn context(&self) -> &tc_transact::fs::Inner { - &self.dir + async fn context(&self) -> TCResult> { + self.dir.get_or_create(self.request.txn_id()).await } - async fn subcontext + Send>(&self, id: I) -> TCResult { - let mut dir = self.dir.write().await; - let dir = dir.create_dir(id.into().to_string())?; - - Ok(Txn { - active: self.active.clone(), + fn subcontext + Send>(&self, id: I) -> Self { + Txn { gateway: self.gateway.clone(), request: self.request.clone(), - dir, - }) + scope: self.scope.clone(), + dir: self.dir.clone().create_dir(id.into()), + } } - async fn subcontext_unique(&self) -> TCResult { - let (_, subcontext) = self.dir.write().await.create_dir_unique()?; - - Ok(Self { - active: self.active.clone(), + fn subcontext_unique(&self) -> Self { + Txn { gateway: self.gateway.clone(), request: self.request.clone(), - dir: subcontext, - }) + scope: self.scope.clone(), + dir: self.dir.clone().create_dir_unique(), + } } } diff --git a/host/fs/src/txn/server.rs b/host/fs/src/txn/server.rs index 3127b58a..604e0ea8 100644 --- a/host/fs/src/txn/server.rs +++ b/host/fs/src/txn/server.rs @@ -15,12 +15,12 @@ use tcgeneric::NetworkTime; use crate::block::CacheBlock; use super::request::*; -use super::{Active, Gateway, Txn, TxnId}; +use super::{Gateway, Txn, TxnId}; /// Server to keep track of the transactions currently active for this host. #[derive(Clone)] pub struct TxnServer { - active: Arc>>>, + active: Arc>>, workspace: DirLock, } @@ -47,16 +47,14 @@ impl TxnServer { let mut active = self.active.write().await; match active.entry(txn_id) { - Entry::Occupied(entry) => { + Entry::Occupied(_entry) => { trace!("txn {} is already known", txn_id); - Ok(Txn::new(entry.get().clone(), gateway, request)) + Ok(Txn::new(self.workspace.clone(), gateway, request)) } Entry::Vacant(entry) => { trace!("creating new workspace for txn {}...", txn_id); - let workspace = self.txn_dir(txn_id).await?; - let active = Arc::new(Active::new(&txn_id, workspace.clone(), expires)); - let txn = Txn::new(active.clone(), gateway, request); - entry.insert(active); + let txn = Txn::new(self.workspace.clone(), gateway, request); + entry.insert(expires); Ok(txn) } } @@ -91,8 +89,7 @@ impl TxnServer { let expired = active .iter() - .filter(|(_, active)| active.expires() <= &now) - .map(|(txn_id, _)| txn_id) + .filter_map(|(txn_id, expires)| if expires <= &now { Some(txn_id) } else { None }) .copied() .collect::>(); @@ -121,14 +118,4 @@ impl TxnServer { workspace.sync().await.expect("sync workspace dir"); } - - async fn txn_dir(&self, txn_id: TxnId) -> TCResult> { - debug!("TxnServer::txn_dir"); - - let mut workspace = self.workspace.write().await; - - workspace - .create_dir(txn_id.to_string()) - .map_err(TCError::from) - } } diff --git a/host/src/cluster/service.rs b/host/src/cluster/service.rs index df657a4a..9c7cadc5 100644 --- a/host/src/cluster/service.rs +++ b/host/src/cluster/service.rs @@ -94,7 +94,7 @@ impl Replica for Version { for (name, attr) in self.attrs.iter() { if let Attr::Chain(chain) = attr { let source = source.clone().append(name.clone()); - let txn = txn.subcontext(name.clone()).await?; + let txn = txn.subcontext(name.clone()); // TODO: parallelize chain.replicate(&txn, source).await?; } @@ -328,7 +328,7 @@ impl Replica for Service { let source = source.clone().append(number); // TODO: parallelize - let txn = txn.subcontext(number).await?; + let txn = txn.subcontext(number); version.replicate(&txn, source).await?; } diff --git a/host/src/http/client.rs b/host/src/http/client.rs index 5b23a18b..534b3a1a 100644 --- a/host/src/http/client.rs +++ b/host/src/http/client.rs @@ -79,7 +79,7 @@ impl crate::gateway::Client for Client { let uri = build_url(&link, txn.id(), &key)?; let req = req_builder("GET", uri, Some(txn.request().token())); - let txn = txn.subcontext_unique().await?; + let txn = txn.subcontext_unique(); let response = self .client .request(req.body(Body::empty()).unwrap()) @@ -110,7 +110,7 @@ impl crate::gateway::Client for Client { let req = req_builder("PUT", uri, Some(txn.request().token())) .header(hyper::header::CONTENT_TYPE, Encoding::Tbon.as_str()); - let txn = txn.subcontext_unique().await?; + let txn = txn.subcontext_unique(); let view = value.into_view(txn).await?; let body = tbon::en::encode(view) .map_err(|cause| bad_request!("unable to encode stream").consume(cause))?; @@ -144,8 +144,8 @@ impl crate::gateway::Client for Client { let req = req_builder("POST", uri, Some(txn.request().token())) .header(hyper::header::CONTENT_TYPE, Encoding::Tbon.as_str()); - let txn = txn.subcontext_unique().await?; - let subcontext = txn.subcontext(label("_params")).await?; + let txn = txn.subcontext_unique(); + let subcontext = txn.subcontext(label("_params")); let params_view = params.clone().into_view(subcontext).await?; let body = tbon::en::encode(params_view) .map_err(|cause| bad_request!("unable to encode stream").consume(cause))?; diff --git a/host/src/http/server.rs b/host/src/http/server.rs index 8ae2a5d7..52b6931c 100644 --- a/host/src/http/server.rs +++ b/host/src/http/server.rs @@ -176,7 +176,7 @@ impl HTTPServer { &hyper::Method::PUT => { let key = get_param(&mut params, "key")?.unwrap_or_default(); - let sub_txn = txn.subcontext_unique().await?; + let sub_txn = txn.subcontext_unique(); let value = destream_body(http_request.into_body(), encoding, sub_txn).await?; let result = self .gateway @@ -191,7 +191,7 @@ impl HTTPServer { } &hyper::Method::POST => { - let sub_txn = txn.subcontext_unique().await?; + let sub_txn = txn.subcontext_unique(); let data = destream_body(http_request.into_body(), encoding, sub_txn).await?; self.gateway.post(txn, path.into(), data).await } diff --git a/host/state/src/lib.rs b/host/state/src/lib.rs index 57e87f00..8bdf44dc 100644 --- a/host/state/src/lib.rs +++ b/host/state/src/lib.rs @@ -1394,11 +1394,7 @@ impl StateVisitor { } StateType::Map => access.next_value(self.txn.clone()).await, StateType::Object(ot) => { - let txn = self - .txn - .subcontext_unique() - .map_err(de::Error::custom) - .await?; + let txn = self.txn.subcontext_unique(); let state = access.next_value(txn).await?; ObjectVisitor::new() @@ -1514,22 +1510,12 @@ impl<'a> de::Visitor for StateVisitor { let mut map = Map::new(); let id = Id::from_str(&key).map_err(de::Error::custom)?; - let txn = self - .txn - .subcontext(id.clone()) - .map_err(de::Error::custom) - .await?; - + let txn = self.txn.subcontext(id.clone()); let value = access.next_value(txn).await?; map.insert(id, value); while let Some(id) = access.next_key::(()).await? { - let txn = self - .txn - .subcontext(id.clone()) - .map_err(de::Error::custom) - .await?; - + let txn = self.txn.subcontext(id.clone()); let state = access.next_value(txn).await?; map.insert(id, state); } @@ -1549,11 +1535,7 @@ impl<'a> de::Visitor for StateVisitor { let mut i = 0usize; loop { - let txn = self - .txn - .subcontext(i) - .map_err(de::Error::custom) - .await?; + let txn = self.txn.subcontext(i); if let Some(next) = access.next_element(txn).await? { seq.push(next); diff --git a/host/transact/src/lib.rs b/host/transact/src/lib.rs index f65bf602..b01b449a 100644 --- a/host/transact/src/lib.rs +++ b/host/transact/src/lib.rs @@ -91,13 +91,13 @@ pub trait Transaction: Clone + Sized + Send + Sync + 'static { /// Allows locking the filesystem directory of this transaction context, /// e.g. to cache un-committed state or to compute an intermediate result. - fn context(&self) -> &DirLock; + async fn context(&self) -> TCResult>; /// Create a new transaction context with the given `id`. - async fn subcontext + Send>(&self, id: I) -> TCResult; + fn subcontext + Send>(&self, id: I) -> Self; /// Create a new transaction subcontext with its own unique [`Dir`]. - async fn subcontext_unique(&self) -> TCResult; + fn subcontext_unique(&self) -> Self; } /// A transactional remote procedure call client diff --git a/tests/tctest/host/test_tensor.py b/tests/tctest/host/test_tensor.py index e4457e62..9652b1fb 100644 --- a/tests/tctest/host/test_tensor.py +++ b/tests/tctest/host/test_tensor.py @@ -678,6 +678,7 @@ def testMul(self): expected = expected * np.arange(0, 3) self.assertEqual(actual, expect_sparse(tc.I64, [2, 3], expected)) + @unittest.skipIf("debug" in TC_PATH, "too slow for debug mode") def testSubAndSum(self): x = 300 y = 250 From f52d0dbe37a7e4ffe3498fccd56701467460aca6 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Thu, 18 Jan 2024 12:02:59 +0530 Subject: [PATCH 05/10] update TxnServer to register new transactions in a background thread --- host/fs/src/txn/mod.rs | 4 ++- host/fs/src/txn/server.rs | 71 +++++++++++++++++++++++++++++---------- host/src/gateway.rs | 6 ++-- host/src/lib.rs | 1 - 4 files changed, 60 insertions(+), 22 deletions(-) diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index e1e3f16f..37009700 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -97,7 +97,9 @@ impl LazyDir { match self { Self::Workspace(workspace) => { let mut parent = workspace.write().await; - parent.get_or_create_dir(txn_id.to_string()).map_err(TCError::from) + parent + .get_or_create_dir(txn_id.to_string()) + .map_err(TCError::from) } Self::Lazy(parent, name) => { let parent = parent.get_or_create(txn_id).await?; diff --git a/host/fs/src/txn/server.rs b/host/fs/src/txn/server.rs index 604e0ea8..3880c6db 100644 --- a/host/fs/src/txn/server.rs +++ b/host/fs/src/txn/server.rs @@ -1,13 +1,14 @@ //! A server to keep track of active transactions. +use std::cmp::Ordering; use std::collections::hash_map::{Entry, HashMap}; use std::convert::TryFrom; use std::sync::Arc; use freqfs::DirLock; use futures::future::TryFutureExt; -use log::{debug, trace}; -use tokio::sync::RwLock; +use log::*; +use tokio::sync::{mpsc, RwLock}; use tc_error::*; use tcgeneric::NetworkTime; @@ -22,19 +23,27 @@ use super::{Gateway, Txn, TxnId}; pub struct TxnServer { active: Arc>>, workspace: DirLock, + tx: mpsc::UnboundedSender<(TxnId, NetworkTime)>, } impl TxnServer { /// Construct a new `TxnServer`. pub async fn new(workspace: DirLock) -> Self { - Self { + let (tx, rx) = mpsc::unbounded_channel(); + + let server = Self { active: Arc::new(RwLock::new(HashMap::new())), workspace, - } + tx, + }; + + spawn_receiver_thread(server.clone(), rx); + + server } /// Return the active `Txn` with the given [`TxnId`], or initiate a new [`Txn`]. - pub async fn new_txn( + pub fn new_txn( &self, gateway: Arc>, txn_id: TxnId, @@ -44,20 +53,12 @@ impl TxnServer { let expires = NetworkTime::try_from(token.1.expires())?; let request = Request::new(txn_id, token.0, token.1); - let mut active = self.active.write().await; - match active.entry(txn_id) { - Entry::Occupied(_entry) => { - trace!("txn {} is already known", txn_id); - Ok(Txn::new(self.workspace.clone(), gateway, request)) - } - Entry::Vacant(entry) => { - trace!("creating new workspace for txn {}...", txn_id); - let txn = Txn::new(self.workspace.clone(), gateway, request); - entry.insert(expires); - Ok(txn) - } - } + self.tx + .send((txn_id, expires)) + .map_err(|cause| internal!("transaction queue failure: {cause}"))?; + + Ok(Txn::new(self.workspace.clone(), gateway, request)) } /// Gracefully shut down this `TxnServer` by allowing all active transactions to drain. @@ -119,3 +120,37 @@ impl TxnServer { workspace.sync().await.expect("sync workspace dir"); } } + +fn spawn_receiver_thread(server: TxnServer, mut rx: mpsc::UnboundedReceiver<(TxnId, NetworkTime)>) { + let new_txn = |active: &mut HashMap, txn_id, expires| { + match active.entry(txn_id) { + Entry::Occupied(mut entry) => { + trace!("txn {} is already known", txn_id); + + match entry.get().cmp(&expires) { + Ordering::Greater | Ordering::Equal => { + // no-op + } + Ordering::Less => { + warn!("expiration time of txn {txn_id} extended"); + entry.insert(expires); + } + } + } + Entry::Vacant(entry) => { + entry.insert(expires); + } + } + }; + + tokio::spawn(async move { + while let Some((txn_id, expires)) = rx.recv().await { + let mut active = server.active.write().await; + new_txn(&mut *active, txn_id, expires); + + while let Ok((txn_id, expires)) = rx.try_recv() { + new_txn(&mut *active, txn_id, expires); + } + } + }); +} diff --git a/host/src/gateway.rs b/host/src/gateway.rs index 08d603c3..88bf0dbc 100644 --- a/host/src/gateway.rs +++ b/host/src/gateway.rs @@ -8,7 +8,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::future::{Future, TryFutureExt}; use log::debug; -use tokio::time::Duration; +use tokio::time::{Duration, MissedTickBehavior}; use tc_error::*; use tc_fs::{Actor, Claims, Gateway as GatewayInstance, Resolver, Token, TxnServer}; @@ -140,6 +140,7 @@ impl Gateway { let token = if let Some(token) = token { use rjwt::Resolve; + // TODO: don't require resolving an actor or validating the existing token in order to create a new token (just validate on receipt) Resolver::new(&self, &self.host().clone().into(), &txn_id) .consume_and_sign(&self.inner.actor, vec![], token, txn_id.time().into()) .map_err(|cause| unauthorized!("credential error").consume(cause)) @@ -150,7 +151,7 @@ impl Gateway { let txn_server = self.inner.txn_server.clone(); - txn_server.new_txn(Arc::new(self), txn_id, token).await + txn_server.new_txn(Arc::new(self), txn_id, token) } /// Start this `Gateway`'s server @@ -264,6 +265,7 @@ impl GatewayInstance for Gateway { fn spawn_cleanup_thread(gateway: Gateway) { let mut interval = tokio::time::interval(INTERVAL); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); tokio::spawn(async move { loop { diff --git a/host/src/lib.rs b/host/src/lib.rs index bdaf0bb8..cf33bac4 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -168,7 +168,6 @@ impl Builder { let txn = txn_server .new_txn(Arc::new(gateway), txn_id, token) - .await .expect("transaction"); // no need to claim ownership of this txn since there's no way to make outbound requests From 98dcdab8352b3e048c5837e297ffe9557e03b228 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Tue, 23 Jan 2024 16:02:04 +0530 Subject: [PATCH 06/10] upgrade to txn_lock v0.9 (#277) * upgrade to txn_lock v0.9 * replace pending chain blocks with a transactional task queue * use a priority queue to make sure only the oldest expired transactions are finalized --- host/chain/src/block.rs | 22 ++- host/chain/src/data/block.rs | 52 ++++++- host/chain/src/data/history.rs | 172 ++++++++++----------- host/chain/src/data/mod.rs | 2 +- host/chain/src/data/store.rs | 6 +- host/chain/src/lib.rs | 63 ++++++-- host/chain/src/public.rs | 9 +- host/chain/src/sync.rs | 127 +++++++-------- host/collection/src/tensor/dense/access.rs | 10 +- host/error/Cargo.toml | 4 +- host/error/src/lib.rs | 2 + host/fs/Cargo.toml | 1 + host/fs/src/txn/server.rs | 103 +++++++----- host/generic/src/time.rs | 2 +- host/transact/Cargo.toml | 4 +- host/transact/src/lib.rs | 15 +- 16 files changed, 343 insertions(+), 251 deletions(-) diff --git a/host/chain/src/block.rs b/host/chain/src/block.rs index 593bece5..c8ffc6f4 100644 --- a/host/chain/src/block.rs +++ b/host/chain/src/block.rs @@ -48,7 +48,6 @@ where } } -#[async_trait] impl ChainInstance for BlockChain where State: StateInstance, @@ -61,12 +60,12 @@ where Collection: TryCastFrom, Scalar: TryCastFrom, { - async fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { - self.history.append_delete(txn_id, key).await + fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { + self.history.append_delete(txn_id, key) } - async fn append_put(&self, txn: &State::Txn, key: Value, value: State) -> TCResult<()> { - self.history.append_put(txn, key, value).await + fn append_put(&self, txn: State::Txn, key: Value, value: State) -> TCResult<()> { + self.history.append_put(txn, key, value) } fn subject(&self) -> &T { @@ -78,7 +77,12 @@ where impl fs::Persist for BlockChain where State: StateInstance, - State::FE: AsType + ThreadSafe + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + ThreadSafe + + for<'a> fs::FileSave<'a>, T: Route + fs::Persist + fmt::Debug, { type Txn = State::Txn; @@ -168,7 +172,11 @@ where impl fs::CopyFrom for BlockChain where State: StateInstance, - State::FE: AsType + ThreadSafe + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, T: Route + fs::Persist + fmt::Debug, { async fn copy_from( diff --git a/host/chain/src/data/block.rs b/host/chain/src/data/block.rs index 3889bd4d..841ecb19 100644 --- a/host/chain/src/data/block.rs +++ b/host/chain/src/data/block.rs @@ -14,6 +14,13 @@ use tc_scalar::Scalar; use tc_transact::TxnId; use tc_value::Value; +use super::StoreEntry; + +pub enum MutationPending { + Delete(Value), + Put(Txn, Value, StoreEntry), +} + #[derive(Clone, Eq, PartialEq)] pub enum MutationRecord { Delete(Value), @@ -110,6 +117,20 @@ impl GetSize for ChainBlock { } impl ChainBlock { + fn txn_hash<'a, R>(txn_id: &TxnId, mutations: R) -> Output + where + R: IntoIterator, + { + let mut mutation_hasher = Sha256::new(); + mutation_hasher.update(Hash::::hash(txn_id)); + + for mutation in mutations { + mutation_hasher.update(Hash::::hash(mutation)); + } + + mutation_hasher.finalize() + } + /// Compute the hash of a [`ChainBlock`] with the given contents pub fn hash<'a, M, R>(last_hash: &'a [u8], mutations: M) -> Output where @@ -121,16 +142,35 @@ impl ChainBlock { let mut txn_hasher = Sha256::new(); for (txn_id, mutations) in mutations { - let mut mutation_hasher = Sha256::new(); - mutation_hasher.update(Hash::::hash(txn_id)); + txn_hasher.update(Self::txn_hash(txn_id, mutations)); + } - for mutation in mutations { - mutation_hasher.update(Hash::::hash(mutation)); - } + hasher.update(txn_hasher.finalize()); + hasher.finalize() + } - txn_hasher.update(mutation_hasher.finalize()); + /// Compute the hash of a [`ChainBlock`] with the given contents and pending contents + pub fn pending_hash<'a, M, R, P>( + last_hash: &'a [u8], + mutations: M, + txn_id: &TxnId, + pending: P, + ) -> Output + where + M: IntoIterator + 'a, + &'a R: IntoIterator + 'a, + P: IntoIterator + 'a, + { + let mut hasher = Sha256::new(); + hasher.update(last_hash); + + let mut txn_hasher = Sha256::new(); + for (txn_id, mutations) in mutations { + txn_hasher.update(Self::txn_hash(txn_id, mutations)); } + txn_hasher.update(Self::txn_hash(txn_id, pending)); + hasher.update(txn_hasher.finalize()); hasher.finalize() } diff --git a/host/chain/src/data/history.rs b/host/chain/src/data/history.rs index 1024ca50..8171d328 100644 --- a/host/chain/src/data/history.rs +++ b/host/chain/src/data/history.rs @@ -20,22 +20,22 @@ use tc_collection::Collection; use tc_error::*; use tc_scalar::Scalar; use tc_transact::fs; -use tc_transact::lock::TxnLock; +use tc_transact::lock::{TxnLock, TxnTaskQueue}; use tc_transact::public::{Public, Route, StateInstance}; use tc_transact::{AsyncHash, IntoView, Transact, Transaction, TxnId}; use tc_value::Value; use tcgeneric::{label, Label, Map, TCBoxStream, TCBoxTryStream, ThreadSafe, Tuple}; -use crate::{null_hash, BLOCK_SIZE, CHAIN}; +use crate::{new_queue, null_hash, BLOCK_SIZE, CHAIN}; -use super::block::{ChainBlock, MutationRecord}; +use super::block::{ChainBlock, MutationPending, MutationRecord}; use super::store::{Store, StoreEntry, StoreEntryView}; const STORE: Label = label("store"); -const PENDING: &str = "pending"; const WRITE_AHEAD: &str = "write_ahead"; pub struct History { + queue: TxnTaskQueue, TCResult>, file: DirLock, store: Store, latest: TxnLock, @@ -45,6 +45,7 @@ pub struct History { impl Clone for History { fn clone(&self) -> Self { Self { + queue: self.queue.clone(), file: self.file.clone(), store: self.store.clone(), latest: self.latest.clone(), @@ -56,7 +57,11 @@ impl Clone for History { impl History where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, { fn new( file: DirLock, @@ -66,23 +71,34 @@ where ) -> Self { debug_assert!(file.try_read().expect("history").contains(&latest)); + let queue = new_queue::(store.clone()); + Self { + queue, file, store, latest: TxnLock::new(latest), cutoff: TxnLock::new(cutoff), } } +} +impl History { pub fn store(&self) -> &Store { &self.store } +} - pub async fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { +impl History +where + State: StateInstance, + State::FE: AsType + for<'a> fs::FileSave<'a>, +{ + pub fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { debug!("History::append_delete {} {}", txn_id, key); - let mut block = self.write_pending().await?; - block.append_delete(txn_id, key); - Ok(()) + self.queue + .push(txn_id, MutationPending::Delete(key)) + .map_err(TCError::from) } async fn read_block( @@ -109,28 +125,6 @@ where block.write_owned().map_err(TCError::from).await } - pub async fn read_pending( - &self, - ) -> TCResult> { - let file = self.file.read().await; - let block: &FileLock = file - .get_file(PENDING) - .ok_or_else(|| internal!("BlockChain is missing its pending block"))?; - - block.read_owned().map_err(TCError::from).await - } - - pub async fn write_pending( - &self, - ) -> TCResult> { - let file = self.file.read().await; - let block: &FileLock = file - .get_file(PENDING) - .ok_or_else(|| internal!("BlockChain is missing its pending block"))?; - - block.write_owned().map_err(TCError::from).await - } - pub async fn read_log(&self) -> TCResult> { let log: FileLock = { let file = self.file.read().await; @@ -141,28 +135,34 @@ where } pub async fn write_ahead(&self, txn_id: TxnId) { + let handles = self.queue.commit(txn_id).await; + + let mutations = handles + .into_iter() + .collect::>>() + .expect("mutations"); + + if mutations.is_empty() { + return; + } + self.store.commit(txn_id).await; let file = self.file.read().await; - let pending: &FileLock = file.get_file(PENDING).expect("pending transactions"); - let mut pending: FileWriteGuard = - pending.write().await.expect("pending block write lock"); - if let Some(mutations) = pending.mutations.remove(&txn_id) { - let write_ahead: &FileLock = - file.get_file(WRITE_AHEAD).expect("write-ahead log"); + let write_ahead: &FileLock = + file.get_file(WRITE_AHEAD).expect("write-ahead log"); - { - let mut write_ahead: FileWriteGuard = write_ahead - .write() - .await - .expect("write-ahead log write lock"); - - write_ahead.mutations.insert(txn_id, mutations); - } + { + let mut write_ahead: FileWriteGuard = write_ahead + .write() + .await + .expect("write-ahead log write lock"); - write_ahead.sync().await.expect("sync write-ahead log"); + write_ahead.mutations.insert(txn_id, mutations); } + + write_ahead.sync().await.expect("sync write-ahead log"); } } @@ -175,19 +175,18 @@ where + AsType + for<'a> fs::FileSave<'a>, { - pub async fn append_put(&self, txn: &State::Txn, key: Value, value: State) -> TCResult<()> + pub fn append_put(&self, txn: State::Txn, key: Value, value: State) -> TCResult<()> where Collection: TryCastFrom, Scalar: TryCastFrom, { - let value = StoreEntry::try_from_state(value)?; - let value = self.store.save_state(txn, value).await?; - debug!("History::append_put {} {} {:?}", txn.id(), key, value); - let mut block = self.write_pending().await?; - block.append_put(*txn.id(), key, value); - Ok(()) + let value = StoreEntry::try_from_state(value)?; + + self.queue + .push(*txn.id(), MutationPending::Put(txn, key, value)) + .map_err(TCError::from) } pub async fn replicate(&self, txn: &State::Txn, subject: &T, other: Self) -> TCResult<()> @@ -323,7 +322,11 @@ where impl fs::Persist for History where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, { type Txn = State::Txn; type Schema = (); @@ -350,7 +353,6 @@ where let cutoff = txn_id; let latest = 0; - create_block(&mut file_lock, PENDING)?; create_block(&mut file_lock, WRITE_AHEAD)?; create_block(&mut file_lock, latest)?; @@ -377,7 +379,6 @@ where let mut cutoff = txn_id; let mut latest = 0; - get_or_create_block(&mut file_lock, PENDING.to_string())?; get_or_create_block(&mut file_lock, WRITE_AHEAD.to_string())?; let mut last_hash = Bytes::from(null_hash().to_vec()); @@ -459,15 +460,34 @@ where Ok(ChainBlock::hash(latest_block.last_hash(), mutations)) } else { - let pending = self.read_pending().await?; - if let Some(mutations) = pending.mutations.get(&txn_id) { + let pending = self.queue.peek(&txn_id).await?; + + if let Some(pending_mutations) = pending { + if let Some(err) = pending_mutations + .iter() + .map(Result::as_ref) + .filter_map(Result::err) + .next() + { + return Err(err.clone()); + } + + let pending_mutations = pending_mutations + .iter() + .map(Result::as_ref) + .filter_map(Result::ok); + let mutations = latest_block .mutations .iter() - .take_while(|(past_txn_id, _)| *past_txn_id <= &txn_id) - .chain(iter::once((&txn_id, mutations))); - - Ok(ChainBlock::hash(latest_block.last_hash(), mutations)) + .take_while(|(past_txn_id, _)| *past_txn_id <= &txn_id); + + Ok(ChainBlock::pending_hash( + latest_block.last_hash(), + mutations, + &txn_id, + pending_mutations, + )) } else { // TODO: validate the length of the hash before calling clone_from_slice Ok(GenericArray::clone_from_slice(latest_block.last_hash())) @@ -575,35 +595,17 @@ where } async fn rollback(&self, txn_id: &TxnId) { - let file = self.file.read().await; - let mut pending: FileWriteGuard = file - .get_file(PENDING) - .expect("pending transactions") - .write() - .await - .expect("pending transaction lock"); - - pending.mutations.remove(txn_id); - self.latest.rollback(txn_id); self.cutoff.rollback(txn_id); + self.queue.rollback(txn_id); self.store.rollback(txn_id).await; } async fn finalize(&self, txn_id: &TxnId) { self.latest.finalize(*txn_id); self.cutoff.finalize(*txn_id); + self.queue.finalize(*txn_id); self.store.finalize(txn_id).await; - - let file = self.file.read().await; - let mut pending: FileWriteGuard = file - .get_file(PENDING) - .expect("pending transactions") - .write() - .await - .expect("pending transaction lock"); - - pending.mutations.remove(txn_id); } } @@ -744,12 +746,6 @@ where let mut guard = file.write().await; - let block = ChainBlock::new(null_hash.to_vec()); - let size_hint = block.get_size(); - guard - .create_file(PENDING.into(), block, size_hint) - .map_err(de::Error::custom)?; - let block = ChainBlock::new(null_hash.to_vec()); let size_hint = block.get_size(); guard diff --git a/host/chain/src/data/mod.rs b/host/chain/src/data/mod.rs index 962a2dc0..e6f42a5d 100644 --- a/host/chain/src/data/mod.rs +++ b/host/chain/src/data/mod.rs @@ -9,7 +9,7 @@ use tc_scalar::Scalar; use tc_transact::public::{Public, Route, StateInstance}; use tc_transact::{Transaction, TxnId}; -pub use block::{ChainBlock, MutationRecord}; +pub use block::{ChainBlock, MutationPending, MutationRecord}; pub use history::{History, HistoryView}; pub(super) use store::{Store, StoreEntry}; diff --git a/host/chain/src/data/store.rs b/host/chain/src/data/store.rs index e12acfda..3dd64f59 100644 --- a/host/chain/src/data/store.rs +++ b/host/chain/src/data/store.rs @@ -1,10 +1,11 @@ +use std::fmt; +use std::marker::PhantomData; + use async_trait::async_trait; -use core::fmt; use destream::en; use futures::future::TryFutureExt; use log::debug; use safecast::*; -use std::marker::PhantomData; use tc_collection::btree::Node as BTreeNode; use tc_collection::tensor::{DenseCacheFile, Node as TensorNode}; @@ -225,7 +226,6 @@ where async fn commit(&self, txn_id: TxnId) -> Self::Commit { debug!("commit chain data store at {}", txn_id); - self.dir.commit(txn_id, true).await } diff --git a/host/chain/src/lib.rs b/host/chain/src/lib.rs index 00bf5e3e..513d37ec 100644 --- a/host/chain/src/lib.rs +++ b/host/chain/src/lib.rs @@ -2,6 +2,7 @@ use std::fmt; use std::marker::PhantomData; +use std::sync::Arc; use async_hash::generic_array::GenericArray; use async_hash::{Output, Sha256}; @@ -17,11 +18,14 @@ use tc_collection::Collection; use tc_error::*; use tc_scalar::Scalar; use tc_transact::fs; +use tc_transact::lock::TxnTaskQueue; use tc_transact::public::{Route, StateInstance}; use tc_transact::{AsyncHash, IntoView, Transact, Transaction, TxnId}; use tc_value::Value; use tcgeneric::*; +use data::{MutationPending, MutationRecord}; + pub use block::BlockChain; pub use data::ChainBlock; pub use sync::SyncChain; @@ -37,6 +41,11 @@ pub const HISTORY: Label = label(".history"); const BLOCK_SIZE: usize = 1_000_000; // TODO: reduce to 4,096 const PREFIX: PathLabel = path_label(&["state", "chain"]); +/// A block in a file managed by a [`Chain`] +pub trait ChainCacheFile: DenseCacheFile + AsType + AsType {} + +impl ChainCacheFile for FE where FE: DenseCacheFile + AsType + AsType {} + /// Defines a method to recover the state of this [`Chain`] from a transaction failure. #[async_trait] pub trait Recover { @@ -46,14 +55,13 @@ pub trait Recover { async fn recover(&self, txn: &Self::Txn) -> TCResult<()>; } -/// Methods common to any instance of a [`Chain`], such as a [`SyncChain`]. -#[async_trait] +/// Methods common to any type of [`Chain`]. pub trait ChainInstance { /// Append the given DELETE op to the latest block in this `Chain`. - async fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()>; + fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()>; /// Append the given PUT op to the latest block in this `Chain`. - async fn append_put(&self, txn: &State::Txn, key: Value, value: State) -> TCResult<()>; + fn append_put(&self, txn: State::Txn, key: Value, value: State) -> TCResult<()>; /// Borrow the subject of this [`Chain`]. fn subject(&self) -> &T; @@ -128,7 +136,6 @@ where } } -#[async_trait] impl ChainInstance for Chain where State: StateInstance, @@ -141,17 +148,17 @@ where Collection: TryCastFrom, Scalar: TryCastFrom, { - async fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { + fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { match self { - Self::Block(chain) => chain.append_delete(txn_id, key).await, - Self::Sync(chain) => chain.append_delete(txn_id, key).await, + Self::Block(chain) => chain.append_delete(txn_id, key), + Self::Sync(chain) => chain.append_delete(txn_id, key), } } - async fn append_put(&self, txn: &State::Txn, key: Value, value: State) -> TCResult<()> { + fn append_put(&self, txn: State::Txn, key: Value, value: State) -> TCResult<()> { match self { - Self::Block(chain) => chain.append_put(txn, key, value).await, - Self::Sync(chain) => chain.append_put(txn, key, value).await, + Self::Block(chain) => chain.append_put(txn, key, value), + Self::Sync(chain) => chain.append_put(txn, key, value), } } @@ -241,7 +248,11 @@ where impl fs::Persist for Chain where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, T: fs::Persist + Route + fmt::Debug, { type Txn = State::Txn; @@ -300,7 +311,11 @@ where impl fs::CopyFrom for Chain where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, T: fs::Persist + Route + fmt::Debug, { async fn copy_from( @@ -525,6 +540,28 @@ where } } +fn new_queue( + store: data::Store, +) -> TxnTaskQueue, TCResult> +where + State: StateInstance, + State::FE: DenseCacheFile + AsType + AsType + Clone, +{ + TxnTaskQueue::new(Arc::pin(move |mutation| { + let store = store.clone(); + + Box::pin(async move { + match mutation { + MutationPending::Delete(key) => Ok(MutationRecord::Delete(key)), + MutationPending::Put(txn, key, state) => { + let value = store.save_state(&txn, state).await?; + Ok(MutationRecord::Put(key, value)) + } + } + }) + })) +} + #[inline] pub fn null_hash() -> Output { GenericArray::default() diff --git a/host/chain/src/public.rs b/host/chain/src/public.rs index 50d273f8..c00be9c0 100644 --- a/host/chain/src/public.rs +++ b/host/chain/src/public.rs @@ -59,11 +59,8 @@ where Some(put_handler) => Some(Box::new(|txn, key, value| { Box::pin(async move { debug!("Chain::put {} <- {:?}", key, value); - self.chain - .append_put(txn, key.clone(), value.clone()) - .await?; - + .append_put(txn.clone(), key.clone(), value.clone())?; put_handler(txn, key, value).await }) })), @@ -90,9 +87,7 @@ where Some(delete_handler) => Some(Box::new(|txn, key| { Box::pin(async move { debug!("Chain::delete {}", key); - - self.chain.append_delete(*txn.id(), key.clone()).await?; - + self.chain.append_delete(*txn.id(), key.clone())?; delete_handler(txn, key).await }) })), diff --git a/host/chain/src/sync.rs b/host/chain/src/sync.rs index d8913aa5..ec120d7b 100644 --- a/host/chain/src/sync.rs +++ b/host/chain/src/sync.rs @@ -7,8 +7,7 @@ use async_hash::{Output, Sha256}; use async_trait::async_trait; use destream::{de, FromStream}; use freqfs::{FileLock, FileWriteGuard}; -use futures::future::TryFutureExt; -use futures::try_join; +use futures::TryFutureExt; use get_size::GetSize; use log::{debug, trace}; use safecast::{AsType, TryCastFrom, TryCastInto}; @@ -19,18 +18,18 @@ use tc_collection::Collection; use tc_error::*; use tc_scalar::Scalar; use tc_transact::fs; +use tc_transact::lock::TxnTaskQueue; use tc_transact::public::{Route, StateInstance}; use tc_transact::{AsyncHash, IntoView, RPCClient, Transact, Transaction, TxnId}; use tc_value::{Link, Value}; use tcgeneric::{label, Label}; -use crate::data::StoreEntry; +use crate::data::{MutationPending, MutationRecord, StoreEntry}; -use super::{null_hash, ChainBlock, ChainInstance, Recover}; +use super::{new_queue, null_hash, ChainBlock, ChainInstance, Recover}; const BLOCKS: Label = label(".blocks"); const COMMITTED: &str = "committed.chain_block"; -const PENDING: &str = "pending.chain_block"; const STORE: Label = label(".store"); /// A [`super::Chain`] which keeps only the data needed to recover the state of its subject in the @@ -38,7 +37,7 @@ const STORE: Label = label(".store"); #[derive(Clone)] pub struct SyncChain { committed: FileLock, - pending: FileLock, + queue: TxnTaskQueue, TCResult>, store: super::data::Store, subject: T, } @@ -51,20 +50,27 @@ where async fn write_ahead(&self, txn_id: TxnId) { trace!("SyncChain::write_ahead {}", txn_id); + let handles = self.queue.commit(txn_id).await; + + let mutations = handles + .into_iter() + .collect::>>() + .expect("mutations"); + + if mutations.is_empty() { + return; + } + self.store.commit(txn_id).await; { - let (mut pending, mut committed): ( - FileWriteGuard, - FileWriteGuard, - ) = try_join!(self.pending.write(), self.committed.write()).expect("SyncChain blocks"); + let mut committed: FileWriteGuard = + self.committed.write().await.expect("SyncChain block"); - if let Some(mutations) = pending.mutations.remove(&txn_id) { - committed.mutations.insert(txn_id, mutations); - } + committed.mutations.insert(txn_id, mutations); } - try_join!(self.pending.sync(), self.committed.sync()).expect("sync SyncChain blocks"); + self.committed.sync().await.expect("sync SyncChain block") } } @@ -83,16 +89,14 @@ where self.subject.restore(*txn.id(), &backup).await?; - let (mut pending, mut committed) = try_join!(self.pending.write(), self.committed.write())?; + let mut committed = self.committed.write().await?; - *pending = ChainBlock::with_txn(null_hash().to_vec(), *txn.id()); *committed = ChainBlock::new(null_hash().to_vec()); Ok(()) } } -#[async_trait] impl ChainInstance for SyncChain where State: StateInstance, @@ -101,25 +105,17 @@ where Collection: TryCastFrom, Scalar: TryCastFrom, { - async fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { - let mut block: FileWriteGuard = self.pending.write().await?; - block.append_delete(txn_id, key); - Ok(()) + fn append_delete(&self, txn_id: TxnId, key: Value) -> TCResult<()> { + self.queue + .push(txn_id, MutationPending::Delete(key)) + .map_err(TCError::from) } - async fn append_put(&self, txn: &State::Txn, key: Value, value: State) -> TCResult<()> { - debug!("SyncChain::append_put {} <- {:?}", key, value); - + fn append_put(&self, txn: State::Txn, key: Value, value: State) -> TCResult<()> { + let txn_id = *txn.id(); let value = StoreEntry::try_from_state(value)?; - let value = self.store.save_state(txn, value).await?; - - debug!("locking pending transaction log block..."); - let mut block: FileWriteGuard = self.pending.write().await?; - - block.append_put(*txn.id(), key, value); - - debug!("locked pending transaction log block"); - Ok(()) + let mutation = MutationPending::Put(txn, key, value); + self.queue.push(txn_id, mutation).map_err(TCError::from) } fn subject(&self) -> &T { @@ -174,27 +170,12 @@ where async fn rollback(&self, txn_id: &TxnId) { debug!("SyncChain::rollback"); + self.queue.rollback(txn_id); self.subject.rollback(txn_id).await; - - { - let mut pending: FileWriteGuard = - self.pending.write().await.expect("pending"); - - pending.mutations.remove(txn_id); - } - - self.pending.sync().await.expect("sync pending block"); } async fn finalize(&self, txn_id: &TxnId) { - { - let mut pending: FileWriteGuard = - self.pending.write().await.expect("pending"); - - pending.mutations.remove(txn_id); - } - - self.pending.sync().await.expect("sync pending block"); + self.queue.finalize(*txn_id); self.subject.finalize(txn_id).await } } @@ -203,7 +184,11 @@ where impl fs::Persist for SyncChain where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, T: fs::Persist + Send + Sync, { type Txn = State::Txn; @@ -222,26 +207,26 @@ where let store = { let dir = dir.create_dir(STORE.to_string())?; + fs::Dir::load(txn_id, dir) .map_ok(super::data::Store::new) .await? }; + let queue = new_queue::(store.clone()); + + // TODO: is this necessary? let mut blocks_dir = dir .create_dir(BLOCKS.to_string()) .and_then(|dir| dir.try_write_owned())?; - let block = ChainBlock::with_txn(null_hash().to_vec(), txn_id); - let size_hint = block.get_size(); - let pending = blocks_dir.create_file(PENDING.to_string(), block, size_hint)?; - let block = ChainBlock::new(null_hash().to_vec()); let size_hint = block.get_size(); let committed = blocks_dir.create_file(COMMITTED.to_string(), block, size_hint)?; Ok(Self { subject, - pending, + queue, committed, store, }) @@ -265,18 +250,12 @@ where .await? }; + let queue = new_queue::(store.clone()); + let mut blocks_dir = dir .get_or_create_dir(BLOCKS.to_string()) .and_then(|dir| dir.try_write_owned())?; - let pending = if let Some(file) = blocks_dir.get_file(&*PENDING) { - file.clone() - } else { - let block = ChainBlock::with_txn(null_hash().to_vec(), txn_id); - let size_hint = block.get_size(); - blocks_dir.create_file(PENDING.to_string(), block, size_hint)? - }; - let committed = if let Some(file) = blocks_dir.get_file(&*COMMITTED) { file.clone() } else { @@ -287,7 +266,7 @@ where Ok(Self { subject, - pending, + queue, committed, store, }) @@ -333,7 +312,11 @@ where impl fs::CopyFrom for SyncChain where State: StateInstance, - State::FE: AsType + for<'a> fs::FileSave<'a>, + State::FE: DenseCacheFile + + AsType + + AsType + + AsType + + for<'a> fs::FileSave<'a>, T: fs::Persist + Route + fmt::Debug, { async fn copy_from( @@ -349,7 +332,7 @@ where impl de::FromStream for SyncChain where State: StateInstance, - State::FE: AsType, + State::FE: DenseCacheFile + AsType + AsType + AsType, T: FromStream, { type Context = State::Txn; @@ -376,6 +359,8 @@ where .await? }; + let queue = new_queue::(store.clone()); + let mut blocks_dir = { let file = { let mut cxt = cxt.write().await; @@ -394,15 +379,9 @@ where .create_file(COMMITTED.to_string(), block, size_hint) .map_err(de::Error::custom)?; - let block = ChainBlock::with_txn(null_hash.to_vec(), *txn.id()); - let size_hint = block.get_size(); - let pending = blocks_dir - .create_file(PENDING.to_string(), block, size_hint) - .map_err(de::Error::custom)?; - Ok(Self { subject, - pending, + queue, committed, store, }) diff --git a/host/collection/src/tensor/dense/access.rs b/host/collection/src/tensor/dense/access.rs index 11e879df..e1fb390a 100644 --- a/host/collection/src/tensor/dense/access.rs +++ b/host/collection/src/tensor/dense/access.rs @@ -4138,17 +4138,17 @@ where Buffer: de::FromStream, { pub fn commit(&self, txn_id: &TxnId) { - self.semaphore.finalize(txn_id, true) + self.semaphore.finalize(txn_id, false) + } + + pub fn rollback(&self, txn_id: &TxnId) { + self.semaphore.finalize(txn_id, false) } pub async fn finalize(&self, txn_id: &TxnId) { self.file.commit().await; self.semaphore.finalize(txn_id, true) } - - pub fn rollback(&self, txn_id: &TxnId) { - self.semaphore.finalize(txn_id, false) - } } impl TensorInstance for DenseVersion diff --git a/host/error/Cargo.toml b/host/error/Cargo.toml index b96789d7..06335c81 100644 --- a/host/error/Cargo.toml +++ b/host/error/Cargo.toml @@ -17,5 +17,5 @@ ha-ndarray = "0.3" log = { version = "0.4", features = ["release_max_level_info"] } pathlink = "0.2" serde = "1.0" -txn_lock = "0.8" -txfs = "0.2" +txfs = { path = "../../../txfs" } +txn_lock = { path = "../../../txn_lock" } diff --git a/host/error/src/lib.rs b/host/error/src/lib.rs index 69db2eae..0a47a85b 100644 --- a/host/error/src/lib.rs +++ b/host/error/src/lib.rs @@ -10,6 +10,7 @@ use destream::en; /// A result of type `T`, or a [`TCError`] pub type TCResult = Result; +#[derive(Clone)] struct ErrorData { message: String, stack: Vec, @@ -118,6 +119,7 @@ impl fmt::Display for ErrorKind { } /// A general error description. +#[derive(Clone)] pub struct TCError { kind: ErrorKind, data: ErrorData, diff --git a/host/fs/Cargo.toml b/host/fs/Cargo.toml index cc68cfd7..e0057bd3 100644 --- a/host/fs/Cargo.toml +++ b/host/fs/Cargo.toml @@ -16,6 +16,7 @@ async-hash = "0.5" async-trait = "0.1" bytes = "1.5" destream = "0.7" +ds-ext = "0.1" futures = "0.3" freqfs = { version = "0.9", features = ["all"] } get-size = "0.1" diff --git a/host/fs/src/txn/server.rs b/host/fs/src/txn/server.rs index 3880c6db..b4928d5a 100644 --- a/host/fs/src/txn/server.rs +++ b/host/fs/src/txn/server.rs @@ -1,10 +1,12 @@ //! A server to keep track of active transactions. use std::cmp::Ordering; -use std::collections::hash_map::{Entry, HashMap}; use std::convert::TryFrom; +use std::hash::{Hash, Hasher}; use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use ds_ext::OrdHashSet; use freqfs::DirLock; use futures::future::TryFutureExt; use log::*; @@ -18,12 +20,54 @@ use crate::block::CacheBlock; use super::request::*; use super::{Gateway, Txn, TxnId}; +// allow the end-user request to time out gracefully before garbage collection +const GRACE: Duration = Duration::from_secs(1); + +struct Active { + txn_id: TxnId, + expires: NetworkTime, +} + +impl Active { + fn new(txn_id: TxnId, expires: NetworkTime) -> Self { + Self { txn_id, expires } + } +} + +impl Eq for Active {} + +impl PartialEq for Active { + fn eq(&self, other: &Self) -> bool { + self.txn_id == other.txn_id + } +} + +impl Ord for Active { + fn cmp(&self, other: &Self) -> Ordering { + self.expires + .cmp(&other.expires) + .then(self.txn_id.cmp(&other.txn_id)) + } +} + +impl Hash for Active { + fn hash(&self, hasher: &mut H) { + self.txn_id.hash(hasher) + } +} + +impl PartialOrd for Active { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Server to keep track of the transactions currently active for this host. #[derive(Clone)] pub struct TxnServer { - active: Arc>>, + active: Arc>>, + tx: mpsc::UnboundedSender, workspace: DirLock, - tx: mpsc::UnboundedSender<(TxnId, NetworkTime)>, } impl TxnServer { @@ -32,7 +76,7 @@ impl TxnServer { let (tx, rx) = mpsc::unbounded_channel(); let server = Self { - active: Arc::new(RwLock::new(HashMap::new())), + active: Arc::new(RwLock::new(OrdHashSet::new())), workspace, tx, }; @@ -55,7 +99,7 @@ impl TxnServer { let request = Request::new(txn_id, token.0, token.1); self.tx - .send((txn_id, expires)) + .send(Active::new(txn_id, expires)) .map_err(|cause| internal!("transaction queue failure: {cause}"))?; Ok(Txn::new(self.workspace.clone(), gateway, request)) @@ -85,17 +129,19 @@ impl TxnServer { where G: Gateway, { + let now = SystemTime::from(now); + let expired = { let mut active = self.active.write().await; + let mut expired = Vec::with_capacity(active.len()); - let expired = active - .iter() - .filter_map(|(txn_id, expires)| if expires <= &now { Some(txn_id) } else { None }) - .copied() - .collect::>(); - - for txn_id in &expired { - active.remove(txn_id); + while let Some(txn) = active.first() { + if SystemTime::from(txn.expires) + GRACE < now { + let txn = active.pop_first().expect("expired txn id"); + expired.push(txn.txn_id); + } else { + break; + } } expired @@ -121,35 +167,14 @@ impl TxnServer { } } -fn spawn_receiver_thread(server: TxnServer, mut rx: mpsc::UnboundedReceiver<(TxnId, NetworkTime)>) { - let new_txn = |active: &mut HashMap, txn_id, expires| { - match active.entry(txn_id) { - Entry::Occupied(mut entry) => { - trace!("txn {} is already known", txn_id); - - match entry.get().cmp(&expires) { - Ordering::Greater | Ordering::Equal => { - // no-op - } - Ordering::Less => { - warn!("expiration time of txn {txn_id} extended"); - entry.insert(expires); - } - } - } - Entry::Vacant(entry) => { - entry.insert(expires); - } - } - }; - +fn spawn_receiver_thread(server: TxnServer, mut rx: mpsc::UnboundedReceiver) { tokio::spawn(async move { - while let Some((txn_id, expires)) = rx.recv().await { + while let Some(txn) = rx.recv().await { let mut active = server.active.write().await; - new_txn(&mut *active, txn_id, expires); + active.insert(txn); - while let Ok((txn_id, expires)) = rx.try_recv() { - new_txn(&mut *active, txn_id, expires); + while let Ok(txn) = rx.try_recv() { + active.insert(txn); } } }); diff --git a/host/generic/src/time.rs b/host/generic/src/time.rs index 32547dcb..7e8ad94e 100644 --- a/host/generic/src/time.rs +++ b/host/generic/src/time.rs @@ -8,7 +8,7 @@ use std::time::Duration; use tc_error::*; /// The current time of the TinyChain network, used to generate transaction IDs. -#[derive(Clone, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct NetworkTime { nanos: u64, } diff --git a/host/transact/Cargo.toml b/host/transact/Cargo.toml index 134f5a49..6d2d6bc4 100644 --- a/host/transact/Cargo.toml +++ b/host/transact/Cargo.toml @@ -28,5 +28,5 @@ tc-error = { path = "../error" } tc-value = { path = "../value" } tcgeneric = { path = "../generic" } tokio = { version = "1.35", features = ["sync"] } -txfs = { version = "0.2", features = ["logging"] } -txn_lock = { version = "0.8", features = ["logging"] } +txfs = { path = "../../../txfs", features = ["logging"] } +txn_lock = { path = "../../../txn_lock", features = ["logging"] } diff --git a/host/transact/src/lib.rs b/host/transact/src/lib.rs index b01b449a..04397f3a 100644 --- a/host/transact/src/lib.rs +++ b/host/transact/src/lib.rs @@ -20,6 +20,9 @@ mod id; pub mod public; pub mod lock { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + use super::TxnId; pub use txn_lock::semaphore::{PermitRead, PermitWrite}; @@ -31,19 +34,25 @@ pub mod lock { pub type TxnLock = txn_lock::scalar::TxnLock; /// A read guard on a committed transactional version - pub type TxnLockVersionGuard = txn_lock::scalar::TxnLockVersionGuard; + pub type TxnLockVersionGuard = Arc; + + /// A transactional message queue. + pub type TxnMessageQueue = txn_lock::queue::message::MessageQueue; /// A transactional read-write lock on a key-value map pub type TxnMapLock = txn_lock::map::TxnMapLock; /// A read guard on a committed transactional version of a set - pub type TxnMapLockVersionGuard = txn_lock::map::TxnMapLockVersionGuard; + pub type TxnMapLockVersionGuard = HashMap; /// A transactional read-write lock on a set of values pub type TxnSetLock = txn_lock::set::TxnSetLock; /// A read guard on a committed transactional version of a set - pub type TxnSetLockVersionGuard = txn_lock::set::TxnSetLockVersionGuard; + pub type TxnSetLockVersionGuard = HashSet; + + /// A transactional task queue. + pub type TxnTaskQueue = txn_lock::queue::task::TaskQueue; } /// Defines a method to compute the hash of this state as of a given [`TxnId`] From 56289be4670d9f3330603ca26a90a028b321a6ef Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Thu, 25 Jan 2024 07:38:55 +0530 Subject: [PATCH 07/10] upgrade to rjwt v0.5 (#278) * impl From for TCError * upgrade to rjwt 0.5 --- host/Cargo.toml | 2 +- host/error/Cargo.toml | 1 + host/error/src/lib.rs | 41 +++++++++++++---- host/fs/Cargo.toml | 2 +- host/fs/src/lib.rs | 2 +- host/fs/src/txn/hypothetical.rs | 2 +- host/fs/src/txn/mod.rs | 78 ++++++++++++--------------------- host/fs/src/txn/request.rs | 41 +++++------------ host/fs/src/txn/server.rs | 6 +-- host/src/cluster/mod.rs | 15 ++++--- host/src/gateway.rs | 25 +++++------ host/src/http/client.rs | 8 ++-- host/src/http/server.rs | 28 ++++++++++-- host/src/lib.rs | 38 ++++++++++++---- host/src/public/cluster/mod.rs | 15 +++++-- 15 files changed, 170 insertions(+), 134 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index add5ba4d..81a6b94f 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -36,7 +36,7 @@ hyper = { version = "0.14", features = ["full"] } log = { version = "0.4", features = ["release_max_level_info"] } num_cpus = "1.16" pin-project = "1.1" -rjwt = "0.4" +rjwt = { path = "../../rjwt/rust" } safecast = "0.2" serde = { version = "1.0", features = [] } serde_json = { version = "1.0" } diff --git a/host/error/Cargo.toml b/host/error/Cargo.toml index 06335c81..916a0418 100644 --- a/host/error/Cargo.toml +++ b/host/error/Cargo.toml @@ -16,6 +16,7 @@ destream = "0.7" ha-ndarray = "0.3" log = { version = "0.4", features = ["release_max_level_info"] } pathlink = "0.2" +rjwt = { path = "../../../rjwt/rust" } serde = "1.0" txfs = { path = "../../../txfs" } txn_lock = { path = "../../../txn_lock" } diff --git a/host/error/src/lib.rs b/host/error/src/lib.rs index 0a47a85b..93b0f28d 100644 --- a/host/error/src/lib.rs +++ b/host/error/src/lib.rs @@ -23,6 +23,7 @@ impl<'en> en::ToStream<'en> for ErrorData { } use en::EncodeMap; + let mut map = encoder.encode_map(Some(2))?; map.encode_entry("message", &self.message)?; map.encode_entry("stack", &self.stack)?; @@ -37,6 +38,7 @@ impl<'en> en::IntoStream<'en> for ErrorData { } use en::EncodeMap; + let mut map = encoder.encode_map(Some(2))?; map.encode_entry("message", self.message)?; map.encode_entry("stack", self.stack)?; @@ -168,6 +170,11 @@ impl TCError { } } + /// Error to indicate a malformed or nonsensical request + pub fn bad_request(info: I) -> Self { + Self::new(ErrorKind::BadGateway, info) + } + /// Error to convey an upstream problem pub fn bad_gateway(locator: I) -> Self { Self::new(ErrorKind::BadGateway, locator) @@ -197,15 +204,19 @@ impl TCError { Self::new(ErrorKind::NotFound, locator) } + /// Error to indicate that the end-user is not authorized to perform the requested action + pub fn unauthorized(info: I) -> Self { + Self::new(ErrorKind::Unauthorized, info) + } + + /// Error to indicate an unexpected input value or type pub fn unexpected(value: V, expected: &str) -> Self { - Self::new( - ErrorKind::BadRequest, - format!("invalid value {value:?}: expected {expected}"), - ) + Self::bad_request(format!("invalid value {value:?}: expected {expected}")) } + /// Error to indicate that the requested action cannot be performed due to technical limitations pub fn unsupported(info: I) -> Self { - Self::new(ErrorKind::BadRequest, info.to_string()) + Self::bad_request(info) } /// The [`ErrorKind`] of this error @@ -214,7 +225,7 @@ impl TCError { } /// The error message of this error - pub fn message(&'_ self) -> &'_ str { + pub fn message(&self) -> &str { &self.data.message } @@ -229,7 +240,7 @@ impl std::error::Error for TCError {} impl From for TCError { fn from(err: pathlink::ParseError) -> Self { - Self::new(ErrorKind::BadRequest, err) + Self::bad_request(err) } } @@ -239,6 +250,18 @@ impl From for TCError { } } +impl From for TCError { + fn from(err: rjwt::Error) -> Self { + match err.into_inner() { + (rjwt::ErrorKind::Auth | rjwt::ErrorKind::Time, msg) => Self::unauthorized(msg), + (rjwt::ErrorKind::Base64 | rjwt::ErrorKind::Format | rjwt::ErrorKind::Json, msg) => { + Self::bad_request(msg) + } + (rjwt::ErrorKind::Fetch, msg) => Self::bad_gateway(msg), + } + } +} + impl From for TCError { fn from(err: txn_lock::Error) -> Self { Self::new(ErrorKind::Conflict, err) @@ -331,7 +354,7 @@ macro_rules! bad_gateway { #[macro_export] macro_rules! bad_request { ($($t:tt)*) => {{ - $crate::TCError::new($crate::ErrorKind::BadRequest, format!($($t)*)) + $crate::TCError::bad_request(format!($($t)*)) }} } @@ -379,7 +402,7 @@ macro_rules! internal { #[macro_export] macro_rules! unauthorized { ($($t:tt)*) => {{ - $crate::TCError::new($crate::ErrorKind::Unauthorized, format!($($t)*)) + $crate::TCError::unauthorized(format!($($t)*)) }} } diff --git a/host/fs/Cargo.toml b/host/fs/Cargo.toml index e0057bd3..86bbf2e1 100644 --- a/host/fs/Cargo.toml +++ b/host/fs/Cargo.toml @@ -23,7 +23,7 @@ get-size = "0.1" get-size-derive = "0.1" log = { version = "0.4", features = ["release_max_level_info"] } num_cpus = "1.16" -rjwt = "0.4" +rjwt = { path = "../../../rjwt/rust" } safecast = "0.2" tbon = "0.5" tc-chain = { path = "../chain" } diff --git a/host/fs/src/lib.rs b/host/fs/src/lib.rs index 6714b886..6d05966e 100644 --- a/host/fs/src/lib.rs +++ b/host/fs/src/lib.rs @@ -1,5 +1,5 @@ pub use block::CacheBlock; -pub use txn::{Actor, Claims, Gateway, Resolver, Token, Txn, TxnServer}; +pub use txn::{Actor, Gateway, Resolver, SignedToken, Token, Txn, TxnServer}; pub use txn::hypothetical; diff --git a/host/fs/src/txn/hypothetical.rs b/host/fs/src/txn/hypothetical.rs index be249495..7e0697e7 100644 --- a/host/fs/src/txn/hypothetical.rs +++ b/host/fs/src/txn/hypothetical.rs @@ -59,7 +59,7 @@ where } pub async fn execute(&self, txn: &Txn, data: State) -> TCResult { - let txn = txn.clone().claim(&self.actor, PATH.into()).await?; + let txn = txn.clone().claim(&self.actor, PATH.into())?; let context = Map::::default(); let result = if Vec::<(Id, State)>::can_cast_from(&data) { diff --git a/host/fs/src/txn/mod.rs b/host/fs/src/txn/mod.rs index 37009700..309048f3 100644 --- a/host/fs/src/txn/mod.rs +++ b/host/fs/src/txn/mod.rs @@ -1,15 +1,14 @@ //! The transaction context [`Txn`]. use std::hash::{Hash, Hasher}; -use std::iter::FromIterator; use std::ops::Deref; use std::pin::Pin; use std::sync::Arc; use async_trait::async_trait; use freqfs::DirLock; -use futures::future::{Future, TryFutureExt}; -use log::debug; +use futures::Future; +use log::{debug, trace}; use safecast::CastInto; use tc_error::*; @@ -17,7 +16,7 @@ use tc_transact::public::StateInstance; use tc_transact::Transaction; use tc_value::uuid::Uuid; use tc_value::{Host, Link, ToUrl, Value}; -use tcgeneric::{Id, PathSegment, TCBoxFuture, TCBoxTryFuture, TCPathBuf, ThreadSafe, Tuple}; +use tcgeneric::{Id, PathSegment, TCBoxFuture, TCBoxTryFuture, TCPathBuf, ThreadSafe}; use crate::block::CacheBlock; @@ -169,40 +168,31 @@ where } /// Return a new `Txn` which grants the given [`Scope`]s to the given [`Actor`]. - pub async fn grant( + pub fn grant( &self, actor: &Actor, cluster_path: TCPathBuf, scopes: Vec, ) -> TCResult { - let token = self.request.token().to_string(); - let txn_id = self.request.txn_id(); - - use rjwt::Resolve; - let host = self.gateway.link(cluster_path); - let resolver = Resolver::new(&*self.gateway, &host, self.request.txn_id()); + trace!("grant {scopes:?} for {cluster_path} to {}", actor.id()); - debug!( - "granting scopes {} to {}", - Tuple::::from_iter(scopes.clone()), - host - ); + let host_id = self.gateway.link(cluster_path); + let txn_id = self.request.txn_id(); + let now = txn_id.time().into(); + let token = actor.consume_and_sign(self.request.token().clone(), host_id, scopes, now)?; - let (token, claims) = resolver - .consume_and_sign(actor, scopes, token, txn_id.time().into()) - .map_err(|cause| unauthorized!("signature error").consume(cause)) - .await?; + trace!("granted scopes to {}", actor.id()); Ok(Self { gateway: self.gateway.clone(), - request: Arc::new(Request::new(*txn_id, token, claims)), + request: Arc::new(Request::new(*txn_id, token)), scope: self.scope.clone(), dir: self.dir.clone(), }) } /// Claim ownership of this transaction. - pub async fn claim(self, actor: &Actor, cluster_path: TCPathBuf) -> TCResult { + pub fn claim(self, actor: &Actor, cluster_path: TCPathBuf) -> TCResult { debug!( "{} claims ownership of transaction {}", cluster_path, @@ -215,7 +205,6 @@ where if self.owner().is_none() { self.grant(actor, cluster_path, vec![self.scope.clone()]) - .await } else { Err(forbidden!( "tried to claim owned transaction {}", @@ -242,18 +231,14 @@ where /// Return the owner of this transaction, if there is one. pub fn owner(&self) -> Option<&Link> { - self.request - .scopes() - .iter() - .filter(|(_, actor_id, _)| *actor_id == &Value::None) - .filter_map(|(host, _actor_id, scopes)| { - if scopes.contains(&self.scope) { - Some(host) - } else { - None - } - }) - .fold(None, |_, host| Some(host)) + let (host, _, _) = self + .request + .token() + .first_claim(|(_host, actor_id, scopes)| { + actor_id == &Value::None && scopes.contains(&self.scope) + })?; + + Some(host) } /// Check if this transaction has a leader for the given cluster. @@ -273,7 +258,7 @@ where } /// Claim leadership of this transaction for the given cluster. - pub async fn lead(self, actor: &Actor, cluster_path: TCPathBuf) -> TCResult { + pub fn lead(self, actor: &Actor, cluster_path: TCPathBuf) -> TCResult { debug!( "{} claim leadership of transaction {}", cluster_path, @@ -293,24 +278,19 @@ where )) } else { let scopes = vec![self.scope.clone()]; - self.grant(actor, cluster_path, scopes).await + self.grant(actor, cluster_path, scopes) } } /// Return the leader of this transaction for the given cluster, if there is one. pub fn leader(&self, cluster_path: &[PathSegment]) -> Option<&Link> { - self.request - .scopes() - .iter() - .filter(|(_, actor_id, _)| *actor_id == &Value::None) - .filter_map(|(host, _actor_id, scopes)| { - if scopes.contains(&self.scope) && cluster_path.starts_with(host.path()) { - Some(host) - } else { - None - } - }) - .next() + let (host, _, _) = self.request.token().last_claim(|(host, actor, scopes)| { + actor == &Value::None + && scopes.contains(&self.scope) + && cluster_path.starts_with(host.path()) + })?; + + Some(host) } } diff --git a/host/fs/src/txn/request.rs b/host/fs/src/txn/request.rs index ca7e4bec..c1f2b76e 100644 --- a/host/fs/src/txn/request.rs +++ b/host/fs/src/txn/request.rs @@ -20,8 +20,8 @@ use super::{Gateway, Txn}; /// The type of [`rjwt::Actor`] used to sign auth tokens pub type Actor = rjwt::Actor; -/// The type of [`rjwt::Claims`] communicated in auth tokens -pub type Claims = rjwt::Claims>; +/// The type of [`rjwt::SignedToken`] used for authentication & authorization +pub type SignedToken = rjwt::SignedToken>; /// The type of scope communicated by [`Claims`] pub type Scope = TCPathBuf; @@ -31,36 +31,26 @@ pub type Token = rjwt::Token>; /// A `Txn`'s authorization. pub struct Request { - token: String, - claims: Claims, + token: SignedToken, txn_id: TxnId, } impl Request { /// Construct a new `Request`. - pub fn new(txn_id: TxnId, token: String, claims: Claims) -> Self { - Self { - token, - claims, - txn_id, - } + pub fn new(txn_id: TxnId, token: SignedToken) -> Self { + Self { token, txn_id } } pub fn expires(&self) -> TCResult { - let expires = self.claims.expires(); + let expires = self.token.expires(); expires .try_into() .map_err(|cause| bad_request!("invalid token expiration time").consume(cause)) } - /// Return this request's authorizations. - pub fn scopes(&self) -> &Claims { - &self.claims - } - /// Return this request's JSON web token (cf. the [`rjwt`] crate) - pub fn token(&self) -> &str { + pub fn token(&self) -> &SignedToken { &self.token } @@ -73,18 +63,13 @@ impl Request { /// Struct responsible for resolving JWT auth identities (cf. the [`rjwt`] crate). pub struct Resolver<'a, State> { gateway: &'a dyn Gateway, - host: &'a Link, txn_id: &'a TxnId, } impl<'a, State> Resolver<'a, State> { /// Construct a new `Resolver`. - pub fn new(gateway: &'a dyn Gateway, host: &'a Link, txn_id: &'a TxnId) -> Self { - Self { - gateway, - host, - txn_id, - } + pub fn new(gateway: &'a dyn Gateway, txn_id: &'a TxnId) -> Self { + Self { gateway, txn_id } } } @@ -93,14 +78,10 @@ impl<'a, State> rjwt::Resolve for Resolver<'a, State> where State: StateInstance>, { - type Host = Link; + type HostId = Link; type ActorId = Value; type Claims = Vec; - fn host(&self) -> Link { - self.host.clone() - } - async fn resolve(&self, host: &Link, actor_id: &Value) -> Result { let public_key: Bytes = self .gateway @@ -110,7 +91,7 @@ where value.try_cast_into(|v| bad_request!("invalid public key: {v:?}")) }) }) - .map_err(|e| rjwt::Error::new(rjwt::ErrorKind::Fetch, e)) + .map_err(|e| rjwt::Error::fetch(e)) .await?; Actor::with_public_key(actor_id.clone(), &public_key) diff --git a/host/fs/src/txn/server.rs b/host/fs/src/txn/server.rs index b4928d5a..26623bd9 100644 --- a/host/fs/src/txn/server.rs +++ b/host/fs/src/txn/server.rs @@ -91,12 +91,12 @@ impl TxnServer { &self, gateway: Arc>, txn_id: TxnId, - token: (String, Claims), + token: SignedToken, ) -> TCResult> { debug!("TxnServer::new_txn"); - let expires = NetworkTime::try_from(token.1.expires())?; - let request = Request::new(txn_id, token.0, token.1); + let expires = NetworkTime::try_from(token.expires())?; + let request = Request::new(txn_id, token); self.tx .send(Active::new(txn_id, expires)) diff --git a/host/src/cluster/mod.rs b/host/src/cluster/mod.rs index 21973877..6bb3de1e 100644 --- a/host/src/cluster/mod.rs +++ b/host/src/cluster/mod.rs @@ -171,14 +171,13 @@ impl Cluster { pub async fn claim(&self, txn: &Txn) -> TCResult { debug_assert!(!txn.has_owner(), "tried to claim an owned transaction"); + trace!("adding {} to the list of owned transactions...", txn.id()); let mut led = self.led.write().await; - let txn = txn - .clone() - .claim(&self.actor, self.schema.path.clone()) - .await?; + let txn = txn.clone().claim(&self.actor, self.schema.path.clone())?; led.insert(*txn.id(), leader::Leader::new()); + trace!("added {} to the list of owned transactions...", txn.id()); Ok(txn) } @@ -189,7 +188,7 @@ impl Cluster { Ok(txn) } else { let mut led = self.led.write().await; - let txn = txn.lead(&self.actor, self.schema.path.clone()).await?; + let txn = txn.lead(&self.actor, self.schema.path.clone())?; led.insert(*txn.id(), leader::Leader::new()); Ok(txn) } @@ -225,7 +224,7 @@ where if txn.has_leader(self.path()) { self.distribute_commit(&txn).await } else { - let txn = txn.lead(&self.actor, self.schema.path.clone()).await?; + let txn = txn.lead(&self.actor, self.schema.path.clone())?; self.distribute_commit(&txn).await } } @@ -313,6 +312,8 @@ where pub async fn distribute_commit(&self, txn: &Txn) -> TCResult<()> { debug!("{:?} will distribute commit {}", self, txn.id()); + assert!(txn.is_leader(self.path())); + #[cfg(debug_assertions)] let results = self.distribute_commit_debug(txn).await?; @@ -445,7 +446,7 @@ where if txn.has_leader(self.path()) { self.add_replica(&txn, self.schema.host.clone()).await } else { - let txn = txn.lead(&self.actor, self.schema.path.clone()).await?; + let txn = txn.lead(&self.actor, self.schema.path.clone())?; self.add_replica(&txn, self.schema.host.clone()).await } } diff --git a/host/src/gateway.rs b/host/src/gateway.rs index 88bf0dbc..9720700d 100644 --- a/host/src/gateway.rs +++ b/host/src/gateway.rs @@ -11,7 +11,7 @@ use log::debug; use tokio::time::{Duration, MissedTickBehavior}; use tc_error::*; -use tc_fs::{Actor, Claims, Gateway as GatewayInstance, Resolver, Token, TxnServer}; +use tc_fs::{Actor, Gateway as GatewayInstance, SignedToken, Token, TxnServer}; use tc_state::State; use tc_transact::TxnId; use tc_value::{Host, Link, Protocol, ToUrl, Value}; @@ -115,7 +115,7 @@ impl Gateway { } /// Return a new, signed auth token with no claims. - pub fn new_token(&self, txn_id: &TxnId) -> TCResult<(String, Claims)> { + pub fn new_token(&self, txn_id: &TxnId) -> TCResult { let token = Token::new( self.inner.host.clone().into(), txn_id.time().into(), @@ -127,24 +127,21 @@ impl Gateway { let signed = self .inner .actor - .sign_token(&token) + .sign_token(token) .map_err(|cause| internal!("signing error").consume(cause))?; - let claims = token.claims(); - - Ok((signed, claims)) + Ok(signed) } /// Authorize a transaction to execute on this host. - pub async fn new_txn(self, txn_id: TxnId, token: Option) -> TCResult { + pub fn new_txn(self, txn_id: TxnId, token: Option) -> TCResult { let token = if let Some(token) = token { - use rjwt::Resolve; - - // TODO: don't require resolving an actor or validating the existing token in order to create a new token (just validate on receipt) - Resolver::new(&self, &self.host().clone().into(), &txn_id) - .consume_and_sign(&self.inner.actor, vec![], token, txn_id.time().into()) - .map_err(|cause| unauthorized!("credential error").consume(cause)) - .await? + self.inner.actor.consume_and_sign( + token, + self.host().clone().into(), + vec![], + txn_id.time().into(), + )? } else { self.new_token(&txn_id)? }; diff --git a/host/src/http/client.rs b/host/src/http/client.rs index 534b3a1a..096cf633 100644 --- a/host/src/http/client.rs +++ b/host/src/http/client.rs @@ -77,7 +77,7 @@ impl crate::gateway::Client for Client { } let uri = build_url(&link, txn.id(), &key)?; - let req = req_builder("GET", uri, Some(txn.request().token())); + let req = req_builder("GET", uri, Some(txn.request().token().jwt())); let txn = txn.subcontext_unique(); let response = self @@ -107,7 +107,7 @@ impl crate::gateway::Client for Client { } let uri = build_url(&link, txn.id(), &key)?; - let req = req_builder("PUT", uri, Some(txn.request().token())) + let req = req_builder("PUT", uri, Some(txn.request().token().jwt())) .header(hyper::header::CONTENT_TYPE, Encoding::Tbon.as_str()); let txn = txn.subcontext_unique(); @@ -141,7 +141,7 @@ impl crate::gateway::Client for Client { } let uri = build_url(&link, txn.id(), &Value::default())?; - let req = req_builder("POST", uri, Some(txn.request().token())) + let req = req_builder("POST", uri, Some(txn.request().token().jwt())) .header(hyper::header::CONTENT_TYPE, Encoding::Tbon.as_str()); let txn = txn.subcontext_unique(); @@ -184,7 +184,7 @@ impl crate::gateway::Client for Client { } let uri = build_url(&link, txn.id(), &key)?; - let req = req_builder("DELETE", uri, Some(txn.request().token())); + let req = req_builder("DELETE", uri, Some(txn.request().token().jwt())); let response = self .client diff --git a/host/src/http/server.rs b/host/src/http/server.rs index 52b6931c..761e5120 100644 --- a/host/src/http/server.rs +++ b/host/src/http/server.rs @@ -8,10 +8,11 @@ use futures::future::{self, TryFutureExt}; use futures::stream::{self, Stream, StreamExt, TryStreamExt}; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Response}; +use log::trace; use serde::de::DeserializeOwned; use tc_error::*; -use tc_fs::Gateway as GatewayInstance; +use tc_fs::{Gateway as GatewayInstance, Resolver}; use tc_state::State; use tc_transact::{IntoView, Transaction, TxnId}; use tcgeneric::{NetworkTime, TCPathBuf}; @@ -56,6 +57,11 @@ impl HTTPServer { Err(cause) => return Ok(transform_error(cause, Encoding::default())), }; + trace!( + "headers check out, routing request to {}...", + request.uri().path() + ); + let state = match self.route(request_encoding, &txn, params, request).await { Ok(state) => state, Err(cause) => return Ok(transform_error(cause, accept_encoding)), @@ -109,6 +115,8 @@ impl HTTPServer { &self, http_request: &hyper::Request, ) -> TCResult<(GetParams, Txn, Encoding, Encoding)> { + trace!("reading headers"); + let content_type = if let Some(header) = http_request.headers().get(hyper::header::CONTENT_TYPE) { header @@ -149,13 +157,27 @@ impl HTTPServer { None }; + let now = NetworkTime::now(); let txn_id = if let Some(txn_id) = params.remove("txn_id") { txn_id.parse()? } else { - TxnId::new(NetworkTime::now()) + TxnId::new(now) + }; + + let token = if let Some(token) = token { + let resolver = Resolver::new(&self.gateway, &txn_id); + let token = rjwt::Resolve::verify(&resolver, token, now.into()).await?; + Some(token) + } else { + None }; - let txn = self.gateway.clone().new_txn(txn_id, token).await?; + trace!("token is {token:?}, creating new txn..."); + + let txn = self.gateway.clone().new_txn(txn_id, token)?; + + trace!("transaction {txn_id} created"); + Ok((params, txn, accept_encoding, content_type)) } diff --git a/host/src/lib.rs b/host/src/lib.rs index cf33bac4..17d68458 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -128,6 +128,8 @@ impl Builder { use tc_transact::fs::Persist; use tc_transact::Transaction; + log::debug!("load or create cluster..."); + let txn_id = *txn.id(); let host = self.gateway.as_ref().expect("gateway config").host(); @@ -166,17 +168,26 @@ impl Builder { let txn_id = tc_transact::TxnId::new(gateway::Gateway::time()); let token = gateway.new_token(&txn_id).expect("token"); + log::debug!("loading userspace..."); + let txn = txn_server .new_txn(Arc::new(gateway), txn_id, token) .expect("transaction"); + log::trace!("new txn at {txn_id}"); + // no need to claim ownership of this txn since there's no way to make outbound requests // because they would be impossible to authorize since userspace is not yet loaded // i.e. there is no way for other hosts to check any of these Clusters' public keys let class: kernel::Class = self.load_or_create(&txn, kernel::CLASS).await; + log::trace!("loaded class"); + let library: kernel::Library = self.load_or_create(&txn, kernel::LIB).await; + log::trace!("loaded library"); + let service: kernel::Service = self.load_or_create(&txn, kernel::SERVICE).await; + log::trace!("loaded service"); futures::try_join!( class.recover(&txn), @@ -195,6 +206,8 @@ impl Builder { } async fn bootstrap(self) -> (gateway::Gateway, UserSpace) { + log::debug!("running bootstrap..."); + let gateway_config = self.gateway.clone().expect("gateway config"); let kernel = kernel::Kernel::bootstrap(); @@ -212,25 +225,34 @@ impl Builder { } async fn replicate(gateway: gateway::Gateway, userspace: UserSpace) -> TCResult<()> { - let txn = gateway - .new_txn(tc_transact::TxnId::new(gateway::Gateway::time()), None) - .await?; - - async fn replicate_cluster(txn: &txn::Txn, cluster: cluster::Cluster) -> TCResult<()> + async fn replicate_cluster( + gateway: gateway::Gateway, + cluster: cluster::Cluster, + ) -> TCResult<()> where T: cluster::Replica + tc_transact::Transact + Send + Sync, { + log::trace!("replicate cluster {}...", cluster.link().path()); + + let txn_id = tc_transact::TxnId::new(gateway::Gateway::time()); + + let txn = gateway.new_txn(txn_id, None)?; + + log::trace!("{:?} replication will use txn {}", cluster, txn_id); + let txn = cluster.claim(&txn).await?; + log::trace!("{:?} will add replica at {}...", cluster, txn.host()); + cluster.add_replica(&txn, txn.host().clone()).await?; cluster.distribute_commit(&txn).await } futures::try_join!( - replicate_cluster(&txn, userspace.0), - replicate_cluster(&txn, userspace.1), - replicate_cluster(&txn, userspace.2), + replicate_cluster(gateway.clone(), userspace.0), + replicate_cluster(gateway.clone(), userspace.1), + replicate_cluster(gateway.clone(), userspace.2), )?; Ok(()) diff --git a/host/src/public/cluster/mod.rs b/host/src/public/cluster/mod.rs index b75d04db..2f2107ef 100644 --- a/host/src/public/cluster/mod.rs +++ b/host/src/public/cluster/mod.rs @@ -130,9 +130,18 @@ where )); } } else { + #[cfg(debug_assertions)] + panic!( + "commit message for an ownerless transaction {} (token is {:?})", + txn.id(), + txn.request().token(), + ); + + #[cfg(not(debug_assertions))] return Err(bad_request!( - "commit message for an ownerless transaction {}", + "commit message for an ownerless transaction {} (token is {:?})", txn.id(), + txn.request().token(), )); } @@ -345,8 +354,8 @@ fn authorize_install(txn: &Txn, parent: &Link, entry_path: &TCPathBuf) -> TCResu let authorized = txn .request() - .scopes() - .get(&parent, &TCPathBuf::default().into()) + .token() + .get_claim(&parent, &TCPathBuf::default().into()) .ok_or_else(|| unauthorized!("install request for {}", parent))?; if authorized.iter().any(|scope| entry_path.starts_with(scope)) { From 49f83f7958ce43fb00e3da9e91c162b0d4f5fd48 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Thu, 25 Jan 2024 12:48:23 +0530 Subject: [PATCH 08/10] move the Python & node.js clients into a single directory and separate the tinychain_async from the tinychain Python package (#279) * move python and js client libraries into a single top-level directory * move tinychain.host.asynchronous into a new tinychain_async package --- CONTRIBUTING.md | 13 +- {node-client => client/node}/.eslintrc.cjs | 0 {node-client => client/node}/.gitignore | 0 {node-client => client/node}/.nvmrc | 0 {node-client => client/node}/.prettierrc | 0 {node-client => client/node}/README.md | 0 {node-client => client/node}/babel.config.cjs | 0 {node-client => client/node}/jest.config.mjs | 0 {node-client => client/node}/package.json | 0 {node-client => client/node}/test/uri.spec.js | 0 .../node}/tinychain/index.js | 0 {node-client => client/node}/tinychain/uri.js | 0 client/{ => py}/.gitignore | 0 client/{ => py}/README.md | 0 client/{ => py}/docs/Makefile | 0 client/{ => py}/docs/btree.rst | 0 client/{ => py}/docs/chain.rst | 0 client/{ => py}/docs/conf.py | 0 client/{ => py}/docs/generic.rst | 0 client/py/docs/host.rst | 6 + client/{ => py}/docs/index.rst | 1 + client/{ => py}/docs/interface.rst | 0 client/{ => py}/docs/make.bat | 0 client/{ => py}/docs/math.interface.rst | 0 client/{ => py}/docs/math.linalg.rst | 0 client/{ => py}/docs/math.operator.rst | 0 client/{ => py}/docs/ml.nn.rst | 0 client/{ => py}/docs/ml.optimizer.rst | 0 client/{ => py}/docs/ml.service.rst | 0 client/py/docs/requirements.txt | 35 +++++ client/{ => py}/docs/scalar.number.rst | 0 client/{ => py}/docs/scalar.ref.rst | 0 client/{ => py}/docs/scalar.value.rst | 0 client/{ => py}/docs/service.rst | 0 client/{ => py}/docs/state.rst | 0 client/{ => py}/docs/table.rst | 0 client/{ => py}/docs/tensor.rst | 0 client/{ => py}/docs/uri.rst | 0 client/{ => py}/pyproject.toml | 0 client/{ => py}/setup.cfg | 1 - client/{ => py}/tinychain/__init__.py | 1 - client/{ => py}/tinychain/base.py | 0 client/{ => py}/tinychain/chain.py | 0 .../{ => py}/tinychain/collection/__init__.py | 0 client/{ => py}/tinychain/collection/base.py | 0 client/{ => py}/tinychain/collection/btree.py | 0 client/{ => py}/tinychain/collection/table.py | 0 .../tinychain/collection/tensor/__init__.py | 0 .../tinychain/collection/tensor/base.py | 0 .../tinychain/collection/tensor/einsum.py | 0 .../tinychain/collection/tensor/functions.py | 0 .../tinychain/collection/tensor/operator.py | 0 client/{ => py}/tinychain/context.py | 0 client/{ => py}/tinychain/decorators.py | 0 client/{ => py}/tinychain/error.py | 0 client/{ => py}/tinychain/generic.py | 0 client/{ => py}/tinychain/graph/__init__.py | 0 client/{ => py}/tinychain/graph/edge.py | 0 client/{ => py}/tinychain/graph/service.py | 0 .../host/sync.py => py/tinychain/host.py} | 12 +- client/{ => py}/tinychain/interface.py | 0 client/{ => py}/tinychain/json.py | 0 client/{ => py}/tinychain/math/__init__.py | 0 client/{ => py}/tinychain/math/base.py | 0 client/{ => py}/tinychain/math/constants.py | 0 client/{ => py}/tinychain/math/equation.py | 0 client/{ => py}/tinychain/math/interface.py | 0 client/{ => py}/tinychain/math/linalg.py | 0 client/{ => py}/tinychain/math/operator.py | 0 client/{ => py}/tinychain/ml/__init__.py | 0 client/{ => py}/tinychain/ml/activation.py | 0 client/{ => py}/tinychain/ml/constants.py | 0 client/{ => py}/tinychain/ml/interface.py | 0 client/{ => py}/tinychain/ml/nn.py | 0 client/{ => py}/tinychain/ml/optimizer.py | 0 client/{ => py}/tinychain/ml/service.py | 0 client/{ => py}/tinychain/ml/variable.py | 0 client/{ => py}/tinychain/reflect/__init__.py | 0 .../{ => py}/tinychain/reflect/functions.py | 0 client/{ => py}/tinychain/reflect/meta.py | 0 client/{ => py}/tinychain/reflect/method.py | 0 client/{ => py}/tinychain/reflect/op.py | 0 client/{ => py}/tinychain/reflect/stub.py | 0 client/{ => py}/tinychain/scalar/__init__.py | 0 client/{ => py}/tinychain/scalar/base.py | 0 client/{ => py}/tinychain/scalar/bound.py | 0 client/{ => py}/tinychain/scalar/number.py | 0 client/{ => py}/tinychain/scalar/op.py | 0 .../{ => py}/tinychain/scalar/ref/__init__.py | 0 client/{ => py}/tinychain/scalar/ref/base.py | 0 .../tinychain/scalar/ref/functions.py | 0 client/{ => py}/tinychain/scalar/value.py | 0 client/{ => py}/tinychain/service.py | 0 client/{ => py}/tinychain/shape.py | 0 client/{ => py}/tinychain/state.py | 0 client/{ => py}/tinychain/uri.py | 0 client/py_async/.gitignore | 133 ++++++++++++++++++ client/py_async/README.md | 1 + client/py_async/docs/Makefile | 20 +++ client/py_async/docs/conf.py | 62 ++++++++ client/py_async/docs/host.rst | 6 + client/py_async/docs/index.rst | 10 ++ client/py_async/docs/make.bat | 35 +++++ client/{ => py_async}/docs/requirements.txt | 0 client/py_async/pyproject.toml | 3 + client/py_async/setup.cfg | 34 +++++ client/py_async/tinychain_async/__init__.py | 1 + .../tinychain_async/host.py} | 6 +- client/tinychain/host/__init__.py | 1 - tests/tctest/benchmark.py | 13 +- tests/tctest/process.py | 4 +- 111 files changed, 367 insertions(+), 31 deletions(-) rename {node-client => client/node}/.eslintrc.cjs (100%) rename {node-client => client/node}/.gitignore (100%) rename {node-client => client/node}/.nvmrc (100%) rename {node-client => client/node}/.prettierrc (100%) rename {node-client => client/node}/README.md (100%) rename {node-client => client/node}/babel.config.cjs (100%) rename {node-client => client/node}/jest.config.mjs (100%) rename {node-client => client/node}/package.json (100%) rename {node-client => client/node}/test/uri.spec.js (100%) rename {node-client => client/node}/tinychain/index.js (100%) rename {node-client => client/node}/tinychain/uri.js (100%) rename client/{ => py}/.gitignore (100%) rename client/{ => py}/README.md (100%) rename client/{ => py}/docs/Makefile (100%) rename client/{ => py}/docs/btree.rst (100%) rename client/{ => py}/docs/chain.rst (100%) rename client/{ => py}/docs/conf.py (100%) rename client/{ => py}/docs/generic.rst (100%) create mode 100644 client/py/docs/host.rst rename client/{ => py}/docs/index.rst (97%) rename client/{ => py}/docs/interface.rst (100%) rename client/{ => py}/docs/make.bat (100%) rename client/{ => py}/docs/math.interface.rst (100%) rename client/{ => py}/docs/math.linalg.rst (100%) rename client/{ => py}/docs/math.operator.rst (100%) rename client/{ => py}/docs/ml.nn.rst (100%) rename client/{ => py}/docs/ml.optimizer.rst (100%) rename client/{ => py}/docs/ml.service.rst (100%) create mode 100644 client/py/docs/requirements.txt rename client/{ => py}/docs/scalar.number.rst (100%) rename client/{ => py}/docs/scalar.ref.rst (100%) rename client/{ => py}/docs/scalar.value.rst (100%) rename client/{ => py}/docs/service.rst (100%) rename client/{ => py}/docs/state.rst (100%) rename client/{ => py}/docs/table.rst (100%) rename client/{ => py}/docs/tensor.rst (100%) rename client/{ => py}/docs/uri.rst (100%) rename client/{ => py}/pyproject.toml (100%) rename client/{ => py}/setup.cfg (98%) rename client/{ => py}/tinychain/__init__.py (97%) rename client/{ => py}/tinychain/base.py (100%) rename client/{ => py}/tinychain/chain.py (100%) rename client/{ => py}/tinychain/collection/__init__.py (100%) rename client/{ => py}/tinychain/collection/base.py (100%) rename client/{ => py}/tinychain/collection/btree.py (100%) rename client/{ => py}/tinychain/collection/table.py (100%) rename client/{ => py}/tinychain/collection/tensor/__init__.py (100%) rename client/{ => py}/tinychain/collection/tensor/base.py (100%) rename client/{ => py}/tinychain/collection/tensor/einsum.py (100%) rename client/{ => py}/tinychain/collection/tensor/functions.py (100%) rename client/{ => py}/tinychain/collection/tensor/operator.py (100%) rename client/{ => py}/tinychain/context.py (100%) rename client/{ => py}/tinychain/decorators.py (100%) rename client/{ => py}/tinychain/error.py (100%) rename client/{ => py}/tinychain/generic.py (100%) rename client/{ => py}/tinychain/graph/__init__.py (100%) rename client/{ => py}/tinychain/graph/edge.py (100%) rename client/{ => py}/tinychain/graph/service.py (100%) rename client/{tinychain/host/sync.py => py/tinychain/host.py} (98%) rename client/{ => py}/tinychain/interface.py (100%) rename client/{ => py}/tinychain/json.py (100%) rename client/{ => py}/tinychain/math/__init__.py (100%) rename client/{ => py}/tinychain/math/base.py (100%) rename client/{ => py}/tinychain/math/constants.py (100%) rename client/{ => py}/tinychain/math/equation.py (100%) rename client/{ => py}/tinychain/math/interface.py (100%) rename client/{ => py}/tinychain/math/linalg.py (100%) rename client/{ => py}/tinychain/math/operator.py (100%) rename client/{ => py}/tinychain/ml/__init__.py (100%) rename client/{ => py}/tinychain/ml/activation.py (100%) rename client/{ => py}/tinychain/ml/constants.py (100%) rename client/{ => py}/tinychain/ml/interface.py (100%) rename client/{ => py}/tinychain/ml/nn.py (100%) rename client/{ => py}/tinychain/ml/optimizer.py (100%) rename client/{ => py}/tinychain/ml/service.py (100%) rename client/{ => py}/tinychain/ml/variable.py (100%) rename client/{ => py}/tinychain/reflect/__init__.py (100%) rename client/{ => py}/tinychain/reflect/functions.py (100%) rename client/{ => py}/tinychain/reflect/meta.py (100%) rename client/{ => py}/tinychain/reflect/method.py (100%) rename client/{ => py}/tinychain/reflect/op.py (100%) rename client/{ => py}/tinychain/reflect/stub.py (100%) rename client/{ => py}/tinychain/scalar/__init__.py (100%) rename client/{ => py}/tinychain/scalar/base.py (100%) rename client/{ => py}/tinychain/scalar/bound.py (100%) rename client/{ => py}/tinychain/scalar/number.py (100%) rename client/{ => py}/tinychain/scalar/op.py (100%) rename client/{ => py}/tinychain/scalar/ref/__init__.py (100%) rename client/{ => py}/tinychain/scalar/ref/base.py (100%) rename client/{ => py}/tinychain/scalar/ref/functions.py (100%) rename client/{ => py}/tinychain/scalar/value.py (100%) rename client/{ => py}/tinychain/service.py (100%) rename client/{ => py}/tinychain/shape.py (100%) rename client/{ => py}/tinychain/state.py (100%) rename client/{ => py}/tinychain/uri.py (100%) create mode 100644 client/py_async/.gitignore create mode 100644 client/py_async/README.md create mode 100644 client/py_async/docs/Makefile create mode 100644 client/py_async/docs/conf.py create mode 100644 client/py_async/docs/host.rst create mode 100644 client/py_async/docs/index.rst create mode 100644 client/py_async/docs/make.bat rename client/{ => py_async}/docs/requirements.txt (100%) create mode 100644 client/py_async/pyproject.toml create mode 100644 client/py_async/setup.cfg create mode 100644 client/py_async/tinychain_async/__init__.py rename client/{tinychain/host/asynchronous.py => py_async/tinychain_async/host.py} (95%) delete mode 100644 client/tinychain/host/__init__.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4181af46..650f06a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,9 +23,12 @@ If you prefer, you can [fork](https://docs.github.com/en/pull-requests/collabora To run the client unit tests, make sure Python knows where to find your local copy of the TinyChain client: ```bash -# for example, if you cloned into /home/usernames/tinychain -# then the path below should be /home/username/tinychain/client -export PYTHONPATH=$PYTHONPATH://client +# optional: create a virtual environment +# note: python may be python3 on some systems +python -m venv myenv + +# note: pip may be pip3 on some systems +pip install -e client/py ``` Then, try running the client tests to make sure you've got everything configured correctly: @@ -64,14 +67,12 @@ python3 -m tests.tctest.client ## Rust Development -If you need to make changes to the TinyChain host software itself, you'll need to run it locally. Follow the steps in the "Manual Install" section of `INSTALL.md` to set up Rust and ArrayFire, then check that you've gotten everything set up correctly by running the host tests: +If you need to make changes to the TinyChain host software itself, you'll need to run it locally. Follow the steps in the [Manual install](https://docs.tinychain.net/fundamentals/install-tinychain) section of the documentation to set up Rust, then check that you've gotten everything set up correctly by running the host tests: ```bash python3 -m tests.tctest.host ``` -If you have any problems, first check that you have all your environment variables set correctly. If `PYTHONPATH`, `AF_PATH`, and `LD_LIBRARY_PATH` are all set correctly and your installation still doesn't behave as expected, [create an issue](https://github.com/haydnv/tinychain/issues) to get help. - ## Licensing By contributing code to the TinyChain project, you represent that you own the copyright on your contributions, or that you have followed the licensing requirements of the copyright holder, and that TinyChain may use your code without any further restrictions than those specified in the Apache 2.0 open-source license. A copy of the license can be found in the `LICENSE` file in the root directory of the project. diff --git a/node-client/.eslintrc.cjs b/client/node/.eslintrc.cjs similarity index 100% rename from node-client/.eslintrc.cjs rename to client/node/.eslintrc.cjs diff --git a/node-client/.gitignore b/client/node/.gitignore similarity index 100% rename from node-client/.gitignore rename to client/node/.gitignore diff --git a/node-client/.nvmrc b/client/node/.nvmrc similarity index 100% rename from node-client/.nvmrc rename to client/node/.nvmrc diff --git a/node-client/.prettierrc b/client/node/.prettierrc similarity index 100% rename from node-client/.prettierrc rename to client/node/.prettierrc diff --git a/node-client/README.md b/client/node/README.md similarity index 100% rename from node-client/README.md rename to client/node/README.md diff --git a/node-client/babel.config.cjs b/client/node/babel.config.cjs similarity index 100% rename from node-client/babel.config.cjs rename to client/node/babel.config.cjs diff --git a/node-client/jest.config.mjs b/client/node/jest.config.mjs similarity index 100% rename from node-client/jest.config.mjs rename to client/node/jest.config.mjs diff --git a/node-client/package.json b/client/node/package.json similarity index 100% rename from node-client/package.json rename to client/node/package.json diff --git a/node-client/test/uri.spec.js b/client/node/test/uri.spec.js similarity index 100% rename from node-client/test/uri.spec.js rename to client/node/test/uri.spec.js diff --git a/node-client/tinychain/index.js b/client/node/tinychain/index.js similarity index 100% rename from node-client/tinychain/index.js rename to client/node/tinychain/index.js diff --git a/node-client/tinychain/uri.js b/client/node/tinychain/uri.js similarity index 100% rename from node-client/tinychain/uri.js rename to client/node/tinychain/uri.js diff --git a/client/.gitignore b/client/py/.gitignore similarity index 100% rename from client/.gitignore rename to client/py/.gitignore diff --git a/client/README.md b/client/py/README.md similarity index 100% rename from client/README.md rename to client/py/README.md diff --git a/client/docs/Makefile b/client/py/docs/Makefile similarity index 100% rename from client/docs/Makefile rename to client/py/docs/Makefile diff --git a/client/docs/btree.rst b/client/py/docs/btree.rst similarity index 100% rename from client/docs/btree.rst rename to client/py/docs/btree.rst diff --git a/client/docs/chain.rst b/client/py/docs/chain.rst similarity index 100% rename from client/docs/chain.rst rename to client/py/docs/chain.rst diff --git a/client/docs/conf.py b/client/py/docs/conf.py similarity index 100% rename from client/docs/conf.py rename to client/py/docs/conf.py diff --git a/client/docs/generic.rst b/client/py/docs/generic.rst similarity index 100% rename from client/docs/generic.rst rename to client/py/docs/generic.rst diff --git a/client/py/docs/host.rst b/client/py/docs/host.rst new file mode 100644 index 00000000..b58754a4 --- /dev/null +++ b/client/py/docs/host.rst @@ -0,0 +1,6 @@ +:mod:`host` +========================== + +.. automodule:: tinychain.host + :members: + :show-inheritance: diff --git a/client/docs/index.rst b/client/py/docs/index.rst similarity index 97% rename from client/docs/index.rst rename to client/py/docs/index.rst index c63b6ffa..2ac8b051 100644 --- a/client/docs/index.rst +++ b/client/py/docs/index.rst @@ -10,6 +10,7 @@ Modules btree chain generic + host interface math.interface math.linalg diff --git a/client/docs/interface.rst b/client/py/docs/interface.rst similarity index 100% rename from client/docs/interface.rst rename to client/py/docs/interface.rst diff --git a/client/docs/make.bat b/client/py/docs/make.bat similarity index 100% rename from client/docs/make.bat rename to client/py/docs/make.bat diff --git a/client/docs/math.interface.rst b/client/py/docs/math.interface.rst similarity index 100% rename from client/docs/math.interface.rst rename to client/py/docs/math.interface.rst diff --git a/client/docs/math.linalg.rst b/client/py/docs/math.linalg.rst similarity index 100% rename from client/docs/math.linalg.rst rename to client/py/docs/math.linalg.rst diff --git a/client/docs/math.operator.rst b/client/py/docs/math.operator.rst similarity index 100% rename from client/docs/math.operator.rst rename to client/py/docs/math.operator.rst diff --git a/client/docs/ml.nn.rst b/client/py/docs/ml.nn.rst similarity index 100% rename from client/docs/ml.nn.rst rename to client/py/docs/ml.nn.rst diff --git a/client/docs/ml.optimizer.rst b/client/py/docs/ml.optimizer.rst similarity index 100% rename from client/docs/ml.optimizer.rst rename to client/py/docs/ml.optimizer.rst diff --git a/client/docs/ml.service.rst b/client/py/docs/ml.service.rst similarity index 100% rename from client/docs/ml.service.rst rename to client/py/docs/ml.service.rst diff --git a/client/py/docs/requirements.txt b/client/py/docs/requirements.txt new file mode 100644 index 00000000..280468e5 --- /dev/null +++ b/client/py/docs/requirements.txt @@ -0,0 +1,35 @@ +alabaster==0.7.12 +Babel==2.11.0 +certifi==2023.7.22 +charset-normalizer==2.1.1 +cryptography==41.0.6 +distlib==0.3.6 +docker==6.0.1 +docutils==0.18 +filelock==3.9.0 +idna==3.4 +imagesize==1.4.1 +Jinja2==3.1.2 +m2r2==0.3.2 +MarkupSafe==2.1.1 +mistune==0.8.4 +packaging==22.0 +pipenv==2022.12.19 +platformdirs==2.6.2 +Pygments==2.15.0 +pytz==2022.7 +requests==2.31.0 +rjwt==0.1.1 +snowballstemmer==2.2.0 +Sphinx==5.3.0 +sphinx-rtd-theme==1.2 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +urllib3==1.26.18 +virtualenv==20.17.1 +virtualenv-clone==0.5.7 +websocket-client==1.4.2 diff --git a/client/docs/scalar.number.rst b/client/py/docs/scalar.number.rst similarity index 100% rename from client/docs/scalar.number.rst rename to client/py/docs/scalar.number.rst diff --git a/client/docs/scalar.ref.rst b/client/py/docs/scalar.ref.rst similarity index 100% rename from client/docs/scalar.ref.rst rename to client/py/docs/scalar.ref.rst diff --git a/client/docs/scalar.value.rst b/client/py/docs/scalar.value.rst similarity index 100% rename from client/docs/scalar.value.rst rename to client/py/docs/scalar.value.rst diff --git a/client/docs/service.rst b/client/py/docs/service.rst similarity index 100% rename from client/docs/service.rst rename to client/py/docs/service.rst diff --git a/client/docs/state.rst b/client/py/docs/state.rst similarity index 100% rename from client/docs/state.rst rename to client/py/docs/state.rst diff --git a/client/docs/table.rst b/client/py/docs/table.rst similarity index 100% rename from client/docs/table.rst rename to client/py/docs/table.rst diff --git a/client/docs/tensor.rst b/client/py/docs/tensor.rst similarity index 100% rename from client/docs/tensor.rst rename to client/py/docs/tensor.rst diff --git a/client/docs/uri.rst b/client/py/docs/uri.rst similarity index 100% rename from client/docs/uri.rst rename to client/py/docs/uri.rst diff --git a/client/pyproject.toml b/client/py/pyproject.toml similarity index 100% rename from client/pyproject.toml rename to client/py/pyproject.toml diff --git a/client/setup.cfg b/client/py/setup.cfg similarity index 98% rename from client/setup.cfg rename to client/py/setup.cfg index 41d63f8a..5497bdd0 100644 --- a/client/setup.cfg +++ b/client/py/setup.cfg @@ -21,7 +21,6 @@ classifiers = Topic :: Software Development :: Code Generators [options] -exclude = ("tests",) packages = find: python_requires = >=3.9 install_requires = diff --git a/client/tinychain/__init__.py b/client/py/tinychain/__init__.py similarity index 97% rename from client/tinychain/__init__.py rename to client/py/tinychain/__init__.py index 935266e1..a085bb8e 100644 --- a/client/tinychain/__init__.py +++ b/client/py/tinychain/__init__.py @@ -1,7 +1,6 @@ import tinychain.error import tinychain.graph import tinychain.host -import tinychain.host.asynchronous import tinychain.math import tinychain.math.interface import tinychain.math.linalg diff --git a/client/tinychain/base.py b/client/py/tinychain/base.py similarity index 100% rename from client/tinychain/base.py rename to client/py/tinychain/base.py diff --git a/client/tinychain/chain.py b/client/py/tinychain/chain.py similarity index 100% rename from client/tinychain/chain.py rename to client/py/tinychain/chain.py diff --git a/client/tinychain/collection/__init__.py b/client/py/tinychain/collection/__init__.py similarity index 100% rename from client/tinychain/collection/__init__.py rename to client/py/tinychain/collection/__init__.py diff --git a/client/tinychain/collection/base.py b/client/py/tinychain/collection/base.py similarity index 100% rename from client/tinychain/collection/base.py rename to client/py/tinychain/collection/base.py diff --git a/client/tinychain/collection/btree.py b/client/py/tinychain/collection/btree.py similarity index 100% rename from client/tinychain/collection/btree.py rename to client/py/tinychain/collection/btree.py diff --git a/client/tinychain/collection/table.py b/client/py/tinychain/collection/table.py similarity index 100% rename from client/tinychain/collection/table.py rename to client/py/tinychain/collection/table.py diff --git a/client/tinychain/collection/tensor/__init__.py b/client/py/tinychain/collection/tensor/__init__.py similarity index 100% rename from client/tinychain/collection/tensor/__init__.py rename to client/py/tinychain/collection/tensor/__init__.py diff --git a/client/tinychain/collection/tensor/base.py b/client/py/tinychain/collection/tensor/base.py similarity index 100% rename from client/tinychain/collection/tensor/base.py rename to client/py/tinychain/collection/tensor/base.py diff --git a/client/tinychain/collection/tensor/einsum.py b/client/py/tinychain/collection/tensor/einsum.py similarity index 100% rename from client/tinychain/collection/tensor/einsum.py rename to client/py/tinychain/collection/tensor/einsum.py diff --git a/client/tinychain/collection/tensor/functions.py b/client/py/tinychain/collection/tensor/functions.py similarity index 100% rename from client/tinychain/collection/tensor/functions.py rename to client/py/tinychain/collection/tensor/functions.py diff --git a/client/tinychain/collection/tensor/operator.py b/client/py/tinychain/collection/tensor/operator.py similarity index 100% rename from client/tinychain/collection/tensor/operator.py rename to client/py/tinychain/collection/tensor/operator.py diff --git a/client/tinychain/context.py b/client/py/tinychain/context.py similarity index 100% rename from client/tinychain/context.py rename to client/py/tinychain/context.py diff --git a/client/tinychain/decorators.py b/client/py/tinychain/decorators.py similarity index 100% rename from client/tinychain/decorators.py rename to client/py/tinychain/decorators.py diff --git a/client/tinychain/error.py b/client/py/tinychain/error.py similarity index 100% rename from client/tinychain/error.py rename to client/py/tinychain/error.py diff --git a/client/tinychain/generic.py b/client/py/tinychain/generic.py similarity index 100% rename from client/tinychain/generic.py rename to client/py/tinychain/generic.py diff --git a/client/tinychain/graph/__init__.py b/client/py/tinychain/graph/__init__.py similarity index 100% rename from client/tinychain/graph/__init__.py rename to client/py/tinychain/graph/__init__.py diff --git a/client/tinychain/graph/edge.py b/client/py/tinychain/graph/edge.py similarity index 100% rename from client/tinychain/graph/edge.py rename to client/py/tinychain/graph/edge.py diff --git a/client/tinychain/graph/service.py b/client/py/tinychain/graph/service.py similarity index 100% rename from client/tinychain/graph/service.py rename to client/py/tinychain/graph/service.py diff --git a/client/tinychain/host/sync.py b/client/py/tinychain/host.py similarity index 98% rename from client/tinychain/host/sync.py rename to client/py/tinychain/host.py index 9692569a..6f66de40 100644 --- a/client/tinychain/host/sync.py +++ b/client/py/tinychain/host.py @@ -9,12 +9,12 @@ import rjwt import urllib.parse -from ..service import Library, Model, Service -from ..context import to_json -from ..error import * -from ..scalar.value import Nil -from ..state import State -from ..uri import URI +from .service import Library, Model, Service +from .context import to_json +from .error import * +from .scalar.value import Nil +from .state import State +from .uri import URI DEFAULT_PORT = 8702 diff --git a/client/tinychain/interface.py b/client/py/tinychain/interface.py similarity index 100% rename from client/tinychain/interface.py rename to client/py/tinychain/interface.py diff --git a/client/tinychain/json.py b/client/py/tinychain/json.py similarity index 100% rename from client/tinychain/json.py rename to client/py/tinychain/json.py diff --git a/client/tinychain/math/__init__.py b/client/py/tinychain/math/__init__.py similarity index 100% rename from client/tinychain/math/__init__.py rename to client/py/tinychain/math/__init__.py diff --git a/client/tinychain/math/base.py b/client/py/tinychain/math/base.py similarity index 100% rename from client/tinychain/math/base.py rename to client/py/tinychain/math/base.py diff --git a/client/tinychain/math/constants.py b/client/py/tinychain/math/constants.py similarity index 100% rename from client/tinychain/math/constants.py rename to client/py/tinychain/math/constants.py diff --git a/client/tinychain/math/equation.py b/client/py/tinychain/math/equation.py similarity index 100% rename from client/tinychain/math/equation.py rename to client/py/tinychain/math/equation.py diff --git a/client/tinychain/math/interface.py b/client/py/tinychain/math/interface.py similarity index 100% rename from client/tinychain/math/interface.py rename to client/py/tinychain/math/interface.py diff --git a/client/tinychain/math/linalg.py b/client/py/tinychain/math/linalg.py similarity index 100% rename from client/tinychain/math/linalg.py rename to client/py/tinychain/math/linalg.py diff --git a/client/tinychain/math/operator.py b/client/py/tinychain/math/operator.py similarity index 100% rename from client/tinychain/math/operator.py rename to client/py/tinychain/math/operator.py diff --git a/client/tinychain/ml/__init__.py b/client/py/tinychain/ml/__init__.py similarity index 100% rename from client/tinychain/ml/__init__.py rename to client/py/tinychain/ml/__init__.py diff --git a/client/tinychain/ml/activation.py b/client/py/tinychain/ml/activation.py similarity index 100% rename from client/tinychain/ml/activation.py rename to client/py/tinychain/ml/activation.py diff --git a/client/tinychain/ml/constants.py b/client/py/tinychain/ml/constants.py similarity index 100% rename from client/tinychain/ml/constants.py rename to client/py/tinychain/ml/constants.py diff --git a/client/tinychain/ml/interface.py b/client/py/tinychain/ml/interface.py similarity index 100% rename from client/tinychain/ml/interface.py rename to client/py/tinychain/ml/interface.py diff --git a/client/tinychain/ml/nn.py b/client/py/tinychain/ml/nn.py similarity index 100% rename from client/tinychain/ml/nn.py rename to client/py/tinychain/ml/nn.py diff --git a/client/tinychain/ml/optimizer.py b/client/py/tinychain/ml/optimizer.py similarity index 100% rename from client/tinychain/ml/optimizer.py rename to client/py/tinychain/ml/optimizer.py diff --git a/client/tinychain/ml/service.py b/client/py/tinychain/ml/service.py similarity index 100% rename from client/tinychain/ml/service.py rename to client/py/tinychain/ml/service.py diff --git a/client/tinychain/ml/variable.py b/client/py/tinychain/ml/variable.py similarity index 100% rename from client/tinychain/ml/variable.py rename to client/py/tinychain/ml/variable.py diff --git a/client/tinychain/reflect/__init__.py b/client/py/tinychain/reflect/__init__.py similarity index 100% rename from client/tinychain/reflect/__init__.py rename to client/py/tinychain/reflect/__init__.py diff --git a/client/tinychain/reflect/functions.py b/client/py/tinychain/reflect/functions.py similarity index 100% rename from client/tinychain/reflect/functions.py rename to client/py/tinychain/reflect/functions.py diff --git a/client/tinychain/reflect/meta.py b/client/py/tinychain/reflect/meta.py similarity index 100% rename from client/tinychain/reflect/meta.py rename to client/py/tinychain/reflect/meta.py diff --git a/client/tinychain/reflect/method.py b/client/py/tinychain/reflect/method.py similarity index 100% rename from client/tinychain/reflect/method.py rename to client/py/tinychain/reflect/method.py diff --git a/client/tinychain/reflect/op.py b/client/py/tinychain/reflect/op.py similarity index 100% rename from client/tinychain/reflect/op.py rename to client/py/tinychain/reflect/op.py diff --git a/client/tinychain/reflect/stub.py b/client/py/tinychain/reflect/stub.py similarity index 100% rename from client/tinychain/reflect/stub.py rename to client/py/tinychain/reflect/stub.py diff --git a/client/tinychain/scalar/__init__.py b/client/py/tinychain/scalar/__init__.py similarity index 100% rename from client/tinychain/scalar/__init__.py rename to client/py/tinychain/scalar/__init__.py diff --git a/client/tinychain/scalar/base.py b/client/py/tinychain/scalar/base.py similarity index 100% rename from client/tinychain/scalar/base.py rename to client/py/tinychain/scalar/base.py diff --git a/client/tinychain/scalar/bound.py b/client/py/tinychain/scalar/bound.py similarity index 100% rename from client/tinychain/scalar/bound.py rename to client/py/tinychain/scalar/bound.py diff --git a/client/tinychain/scalar/number.py b/client/py/tinychain/scalar/number.py similarity index 100% rename from client/tinychain/scalar/number.py rename to client/py/tinychain/scalar/number.py diff --git a/client/tinychain/scalar/op.py b/client/py/tinychain/scalar/op.py similarity index 100% rename from client/tinychain/scalar/op.py rename to client/py/tinychain/scalar/op.py diff --git a/client/tinychain/scalar/ref/__init__.py b/client/py/tinychain/scalar/ref/__init__.py similarity index 100% rename from client/tinychain/scalar/ref/__init__.py rename to client/py/tinychain/scalar/ref/__init__.py diff --git a/client/tinychain/scalar/ref/base.py b/client/py/tinychain/scalar/ref/base.py similarity index 100% rename from client/tinychain/scalar/ref/base.py rename to client/py/tinychain/scalar/ref/base.py diff --git a/client/tinychain/scalar/ref/functions.py b/client/py/tinychain/scalar/ref/functions.py similarity index 100% rename from client/tinychain/scalar/ref/functions.py rename to client/py/tinychain/scalar/ref/functions.py diff --git a/client/tinychain/scalar/value.py b/client/py/tinychain/scalar/value.py similarity index 100% rename from client/tinychain/scalar/value.py rename to client/py/tinychain/scalar/value.py diff --git a/client/tinychain/service.py b/client/py/tinychain/service.py similarity index 100% rename from client/tinychain/service.py rename to client/py/tinychain/service.py diff --git a/client/tinychain/shape.py b/client/py/tinychain/shape.py similarity index 100% rename from client/tinychain/shape.py rename to client/py/tinychain/shape.py diff --git a/client/tinychain/state.py b/client/py/tinychain/state.py similarity index 100% rename from client/tinychain/state.py rename to client/py/tinychain/state.py diff --git a/client/tinychain/uri.py b/client/py/tinychain/uri.py similarity index 100% rename from client/tinychain/uri.py rename to client/py/tinychain/uri.py diff --git a/client/py_async/.gitignore b/client/py_async/.gitignore new file mode 100644 index 00000000..ca29a4ba --- /dev/null +++ b/client/py_async/.gitignore @@ -0,0 +1,133 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# IDE metadata +.project +.pydevproject diff --git a/client/py_async/README.md b/client/py_async/README.md new file mode 100644 index 00000000..7836cb5e --- /dev/null +++ b/client/py_async/README.md @@ -0,0 +1 @@ +An aynchronous Python client for TinyChain: http://git.io/JtryR diff --git a/client/py_async/docs/Makefile b/client/py_async/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/client/py_async/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/client/py_async/docs/conf.py b/client/py_async/docs/conf.py new file mode 100644 index 00000000..53ba4f1a --- /dev/null +++ b/client/py_async/docs/conf.py @@ -0,0 +1,62 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- + +project = 'TinyChain' +copyright = '2024' +author = 'The TinyChain Contributors' + +# The full version, including alpha/beta/rc tags +release = '0.15.0' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'm2r2', + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', +] + +source_suffix = ['.rst', '.md'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] diff --git a/client/py_async/docs/host.rst b/client/py_async/docs/host.rst new file mode 100644 index 00000000..b58754a4 --- /dev/null +++ b/client/py_async/docs/host.rst @@ -0,0 +1,6 @@ +:mod:`host` +========================== + +.. automodule:: tinychain.host + :members: + :show-inheritance: diff --git a/client/py_async/docs/index.rst b/client/py_async/docs/index.rst new file mode 100644 index 00000000..4620f0f3 --- /dev/null +++ b/client/py_async/docs/index.rst @@ -0,0 +1,10 @@ +.. mdinclude:: ../README.md + + +Modules +------- + +.. toctree:: + :maxdepth: 1 + + host diff --git a/client/py_async/docs/make.bat b/client/py_async/docs/make.bat new file mode 100644 index 00000000..2119f510 --- /dev/null +++ b/client/py_async/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/client/docs/requirements.txt b/client/py_async/docs/requirements.txt similarity index 100% rename from client/docs/requirements.txt rename to client/py_async/docs/requirements.txt diff --git a/client/py_async/pyproject.toml b/client/py_async/pyproject.toml new file mode 100644 index 00000000..121a39f5 --- /dev/null +++ b/client/py_async/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools >= 40.6.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/client/py_async/setup.cfg b/client/py_async/setup.cfg new file mode 100644 index 00000000..6043ae5b --- /dev/null +++ b/client/py_async/setup.cfg @@ -0,0 +1,34 @@ +[metadata] +name = tinychain_async +version = 0.15.0 +author = The TinyChain Contributors +author_email = code@tinychain.net +description = An asynchronous Python client for TinyChain: http://git.io/JtryR +keywords = 'distributed cross-service transaction ml platform' +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/haydnv/tinychain +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + Intended Audience :: Financial and Insurance Industry + Intended Audience :: Information Technology + Intended Audience :: Science/Research + Programming Language :: Python :: 3 + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Topic :: Software Development + +[options] +packages = find: +python_requires = >=3.9 +install_requires = + requests + +[build_sphinx] +source-dir = docs/source +build-dir = docs/_build +all_files = 1 + +[upload_sphinx] +upload-dir = docs/_build/html diff --git a/client/py_async/tinychain_async/__init__.py b/client/py_async/tinychain_async/__init__.py new file mode 100644 index 00000000..23c3f629 --- /dev/null +++ b/client/py_async/tinychain_async/__init__.py @@ -0,0 +1 @@ +import tinychain_async.host diff --git a/client/tinychain/host/asynchronous.py b/client/py_async/tinychain_async/host.py similarity index 95% rename from client/tinychain/host/asynchronous.py rename to client/py_async/tinychain_async/host.py index 09dd1d97..68057013 100644 --- a/client/tinychain/host/asynchronous.py +++ b/client/py_async/tinychain_async/host.py @@ -2,10 +2,10 @@ import asyncio import json -from ..context import to_json -from ..uri import URI +from tinychain.context import to_json +from tinychain.uri import URI -from .sync import auth_header, Local, Host +from tinychain.host import auth_header, Local, Host BACKOFF = 0.01 diff --git a/client/tinychain/host/__init__.py b/client/tinychain/host/__init__.py deleted file mode 100644 index d7a307d9..00000000 --- a/client/tinychain/host/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .sync import Local, Host diff --git a/tests/tctest/benchmark.py b/tests/tctest/benchmark.py index b0495abb..6562867a 100644 --- a/tests/tctest/benchmark.py +++ b/tests/tctest/benchmark.py @@ -9,7 +9,7 @@ import time import rjwt -import tinychain as tc +import tinychain_async as tc_async from .process import Local @@ -58,7 +58,7 @@ def start_local_host(ns, host_uri=None, public_key=None, wait_time=1, **flags): **flags) process.start(wait_time) - return tc.host.asyncronous.Local(process, f"http://{process.ADDRESS}:{port}") + return tc_async.host.Local(process, f"http://{process.ADDRESS}:{port}") class Benchmark(object): @@ -70,7 +70,6 @@ def __iter__(self): yield item async def run(self, requests, concurrency): - requests = [pause_and_run(request) for request in requests] responses = [] start = time.time() @@ -96,8 +95,6 @@ def stop(self): async def main(benchmarks): - import sys - parser = argparse.ArgumentParser(description="Run benchmarks") parser.add_argument('-k', type=str, help="filter benchmarks to run by name") parser.add_argument('--cache_size', type=str, default=DEFAULT_CACHE_SIZE, help="host cache size") @@ -141,9 +138,3 @@ async def main(benchmarks): benchmark.stop() # clean the workspace again after running a benchmark shutil.rmtree(WORKSPACE) - - -async def pause_and_run(task, pause_length = None): - pause_length = (random.random() / 10) if pause_length is None else pause_length - await asyncio.sleep(pause_length) - return await task diff --git a/tests/tctest/process.py b/tests/tctest/process.py index 7463e220..494c1c12 100644 --- a/tests/tctest/process.py +++ b/tests/tctest/process.py @@ -6,8 +6,8 @@ import subprocess import time -import rjwt import tinychain as tc +import tinychain_async as tc_async CONFIG = "config" @@ -215,7 +215,7 @@ def start_local_host(ns, host_uri=None, public_key=None, wait_time=1, **flags): def start_local_host_async(ns, host_uri=None, public_key=None, wait_time=1, **flags): process, port = _start_local_host_process(ns, host_uri, public_key, wait_time, **flags) - return tc.host.asynchronous.Local(process, f"http://{process.ADDRESS}:{port}") + return tc_async.host.Local(process, f"http://{process.ADDRESS}:{port}") # use this alias to switch between Local and Docker host types From a4b64f8a1d981583450e3a47f03f08f9e34d2988 Mon Sep 17 00:00:00 2001 From: Haydn Vestal Date: Mon, 29 Jan 2024 19:01:37 +0530 Subject: [PATCH 09/10] bump version number to 0.16 & update sub-crates (#280) * move all clients under a single directory and separate sync and async Python client functionality into two different packages * bump crate versions --- client/py/docs/conf.py | 2 +- client/py_async/docs/conf.py | 3 +-- client/py_async/setup.cfg | 2 +- host/Cargo.toml | 22 +++++++++++----------- host/chain/Cargo.toml | 14 +++++++------- host/collection/Cargo.toml | 12 ++++++------ host/collection/src/tensor/shape.rs | 10 ---------- host/error/Cargo.toml | 8 ++++---- host/fs/Cargo.toml | 19 ++++++++++--------- host/generic/Cargo.toml | 4 ++-- host/scalar/Cargo.toml | 10 +++++----- host/state/Cargo.toml | 18 +++++++++--------- host/transact/Cargo.toml | 12 ++++++------ host/value/Cargo.toml | 6 +++--- 14 files changed, 66 insertions(+), 76 deletions(-) diff --git a/client/py/docs/conf.py b/client/py/docs/conf.py index 2edf7dcc..17884c67 100644 --- a/client/py/docs/conf.py +++ b/client/py/docs/conf.py @@ -22,7 +22,7 @@ author = 'The TinyChain Contributors' # The full version, including alpha/beta/rc tags -release = '0.15.0' +release = '0.16.0' # -- General configuration --------------------------------------------------- diff --git a/client/py_async/docs/conf.py b/client/py_async/docs/conf.py index 53ba4f1a..5a6a2544 100644 --- a/client/py_async/docs/conf.py +++ b/client/py_async/docs/conf.py @@ -22,8 +22,7 @@ author = 'The TinyChain Contributors' # The full version, including alpha/beta/rc tags -release = '0.15.0' - +release = '0.16.0' # -- General configuration --------------------------------------------------- diff --git a/client/py_async/setup.cfg b/client/py_async/setup.cfg index 6043ae5b..3925a95d 100644 --- a/client/py_async/setup.cfg +++ b/client/py_async/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = tinychain_async -version = 0.15.0 +version = 0.16.0 author = The TinyChain Contributors author_email = code@tinychain.net description = An asynchronous Python client for TinyChain: http://git.io/JtryR diff --git a/host/Cargo.toml b/host/Cargo.toml index 81a6b94f..9994eda0 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tinychain" -version = "0.15.0" +version = "0.16.0" authors = ["code@tinychain.net"] edition = "2021" description = "Next-gen cloud service runtime" @@ -36,19 +36,19 @@ hyper = { version = "0.14", features = ["full"] } log = { version = "0.4", features = ["release_max_level_info"] } num_cpus = "1.16" pin-project = "1.1" -rjwt = { path = "../../rjwt/rust" } +rjwt = "0.5" safecast = "0.2" serde = { version = "1.0", features = [] } serde_json = { version = "1.0" } tbon = "0.5" -tc-chain = { path = "chain" } -tc-collection = { path = "collection" } -tc-error = { path = "error" } -tc-fs = { path = "fs" } -tc-scalar = { path = "scalar" } -tc-state = { path = "state" } -tc-transact = { path = "transact" } -tc-value = { path = "value" } -tcgeneric = { path = "generic" } +tc-chain = "0.4" +tc-collection = "0.4" +tc-error = "0.11" +tc-fs = "0.4" +tc-scalar = "0.4" +tc-state = "0.4" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" tokio = { version = "1.35", features = ["rt-multi-thread", "signal"] } url = { version = "2.5" } diff --git a/host/chain/Cargo.toml b/host/chain/Cargo.toml index ffd64b45..9d186106 100644 --- a/host/chain/Cargo.toml +++ b/host/chain/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tc-chain" authors = ["code@tinychain.net"] -version = "0.3.0" +version = "0.4.0" edition = "2021" description = "TinyChain's BlockChain and SyncChain data structures" license = "Apache-2.0" @@ -21,9 +21,9 @@ log = { version = "0.4" } num_cpus = "1.16" safecast = "0.2" tbon = "0.5" -tc-collection = { path = "../collection" } -tc-error = { path = "../error" } -tc-scalar = { path = "../scalar" } -tc-transact = { path = "../transact" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-collection = "0.4" +tc-error = "0.11" +tc-scalar = "0.4" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" diff --git a/host/collection/Cargo.toml b/host/collection/Cargo.toml index 9580d0d2..d4131ca2 100644 --- a/host/collection/Cargo.toml +++ b/host/collection/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-collection" -version = "0.3.0" +version = "0.4.0" authors = ["code@tinychain.net"] edition = "2021" description = "TinyChain's collection types" @@ -32,9 +32,9 @@ smallvec = "1.11" pin-project = "1.1" rayon = "1.8" safecast = "0.2" -tc-error = { path = "../error" } -tc-scalar = { path = "../scalar" } -tc-transact = { path = "../transact" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-error = "0.11" +tc-scalar = "0.4" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" tokio = { version = "1.35", features = ["sync"] } diff --git a/host/collection/src/tensor/shape.rs b/host/collection/src/tensor/shape.rs index bb8d5ae1..cc5a7061 100644 --- a/host/collection/src/tensor/shape.rs +++ b/host/collection/src/tensor/shape.rs @@ -278,16 +278,6 @@ impl Range { } /// Expand this `Range` to the entire given [`Shape`]. - /// - /// Example: - /// ``` - /// # use smallvec::smallvec; - /// # use tc_collection::tensor::{Range, Shape}; - /// let mut range = Range::from(&[0u64][..]); - /// assert_eq!( - /// range.to_shape(&Shape::from(smallvec![2, 3, 4])).unwrap(), - /// Shape::from(smallvec![3, 4])); - /// ``` pub fn normalize(mut self, shape: &[u64]) -> Self { assert!(shape.len() >= self.len()); diff --git a/host/error/Cargo.toml b/host/error/Cargo.toml index 916a0418..33a76679 100644 --- a/host/error/Cargo.toml +++ b/host/error/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-error" -version = "0.10.0" +version = "0.11.0" authors = ["code@tinychain.net"] edition = "2021" description = "TinyChain's generic error struct" @@ -16,7 +16,7 @@ destream = "0.7" ha-ndarray = "0.3" log = { version = "0.4", features = ["release_max_level_info"] } pathlink = "0.2" -rjwt = { path = "../../../rjwt/rust" } +rjwt = "0.5" serde = "1.0" -txfs = { path = "../../../txfs" } -txn_lock = { path = "../../../txn_lock" } +txfs = "0.3" +txn_lock = "0.9" diff --git a/host/fs/Cargo.toml b/host/fs/Cargo.toml index 86bbf2e1..b0b24c44 100644 --- a/host/fs/Cargo.toml +++ b/host/fs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-fs" -version = "0.3.0" +version = "0.4.0" authors = ["code@tinychain.net"] edition = "2021" description = "TinyChain's transactional filesystem interface layer" @@ -23,15 +23,16 @@ get-size = "0.1" get-size-derive = "0.1" log = { version = "0.4", features = ["release_max_level_info"] } num_cpus = "1.16" -rjwt = { path = "../../../rjwt/rust" } +rjwt = "0.5" safecast = "0.2" tbon = "0.5" -tc-chain = { path = "../chain" } -tc-collection = { path = "../collection" } -tc-error = { path = "../error" } -tc-scalar = { path = "../scalar" } -tc-transact = { path = "../transact" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-chain = "0.4" +tc-collection = "0.4" +tc-error = "0.11" +tc-scalar = "0.4" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" tokio = { version = "1.35", features = ["rt-multi-thread", "signal"] } tokio-util = { version = "0.7", features = ["io"] } + diff --git a/host/generic/Cargo.toml b/host/generic/Cargo.toml index 58192eed..fe1e66dc 100644 --- a/host/generic/Cargo.toml +++ b/host/generic/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tcgeneric" -version = "0.9.0" +version = "0.10.0" authors = ["code@tinychain.net"] edition = "2021" description = "Generic data types used internally by TinyChain" @@ -23,4 +23,4 @@ safecast = "0.2" serde = { version = "1.0", features = [] } sha2 = "0.10" smallvec = "1.11" -tc-error = { path = "../error" } +tc-error = "0.11" diff --git a/host/scalar/Cargo.toml b/host/scalar/Cargo.toml index cbb4892f..6a31a5b7 100644 --- a/host/scalar/Cargo.toml +++ b/host/scalar/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tc-scalar" authors = ["code@tinychain.net"] -version = "0.3.0" +version = "0.4.0" edition = "2021" description = "TinyChain's scalar value enum, including concurrently-resolvable op definitions and references" license = "Apache-2.0" @@ -24,7 +24,7 @@ num_cpus = "1.13" safecast = "0.2" serde = { version = "1.0", features = [] } smallvec = "1.11" -tc-error = { path = "../error" } -tc-transact = { path = "../transact" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-error = "0.11" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" diff --git a/host/state/Cargo.toml b/host/state/Cargo.toml index 906134fd..f251dbb0 100644 --- a/host/state/Cargo.toml +++ b/host/state/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-state" -version = "0.3.0" +version = "0.4.0" authors = ["code@tinychain.net"] edition = "2021" description = "TinyChain's general state enum" @@ -22,11 +22,11 @@ get-size-derive = "0.1" log = "0.4" num_cpus = "1.16" safecast = "0.2" -tc-chain = { path = "../chain" } -tc-collection = { path = "../collection" } -tc-error = { path = "../error" } -tc-fs = { path = "../fs" } -tc-scalar = { path = "../scalar" } -tc-transact = { path = "../transact" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-chain = "0.4" +tc-collection = "0.4" +tc-error = "0.11" +tc-fs = "0.4" +tc-scalar = "0.4" +tc-transact = "0.20" +tc-value = "0.12" +tcgeneric = "0.10" diff --git a/host/transact/Cargo.toml b/host/transact/Cargo.toml index 6d2d6bc4..12bbb1ce 100644 --- a/host/transact/Cargo.toml +++ b/host/transact/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-transact" -version = "0.19.0" +version = "0.20.0" authors = ["code@tinychain.net"] edition = "2021" description = "Traits and locking utilities for a TinyChain transaction." @@ -24,9 +24,9 @@ num_cpus = "1.16" rand = "0.8" safecast = "0.2" sha2 = "0.10" -tc-error = { path = "../error" } -tc-value = { path = "../value" } -tcgeneric = { path = "../generic" } +tc-error = "0.11" +tc-value = "0.12" +tcgeneric = "0.10" tokio = { version = "1.35", features = ["sync"] } -txfs = { path = "../../../txfs", features = ["logging"] } -txn_lock = { path = "../../../txn_lock", features = ["logging"] } +txfs = { version = "0.3", features = ["logging"] } +txn_lock = { version = "0.9", features = ["logging"] } diff --git a/host/value/Cargo.toml b/host/value/Cargo.toml index b26857d6..51d34616 100644 --- a/host/value/Cargo.toml +++ b/host/value/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tc-value" -version = "0.11.0" +version = "0.12.0" authors = ["code@tinychain.net"] edition = "2021" description = "TinyChain's representation of a value which can be collated and stored." @@ -31,6 +31,6 @@ safecast = "0.2" serde = { version = "1.0", features = [] } serde_json = { version = "1.0", features = [] } smallvec = { version = "1.11", features = ["serde"] } -tc-error = { path = "../error" } -tcgeneric = { path = "../generic" } +tc-error = "0.11" +tcgeneric = "0.10" uuid = { version = "1.6", features = ["v4"] } From 8707dcf05ca16009eee2dba15be0c8284d1866f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:25:57 +0000 Subject: [PATCH 10/10] Bump jinja2 from 3.1.2 to 3.1.3 in /client/docs Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- client/py_async/docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/py_async/docs/requirements.txt b/client/py_async/docs/requirements.txt index 942b6130..3c1023e2 100644 --- a/client/py_async/docs/requirements.txt +++ b/client/py_async/docs/requirements.txt @@ -10,7 +10,7 @@ docutils==0.18 filelock==3.9.0 idna==3.4 imagesize==1.4.1 -Jinja2==3.1.2 +Jinja2==3.1.3 m2r2==0.3.2 MarkupSafe==2.1.1 mistune==0.8.4