diff --git a/pipeline/src/builder.rs b/pipeline/src/builder.rs index e5a429914..ac3ead89f 100644 --- a/pipeline/src/builder.rs +++ b/pipeline/src/builder.rs @@ -7,21 +7,28 @@ use crate::{executor::Executor, ROOT_DIR}; pub fn parse_metadata(path: &str) -> Metadata { let manifest = std::path::Path::new(path).join("Cargo.toml"); let mut metadata_cmd = cargo_metadata::MetadataCommand::new(); + metadata_cmd .no_deps() .manifest_path(manifest) .exec() - .unwrap() + .expect("Couldn't parse metadata") } pub trait GuestMetadata { // /// Kind of target ("bin", "example", "test", "bench", "lib", "custom-build") fn get_tests(&self, names: &[&str]) -> Vec; + fn get_bins(&self, names: &[&str]) -> Vec; + fn tests(&self) -> Vec<&Target>; + fn bins(&self) -> Vec<&Target>; + fn benchs(&self) -> Vec<&Target>; + fn libs(&self) -> Vec<&Target>; + fn build_scripts(&self) -> Vec<&Target>; } @@ -103,7 +110,6 @@ pub struct CommandBuilder { pub sanitized_env: Vec, pub cargo: Option, - // rustc compiler specific to toolchain pub rustc: Option, // -C flags @@ -121,27 +127,32 @@ pub struct CommandBuilder { } impl CommandBuilder { + fn get_path_buf(tool: &str, toolchain: &str) -> Option { + let std::io::Result::Ok(std::process::Output { stdout, .. }) = sanitized_cmd("rustup") + .args([&format!("+{toolchain}"), "which", tool]) + .output() + else { + return None; + }; + + let Ok(out) = String::from_utf8(stdout) else { + return None; + }; + + let out = out.trim(); + + println!("Using {tool}: {out}"); + + Some(PathBuf::from(out)) + } + pub fn new(meta: &Metadata, target: &str, toolchain: &str) -> Self { - let tools = ["cargo", "rustc"] - .into_iter() - .map(|tool| { - let out = sanitized_cmd("rustup") - .args([format!("+{toolchain}").as_str(), "which", tool]) - .output() - .expect("rustup failed to find {toolchain} toolchain") - .stdout; - let out = String::from_utf8(out).unwrap(); - let out = out.trim(); - println!("Using rustc: {out}"); - PathBuf::from(out) - }) - .collect::>(); Self { meta: meta.clone(), target: target.to_string(), + cargo: CommandBuilder::get_path_buf("cargo", toolchain), + rustc: CommandBuilder::get_path_buf("rustc", toolchain), sanitized_env: Vec::new(), - cargo: Some(tools[0].clone()), - rustc: Some(tools[1].clone()), rust_flags: None, z_flags: None, cc_compiler: None, @@ -203,33 +214,34 @@ impl CommandBuilder { pub fn sanitize(&self, cmd: &mut Command, filter_cargo: bool) { if filter_cargo { - for (key, _val) in env::vars().filter(|x| x.0.starts_with("CARGO")) { + for (key, _val) in env::vars().filter(|(key, _)| key.starts_with("CARGO")) { cmd.env_remove(key); } } - self.sanitized_env.iter().for_each(|e| { - cmd.env_remove(e); - }); + for key in self.sanitized_env.iter() { + cmd.env_remove(key); + } } /// Runs cargo build and returns paths of the artifacts - // target/ - // ├── debug/ - // ├── deps/ - // │ |── main- --> this is the output - // │ |── main- - // │ └── bin2- --> this is the output - // ├── build/ - // ├── main --> this is the output (same) - // └── bin2 --> this is the output (same) + /// target/ + /// ├── debug/ + /// ├── deps/ + /// │ |── main- --> this is the output + /// │ |── main- + /// │ └── bin2- --> this is the output + /// ├── build/ + /// ├── main --> this is the output (same) + /// └── bin2 --> this is the output (same) pub fn build_command(&self, profile: &str, bins: &[String]) -> Executor { - let args = vec!["build".to_string()]; - let cmd = self.inner_command(args, profile, bins.to_owned()); + let cmd = self.inner_command(vec!["build".to_owned()], profile, bins.to_owned()); + let target_path: PathBuf = self .meta .target_directory .join(self.target.clone()) .join(profile); + let artifacts = self .meta .bins() @@ -246,25 +258,31 @@ impl CommandBuilder { } /// Runs cargo test and returns *incomplete* paths of the artifacts - // target/ - // ├── debug/ - // ├── deps/ - // │ |── main- - // │ |── main- --> this is the test - // │ |── bin2- - // │ └── my-test- --> this is the test - // ├── build/ - // Thus the test artifacts path are hypothetical because we don't know the hash yet + /// target/ + /// ├── debug/ + /// ├── deps/ + /// │ |── main- + /// │ |── main- --> this is the test + /// │ |── bin2- + /// │ └── my-test- --> this is the test + /// ├── build/ + /// Thus the test artifacts path are hypothetical because we don't know the hash yet pub fn test_command(&self, profile: &str, bins: &Vec) -> Executor { - let args = vec!["test".to_string(), "--no-run".to_string()]; - let cmd = self.inner_command(args, profile, bins.clone()); + let cmd = self.inner_command( + vec!["test".to_owned(), "--no-run".to_owned()], + profile, + bins.clone(), + ); + let target_path: PathBuf = self .meta .target_directory .join(self.target.clone()) .join(profile) .join("deps"); - println!("tests {:?}", bins); + + println!("tests {bins:?}"); + let artifacts = self .meta .tests() @@ -302,13 +320,15 @@ impl CommandBuilder { // `--{profile} {bin} --target {target} --locked -Z {z_flags}` if profile != "debug" { // error: unexpected argument '--debug' found; tip: `--debug` is the default - args.push(format!("--{}", profile)); + args.push(format!("--{profile}")); } + args.extend(vec![ - "--target".to_string(), - target.clone(), + "--target".to_owned(), + target, // "--locked".to_string(), ]); + if !bins.is_empty() { let libs = meta .libs() @@ -320,22 +340,28 @@ impl CommandBuilder { args.extend(format_flags("--lib", &libs)); args.extend(format_flags("--bin", &bins)); } + if let Some(z_flags) = z_flags { args.extend(format_flags("-Z", &z_flags)); } // Construct command from the toolchain-specific cargo - let mut cmd = - Command::new(cargo.map_or("cargo".to_string(), |c| String::from(c.to_str().unwrap()))); + let mut cmd = Command::new(cargo.map_or("cargo".to_owned(), |c| { + c.to_str().expect("Output is not valid UTF-8").to_owned() + })); + // Clear unwanted env vars self.sanitize(&mut cmd, true); - cmd.current_dir(ROOT_DIR.get().unwrap()); + cmd.current_dir(ROOT_DIR.get().expect("No reference to ROOT_DIR")); // Set Rustc compiler path and flags cmd.env( "RUSTC", - rustc.map_or("rustc".to_string(), |c| String::from(c.to_str().unwrap())), + rustc.map_or("rustc".to_string(), |c| { + c.to_str().expect("Output is not valid UTF-8").to_owned() + }), ); + if let Some(rust_flags) = rust_flags { cmd.env( "CARGO_ENCODED_RUSTFLAGS", @@ -347,6 +373,7 @@ impl CommandBuilder { if let Some(cc_compiler) = cc_compiler { cmd.env("CC", cc_compiler); } + if let Some(c_flags) = c_flags { cmd.env(format!("CC_{}", self.target), c_flags.join(" ")); } @@ -363,16 +390,15 @@ fn to_strings(strs: &[&str]) -> Vec { } pub fn format_flags(flag: &str, items: &[String]) -> Vec { - let res = items.iter().fold(Vec::new(), |mut res, i| { + items.iter().fold(Vec::new(), |mut res, i| { res.extend([flag.to_owned(), i.to_owned()]); res - }); - res + }) } fn sanitized_cmd(tool: &str) -> Command { let mut cmd = Command::new(tool); - for (key, _val) in env::vars().filter(|x| x.0.starts_with("CARGO")) { + for (key, _val) in env::vars().filter(|(key, _)| key.starts_with("CARGO")) { cmd.env_remove(key); } cmd.env_remove("RUSTUP_TOOLCHAIN"); diff --git a/pipeline/src/executor.rs b/pipeline/src/executor.rs index a7bca210b..8d5e9e267 100644 --- a/pipeline/src/executor.rs +++ b/pipeline/src/executor.rs @@ -1,16 +1,13 @@ -use anyhow::Result; - -use crate::ROOT_DIR; +use anyhow::bail; use regex::Regex; use std::io::BufRead; +use std::path::Path; use std::{ - fs, - io::{BufReader, Write}, + io::BufReader, path::PathBuf, process::{Command, Stdio}, thread, }; -use std::{fs::File, path::Path}; #[derive(Debug)] pub struct Executor { @@ -26,30 +23,44 @@ impl Executor { .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .unwrap(); + .expect("Couldn't spawn child process"); - let stdout = BufReader::new(child.stdout.take().unwrap()); - let stderr = BufReader::new(child.stderr.take().unwrap()); + let stdout = BufReader::new(child.stdout.take().expect("Couldn't take stdout of child")); + let stderr = BufReader::new(child.stderr.take().expect("Couldn't take stderr of child")); let stdout_handle = thread::spawn(move || { - stdout.lines().for_each(|line| { - println!("[docker] {}", line.unwrap()); - }); + for line in stdout.lines().enumerate().map(|(index, line)| { + line.unwrap_or_else(|e| { + panic!("Couldn't get stdout line: {index}\n with error: {e}") + }) + }) { + println!("[docker] {line}"); + } }); - stderr.lines().for_each(|line| { - let line = line.unwrap(); - println!("[zkvm-stdout] {}", line); + + for line in stderr.lines().enumerate().map(|(index, line)| { + line.unwrap_or_else(|e| panic!("Couldn't get stderr line: {index}\n with error: {e}")) + }) { + println!("[zkvm-stdout] {line}"); + if self.test && line.contains("Executable unittests") { if let Some(test) = extract_path(&line) { - self.artifacts + let Some(artifact) = self + .artifacts .iter_mut() .find(|a| file_name(&test).contains(&file_name(a).replace('-', "_"))) - .map(|a| *a = test) - .expect("Failed to find test artifact"); + else { + bail!("Failed to find test artifact"); + }; + + *artifact = test; } } - }); - stdout_handle.join().unwrap(); + } + + stdout_handle + .join() + .expect("Couldn't wait for stdout handle to finish"); let result = child.wait()?; if !result.success() { @@ -60,59 +71,93 @@ impl Executor { } #[cfg(feature = "sp1")] - pub fn sp1_placement(&self, dest: &str) -> Result<()> { - let root = ROOT_DIR.get().unwrap(); + pub fn sp1_placement(&self, dest: &str) -> anyhow::Result<()> { + use std::fs; + + let root = crate::ROOT_DIR.get().expect("No reference to ROOT_DIR"); let dest = PathBuf::from(dest); + if !dest.exists() { - fs::create_dir_all(&dest).unwrap(); + fs::create_dir_all(&dest).expect("Couldn't create destination directories"); } + for src in &self.artifacts { let mut name = file_name(src); if self.test { - name = format!("test-{}", name.split('-').collect::>()[0]); + name = format!( + "test-{}", + name.split('-').next().expect("Couldn't get test name") + ); } + fs::copy( - root.join(src.to_str().unwrap()), + root.join(src.to_str().expect("File name is not valid UTF-8")), &dest.join(&name.replace('_', "-")), )?; - println!("Write elf from\n {:?}\nto\n {:?}", src, dest); + + println!("Write elf from\n {src:?}\nto\n {dest:?}"); } + Ok(()) } #[cfg(feature = "risc0")] - pub fn risc0_placement(&self, dest: &str) -> Result<()> { + pub fn risc0_placement(&self, dest: &str) -> anyhow::Result<()> { use crate::risc0_util::GuestListEntry; - let root = ROOT_DIR.get().unwrap(); + use std::{fs, io::Write}; + + let root = crate::ROOT_DIR.get().expect("No reference to ROOT_DIR"); let dest_dir = PathBuf::from(dest); if !dest_dir.exists() { - fs::create_dir_all(&dest_dir).unwrap(); + fs::create_dir_all(&dest_dir).expect("Couldn't create destination directories"); } + for src in &self.artifacts { let mut name = file_name(src); + if self.test { - name = format!("test-{}", name.split('-').collect::>()[0]).to_string(); + name = format!( + "test-{}", + name.split('-').next().expect("Couldn't get test name") + ); } + let mut dest_file = - File::create(&dest_dir.join(&format!("{}.rs", name.replace('-', "_")))).unwrap(); - let guest = GuestListEntry::build(&name, root.join(src).to_str().unwrap()).unwrap(); + fs::File::create(&dest_dir.join(&format!("{}.rs", name.replace('-', "_")))) + .expect("Couldn't create destination file"); + + let guest = GuestListEntry::build( + &name, + root.join(src).to_str().expect("Path is not valid UTF-8"), + ) + .expect("Couldn't build the guest list entry"); + dest_file.write_all( guest - .codegen_consts(&fs::canonicalize(&dest_dir).unwrap()) + .codegen_consts( + &std::fs::canonicalize(&dest_dir) + .expect("Couldn't canonicalize the destination path"), + ) .as_bytes(), )?; - println!("Write from\n {:?}\nto\n {:?}", src, dest_file); + + println!("Write from\n {src:?}\nto\n {dest_file:?}"); } + Ok(()) } } fn file_name(path: &Path) -> String { - String::from(path.file_name().unwrap().to_str().unwrap()) + path.file_name() + .expect("no filename in path") + .to_str() + .expect("filename is non unicode") + .to_owned() } fn extract_path(line: &str) -> Option { - let re = Regex::new(r"\(([^)]+)\)").unwrap(); + let re = Regex::new(r"\(([^)]+)\)").expect("Couldn't create regex"); re.captures(line) .and_then(|caps| caps.get(1).map(|m| m.as_str().to_string())) .map(PathBuf::from) diff --git a/pipeline/src/lib.rs b/pipeline/src/lib.rs index fb17ae62a..3b76629f5 100644 --- a/pipeline/src/lib.rs +++ b/pipeline/src/lib.rs @@ -12,8 +12,11 @@ pub static ROOT_DIR: OnceCell = OnceCell::new(); pub trait Pipeline { fn new(root: &str, profile: &str) -> Self; + fn builder(&self) -> CommandBuilder; + fn bins(&self, bins: &[&str], dest: &str); + fn tests(&self, bins: &[&str], dest: &str); } @@ -21,14 +24,14 @@ pub fn rerun_if_changed(env_vars: &[&str]) { // Only work in build.rs // Tell cargo to rerun the script only if program/{src, Cargo.toml, Cargo.lock} changes // Ref: https://doc.rust-lang.org/nightly/cargo/reference/build-scripts.html#rerun-if-changed - let root = ROOT_DIR.get().unwrap(); - [ + let root = ROOT_DIR.get().expect("No reference to ROOT_DIR"); + for p in [ root.join("src"), root.join("Cargo.toml"), root.join("Cargo.lock"), - ] - .iter() - .for_each(|p| println!("cargo::rerun-if-changed={}", p.display())); + ] { + println!("cargo::rerun-if-changed={}", p.display()); + } for v in env_vars { println!("cargo::rerun-if-env-changed={}", v); } diff --git a/pipeline/src/risc0_util.rs b/pipeline/src/risc0_util.rs index 01b56c1af..15c6e9817 100644 --- a/pipeline/src/risc0_util.rs +++ b/pipeline/src/risc0_util.rs @@ -7,21 +7,6 @@ use std::{ pub const DIGEST_WORDS: usize = 8; -#[allow(dead_code)] -pub fn risc0_data() -> Result { - let dir = if let Ok(dir) = std::env::var("RISC0_DATA_DIR") { - dir.into() - } else if let Some(root) = dirs::data_dir() { - root.join("cargo-risczero") - } else if let Some(home) = dirs::home_dir() { - home.join(".cargo-risczero") - } else { - anyhow::bail!("Could not determine cargo-risczero data dir. Set RISC0_DATA_DIR env var."); - }; - - Ok(dir) -} - /// Represents an item in the generated list of compiled guest binaries #[derive(Debug, Clone)] pub struct GuestListEntry { @@ -56,11 +41,11 @@ impl GuestListEntry { panic!("method path cannot include #: {}", self.path); } let relative_path = pathdiff::diff_paths( - fs::canonicalize(Path::new(&self.path.as_ref())).unwrap(), + fs::canonicalize(Path::new(&self.path.as_ref())).expect("Couldn't canonicalize path"), dest, ) - .map(|p| String::from(p.to_str().unwrap())) - .unwrap(); + .map(|p| String::from(p.to_str().expect("Path is not valid UTF-8"))) + .expect("No relative path for destination"); let upper = self.name.to_uppercase().replace('-', "_"); let image_id: [u32; DIGEST_WORDS] = self.image_id;