feat: new include_elf! macro for importing ELF (#1620)

crates/build/src/build.rs

@@ -6,7 +6,7 @@ use cargo_metadata::camino::Utf8PathBuf;
 use crate::{
     command::{docker::create_docker_command, local::create_local_command, utils::execute_command},
     utils::{cargo_rerun_if_changed, copy_elf_to_output_dir, current_datetime},
-    BuildArgs,
+    BuildArgs, BUILD_TARGET, HELPER_TARGET_SUBDIR,
 };
 
 /// Build a program with the specified [`BuildArgs`]. The `program_dir` is specified as an argument
@@ -20,12 +20,12 @@ use crate::{
 ///
 /// # Returns
 ///
-/// * `Result<Utf8PathBuf>` - The path to the built program as a `Utf8PathBuf` on success, or an
-///   error on failure.
+/// * `Result<Vec<(String, Utf8PathBuf)>>` - A list of mapping from bin target names to the paths to
+///   the built program as a `Utf8PathBuf` on success, or an error on failure.
 pub fn execute_build_program(
     args: &BuildArgs,
     program_dir: Option<PathBuf>,
-) -> Result<Utf8PathBuf> {
+) -> Result<Vec<(String, Utf8PathBuf)>> {
     // If the program directory is not specified, use the current directory.
     let program_dir = program_dir
         .unwrap_or_else(|| std::env::current_dir().expect("Failed to get current directory."));
@@ -46,7 +46,15 @@ pub fn execute_build_program(
 
     execute_command(cmd, args.docker)?;
 
-    copy_elf_to_output_dir(args, &program_metadata)
+    let target_elf_paths = generate_elf_paths(&program_metadata, Some(args))?;
+
+    // Temporary backward compatibility with the deprecated behavior of copying the ELF file.
+    // TODO: add option to turn off this behavior
+    if target_elf_paths.len() == 1 {
+        copy_elf_to_output_dir(args, &program_metadata)?;
+    }
+
+    Ok(target_elf_paths)
 }
 
 /// Internal helper function to build the program with or without arguments.
@@ -64,6 +72,9 @@ pub(crate) fn build_program_internal(path: &str, args: Option<BuildArgs>) {
         .map(|v| v.eq_ignore_ascii_case("true"))
         .unwrap_or(false);
     if skip_program_build {
+        // Still need to set ELF env vars even if build is skipped.
+        generate_elf_paths(&metadata, args.as_ref()).expect("failed to collect target ELF paths");
+
         println!(
             "cargo:warning=Build skipped for {} at {} due to SP1_SKIP_PROGRAM_BUILD flag",
             root_package_name,
@@ -82,6 +93,9 @@ pub(crate) fn build_program_internal(path: &str, args: Option<BuildArgs>) {
         .map(|val| val.contains("clippy-driver"))
         .unwrap_or(false);
     if is_clippy_driver {
+        // Still need to set ELF env vars even if build is skipped.
+        generate_elf_paths(&metadata, args.as_ref()).expect("failed to collect target ELF paths");
+
         println!("cargo:warning=Skipping build due to clippy invocation.");
         return;
     }
@@ -98,3 +112,46 @@ pub(crate) fn build_program_internal(path: &str, args: Option<BuildArgs>) {
 
     println!("cargo:warning={} built at {}", root_package_name, current_datetime());
 }
+
+/// Collects the list of targets that would be built and their output ELF file paths. Also prints
+/// cargo directives setting relevant `SP1_ELF_` environment variables.
+fn generate_elf_paths(
+    metadata: &cargo_metadata::Metadata,
+    args: Option<&BuildArgs>,
+) -> Result<Vec<(String, Utf8PathBuf)>> {
+    let mut target_elf_paths = vec![];
+
+    for program_crate in metadata.workspace_default_members.iter() {
+        let program = metadata
+            .packages
+            .iter()
+            .find(|p| &p.id == program_crate)
+            .ok_or_else(|| anyhow::anyhow!("cannot find package for {}", program_crate))?;
+
+        for bin_target in program.targets.iter().filter(|t| {
+            t.kind.contains(&"bin".to_owned()) && t.crate_types.contains(&"bin".to_owned())
+        }) {
+            // Filter out irrelevant targets if `--bin` is used.
+            if let Some(args) = args {
+                if !args.binary.is_empty() && bin_target.name != args.binary {
+                    continue;
+                }
+            }
+
+            let elf_path = metadata.target_directory.join(HELPER_TARGET_SUBDIR);
+            let elf_path = match args {
+                Some(args) if args.docker => elf_path.join("docker"),
+                _ => elf_path,
+            };
+            let elf_path = elf_path.join(BUILD_TARGET).join("release").join(&bin_target.name);
+
+            target_elf_paths.push((bin_target.name.to_owned(), elf_path));
+        }
+    }
+
+    for (target_name, elf_path) in target_elf_paths.iter() {
+        println!("cargo:rustc-env=SP1_ELF_{}={}", target_name, elf_path);
+    }
+
+    Ok(target_elf_paths)
+}

crates/cli/src/commands/prove.rs

@@ -70,7 +70,7 @@ pub struct ProveCmd {
 
 impl ProveCmd {
     pub fn run(&self) -> Result<()> {
-        let elf_path = execute_build_program(&self.build_args, None)?;
+        let elf_paths = execute_build_program(&self.build_args, None)?;
 
         if !self.profile {
             match env::var("RUST_LOG") {
@@ -86,6 +86,14 @@ impl ProveCmd {
             setup_tracer();
         }
 
+        // The command predates multi-target build support. This allows the command to continue to
+        // work when only one package is built, preserving backward compatibility.
+        let elf_path = if elf_paths.len() == 1 {
+            elf_paths[0].1.to_owned()
+        } else {
+            anyhow::bail!("the prove command does not work with multi-target builds");
+        };
+
         let mut elf = Vec::new();
         File::open(elf_path.as_path().as_str())
             .expect("failed to open input file")

crates/sdk/src/lib.rs

@@ -297,6 +297,22 @@ pub fn block_on<T>(fut: impl Future<Output = T>) -> T {
     }
 }
 
+/// Returns the raw ELF bytes by the zkVM program target name.
+///
+/// Note that this only works when using `sp1_build::build_program` or
+/// `sp1_build::build_program_with_args` in a build script.
+///
+/// By default, the program target name is the same as the program crate name. However, this might
+/// not be the case for non-standard project structures. For example, placing the entrypoint source
+/// file at `src/bin/my_entry.rs` would result in the program target being named `my_entry`, in
+/// which case the invocation should be `include_elf!("my_entry")` instead.
+#[macro_export]
+macro_rules! include_elf {
+    ($arg:tt) => {{
+        include_bytes!(env!(concat!("SP1_ELF_", $arg)))
+    }};
+}
+
 #[cfg(test)]
 mod tests {
 

examples/aggregation/script/build.rs

@@ -1,12 +1,4 @@
-use sp1_build::{build_program_with_args, BuildArgs};
-
 fn main() {
-    build_program_with_args(
-        "../program",
-        BuildArgs { output_directory: "aggregation/program/elf".into(), ..Default::default() },
-    );
-    build_program_with_args(
-        "../../fibonacci/program",
-        BuildArgs { output_directory: "fibonacci/program/elf".into(), ..Default::default() },
-    );
+    sp1_build::build_program("../program");
+    sp1_build::build_program("../../fibonacci/program");
 }

examples/aggregation/script/src/main.rs

@@ -1,15 +1,15 @@
 //! A simple example showing how to aggregate proofs of multiple programs with SP1.
 
 use sp1_sdk::{
-    HashableKey, ProverClient, SP1Proof, SP1ProofWithPublicValues, SP1Stdin, SP1VerifyingKey,
+    include_elf, HashableKey, ProverClient, SP1Proof, SP1ProofWithPublicValues, SP1Stdin,
+    SP1VerifyingKey,
 };
 
 /// A program that aggregates the proofs of the simple program.
-const AGGREGATION_ELF: &[u8] = include_bytes!("../../program/elf/riscv32im-succinct-zkvm-elf");
+const AGGREGATION_ELF: &[u8] = include_elf!("aggregation-program");
 
 /// A program that just runs a simple computation.
-const FIBONACCI_ELF: &[u8] =
-    include_bytes!("../../../fibonacci/program/elf/riscv32im-succinct-zkvm-elf");
+const FIBONACCI_ELF: &[u8] = include_elf!("fibonacci-program");
 
 /// An input to the aggregation program.
 ///

examples/bls12381/script/build.rs

@@ -1,8 +1,3 @@
-use sp1_build::{build_program_with_args, BuildArgs};
-
 fn main() {
-    build_program_with_args(
-        "../program",
-        BuildArgs { output_directory: "bls12381/program/elf".into(), ..Default::default() },
-    );
+    sp1_build::build_program("../program");
 }

examples/bls12381/script/src/main.rs

@@ -1,5 +1,5 @@
-use sp1_sdk::{utils, ProverClient, SP1Stdin};
-pub const ELF: &[u8] = include_bytes!("../../program/elf/riscv32im-succinct-zkvm-elf");
+use sp1_sdk::{include_elf, utils, ProverClient, SP1Stdin};
+pub const ELF: &[u8] = include_elf!("bls12381-program");
 
 fn main() {
     utils::setup_logger();

examples/bn254/script/build.rs

@@ -1,8 +1,3 @@
-use sp1_build::{build_program_with_args, BuildArgs};
-
 fn main() {
-    build_program_with_args(
-        "../program",
-        BuildArgs { output_directory: "bn254/program/elf".into(), ..Default::default() },
-    );
+    sp1_build::build_program("../program");
 }