diff --git a/rs/cross-chain/proposal-cli/src/candid/tests.rs b/rs/cross-chain/proposal-cli/src/candid/tests.rs index 34ccd308921..4ed6d09aad6 100644 --- a/rs/cross-chain/proposal-cli/src/candid/tests.rs +++ b/rs/cross-chain/proposal-cli/src/candid/tests.rs @@ -38,6 +38,8 @@ fn should_parse_constructor_parameters() { if canister == TargetCanister::IcpArchive1 || canister == TargetCanister::IcpArchive2 || canister == TargetCanister::IcpArchive3 + //canister lives outside the monorepo + || canister == TargetCanister::EvmRpc { continue; } diff --git a/rs/cross-chain/proposal-cli/src/canister/mod.rs b/rs/cross-chain/proposal-cli/src/canister/mod.rs index f3716c4a0b6..a14d0416f65 100644 --- a/rs/cross-chain/proposal-cli/src/canister/mod.rs +++ b/rs/cross-chain/proposal-cli/src/canister/mod.rs @@ -3,10 +3,11 @@ mod tests; use std::fmt::Display; use std::path::PathBuf; +use std::process::Command; use std::str::FromStr; use strum_macros::EnumIter; -#[derive(Clone, Eq, PartialEq, Debug, EnumIter)] +#[derive(Clone, Eq, PartialEq, Debug, Ord, PartialOrd, EnumIter)] #[allow(clippy::enum_variant_names)] pub enum TargetCanister { CkBtcArchive, @@ -24,6 +25,7 @@ pub enum TargetCanister { IcpIndex, IcpLedger, LedgerSuiteOrchestrator, + EvmRpc, } impl TargetCanister { @@ -40,6 +42,30 @@ impl TargetCanister { TargetCanister::IcpIndex => "icp-index", TargetCanister::IcpLedger => "icp-ledger", TargetCanister::LedgerSuiteOrchestrator => "orchestrator", + TargetCanister::EvmRpc => "evm_rpc", + } + } + + pub fn git_repository_url(&self) -> &str { + match &self { + TargetCanister::CkBtcArchive + | TargetCanister::CkBtcIndex + | TargetCanister::CkBtcKyt + | TargetCanister::CkBtcLedger + | TargetCanister::CkBtcMinter + | TargetCanister::CkEthArchive + | TargetCanister::CkEthIndex + | TargetCanister::CkEthLedger + | TargetCanister::CkEthMinter + | TargetCanister::IcpArchive1 + | TargetCanister::IcpArchive2 + | TargetCanister::IcpArchive3 + | TargetCanister::IcpIndex + | TargetCanister::IcpLedger + | TargetCanister::LedgerSuiteOrchestrator => "https://github.com/dfinity/ic.git", + TargetCanister::EvmRpc => { + "https://github.com/internet-computer-protocol/evm-rpc-canister.git" + } } } @@ -71,11 +97,31 @@ impl TargetCanister { TargetCanister::LedgerSuiteOrchestrator => { PathBuf::from("rs/ethereum/ledger-suite-orchestrator/ledger_suite_orchestrator.did") } + TargetCanister::EvmRpc => PathBuf::from("candid/evm_rpc.did"), } } - pub fn repo_dir(&self) -> PathBuf { - self.candid_file().parent().unwrap().to_path_buf() + pub fn repo_dir(&self) -> Option { + match &self { + TargetCanister::CkBtcArchive + | TargetCanister::CkBtcIndex + | TargetCanister::CkBtcKyt + | TargetCanister::CkBtcLedger + | TargetCanister::CkBtcMinter + | TargetCanister::CkEthArchive + | TargetCanister::CkEthIndex + | TargetCanister::CkEthLedger + | TargetCanister::CkEthMinter + | TargetCanister::IcpArchive1 + | TargetCanister::IcpArchive2 + | TargetCanister::IcpArchive3 + | TargetCanister::IcpIndex + | TargetCanister::IcpLedger + | TargetCanister::LedgerSuiteOrchestrator => { + Some(self.candid_file().parent().unwrap().to_path_buf()) + } + TargetCanister::EvmRpc => None, + } } pub fn git_log_dirs(&self) -> Vec { @@ -111,14 +157,41 @@ impl TargetCanister { PathBuf::from("rs/ledger_suite/common/ledger_core/src"), ] } - _ => { - vec![self.repo_dir()] - } + TargetCanister::CkBtcArchive + | TargetCanister::CkBtcIndex + | TargetCanister::CkBtcKyt + | TargetCanister::CkBtcLedger + | TargetCanister::CkBtcMinter + | TargetCanister::CkEthArchive + | TargetCanister::CkEthIndex + | TargetCanister::CkEthLedger + | TargetCanister::CkEthMinter + | TargetCanister::LedgerSuiteOrchestrator + | TargetCanister::EvmRpc => self.repo_dir().into_iter().collect(), } } pub fn artifact(&self) -> PathBuf { - PathBuf::from("artifacts/canisters").join(self.artifact_file_name()) + match &self { + TargetCanister::CkBtcArchive + | TargetCanister::CkBtcIndex + | TargetCanister::CkBtcKyt + | TargetCanister::CkBtcLedger + | TargetCanister::CkBtcMinter + | TargetCanister::CkEthArchive + | TargetCanister::CkEthIndex + | TargetCanister::CkEthLedger + | TargetCanister::CkEthMinter + | TargetCanister::IcpArchive1 + | TargetCanister::IcpArchive2 + | TargetCanister::IcpArchive3 + | TargetCanister::IcpIndex + | TargetCanister::IcpLedger + | TargetCanister::LedgerSuiteOrchestrator => { + PathBuf::from("artifacts/canisters").join(self.artifact_file_name()) + } + TargetCanister::EvmRpc => PathBuf::from(self.artifact_file_name()), + } } pub fn artifact_file_name(&self) -> &str { @@ -140,9 +213,39 @@ impl TargetCanister { TargetCanister::LedgerSuiteOrchestrator => { "ic-ledger-suite-orchestrator-canister.wasm.gz" } + TargetCanister::EvmRpc => "evm_rpc.wasm.gz", + } + } + + pub fn build_artifact(&self) -> Command { + match &self { + TargetCanister::CkBtcArchive + | TargetCanister::CkBtcIndex + | TargetCanister::CkBtcKyt + | TargetCanister::CkBtcLedger + | TargetCanister::CkBtcMinter + | TargetCanister::CkEthArchive + | TargetCanister::CkEthIndex + | TargetCanister::CkEthLedger + | TargetCanister::CkEthMinter + | TargetCanister::IcpArchive1 + | TargetCanister::IcpArchive2 + | TargetCanister::IcpArchive3 + | TargetCanister::IcpIndex + | TargetCanister::IcpLedger + | TargetCanister::LedgerSuiteOrchestrator => { + let mut cmd = Command::new("./ci/container/build-ic.sh"); + cmd.arg("--canisters"); + cmd + } + TargetCanister::EvmRpc => Command::new("./scripts/docker-build"), } } + pub fn build_artifact_as_str(&self) -> String { + format!("{:?}", self.build_artifact()) + } + pub fn canister_ids_json_file(&self) -> PathBuf { match self { TargetCanister::CkBtcArchive @@ -164,6 +267,7 @@ impl TargetCanister { | TargetCanister::IcpArchive3 | TargetCanister::IcpIndex | TargetCanister::IcpLedger => PathBuf::from("rs/ledger_suite/icp/canister_ids.json"), + TargetCanister::EvmRpc => PathBuf::from("canister_ids.json"), } } @@ -195,6 +299,7 @@ impl FromStr for TargetCanister { ["icp", "archive3"] => Ok(TargetCanister::IcpArchive3), ["icp", "index"] => Ok(TargetCanister::IcpIndex), ["icp", "ledger"] => Ok(TargetCanister::IcpLedger), + ["evm", "rpc"] => Ok(TargetCanister::EvmRpc), _ => Err(format!("Unknown canister name: {}", canister)), } } @@ -218,6 +323,7 @@ impl Display for TargetCanister { TargetCanister::IcpIndex => write!(f, "ICP index"), TargetCanister::IcpLedger => write!(f, "ICP ledger"), TargetCanister::LedgerSuiteOrchestrator => write!(f, "ledger suite orchestrator"), + TargetCanister::EvmRpc => write!(f, "EVM RPC"), } } } diff --git a/rs/cross-chain/proposal-cli/src/git/mod.rs b/rs/cross-chain/proposal-cli/src/git/mod.rs index 36f4c40fe05..e2c56e62b4a 100644 --- a/rs/cross-chain/proposal-cli/src/git/mod.rs +++ b/rs/cross-chain/proposal-cli/src/git/mod.rs @@ -57,17 +57,17 @@ pub struct GitRepository { } impl GitRepository { - pub fn clone_ic() -> Self { + pub fn clone(url: &str) -> Self { let repo = TempDir::new().expect("failed to create a temporary directory"); // Blobless clone // see https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ let git_clone = Command::new("git") .arg("clone") .arg("--filter=blob:none") - .arg("https://github.com/dfinity/ic.git") + .arg(url) .arg(repo.path()) .status() - .expect("failed to clone the IC repository"); + .expect("failed to clone the repository"); assert!(git_clone.success()); GitRepository { dir: repo } @@ -159,7 +159,7 @@ impl GitRepository { git_log.arg(repo_dir); } let log = git_log.output().expect("failed to run git log"); - assert!(log.status.success()); + assert!(log.status.success(), "failed to run git log: {:?}", log); let executed_command = iter::once(git_log.get_program()) .chain(git_log.get_args()) @@ -183,16 +183,21 @@ impl GitRepository { &mut self, canister: &[TargetCanister], ) -> Vec { - self.build_canisters(); - canister.iter().map(|c| self.sha256_artifact(c)).collect() + canister + .iter() + .map(|c| { + self.build_canister_artifact(c); + self.sha256_artifact(c) + }) + .collect() } - fn build_canisters(&mut self) { - let build = Command::new("./ci/container/build-ic.sh") - .arg("--canisters") + fn build_canister_artifact(&mut self, canister: &TargetCanister) { + let build = canister + .build_artifact() .current_dir(self.dir.path()) .status() - .expect("failed to build canister artifacts"); + .expect("failed to build canister artifact"); assert!(build.success()); } diff --git a/rs/cross-chain/proposal-cli/src/main.rs b/rs/cross-chain/proposal-cli/src/main.rs index efe7b95f483..a980cb54121 100644 --- a/rs/cross-chain/proposal-cli/src/main.rs +++ b/rs/cross-chain/proposal-cli/src/main.rs @@ -12,6 +12,7 @@ use crate::ic_admin::ProposalFiles; use crate::proposal::{InstallProposalTemplate, ProposalTemplate, UpgradeProposalTemplate}; use clap::{Parser, Subcommand}; use ic_admin::IcAdminArgs; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; @@ -106,35 +107,44 @@ async fn main() { submit, } => { check_dir_has_required_permissions(&output_dir).expect("invalid output directory"); - - let mut ic_repo = GitRepository::clone_ic(); - let dashboard = DashboardClient::new(); - let release_notes = ic_repo.release_notes_batch(&canisters, &from, &to); - ic_repo.checkout(&to); - let upgrade_args: Vec<_> = ic_repo.encode_args_batch(&canisters, args); - let canister_ids = ic_repo.parse_canister_id_batch(&canisters); - let last_upgrade_proposal_ids: Vec<_> = dashboard - .list_canister_upgrade_proposals_batch(&canister_ids) - .await - .into_iter() - .map(|set| set.last().cloned()) - .collect(); - let compressed_wasm_hashes = ic_repo.build_canister_artifact_batch(&canisters); - - for (index, canister) in canisters.into_iter().enumerate() { - let output_dir = output_dir.join(canister.to_string()).join(to.to_string()); - - let proposal = UpgradeProposalTemplate { - canister: canister.clone(), - to: to.clone(), - compressed_wasm_hash: compressed_wasm_hashes[index].clone(), - canister_id: canister_ids[index], - last_upgrade_proposal_id: last_upgrade_proposal_ids[index], - upgrade_args: upgrade_args[index].clone(), - release_notes: release_notes[index].clone(), - }; - - write_to_disk(output_dir, proposal, submit.clone(), &ic_repo); + let canister_per_git_repo = canisters_per_git_repo(canisters); + for git_repo_url in canister_per_git_repo.keys() { + let canisters: Vec<_> = canister_per_git_repo + .get(git_repo_url) + .unwrap() + .iter() + .cloned() + .collect(); + let mut git_repo = GitRepository::clone(git_repo_url); + let dashboard = DashboardClient::new(); + let release_notes = git_repo.release_notes_batch(&canisters, &from, &to); + git_repo.checkout(&to); + let upgrade_args: Vec<_> = git_repo.encode_args_batch(&canisters, args.clone()); + let canister_ids = git_repo.parse_canister_id_batch(&canisters); + let last_upgrade_proposal_ids: Vec<_> = dashboard + .list_canister_upgrade_proposals_batch(&canister_ids) + .await + .into_iter() + .map(|set| set.last().cloned()) + .collect(); + let compressed_wasm_hashes = git_repo.build_canister_artifact_batch(&canisters); + + for (index, canister) in canisters.into_iter().enumerate() { + let output_dir = output_dir.join(canister.to_string()).join(to.to_string()); + + let proposal = UpgradeProposalTemplate { + canister: canister.clone(), + to: to.clone(), + compressed_wasm_hash: compressed_wasm_hashes[index].clone(), + canister_id: canister_ids[index], + last_upgrade_proposal_id: last_upgrade_proposal_ids[index], + upgrade_args: upgrade_args[index].clone(), + release_notes: release_notes[index].clone(), + build_artifact_command: canister.build_artifact_as_str(), + }; + + write_to_disk(output_dir, proposal, submit.clone(), &git_repo); + } } } Commands::Install { @@ -144,25 +154,36 @@ async fn main() { output_dir, submit, } => { - let mut ic_repo = GitRepository::clone_ic(); - - ic_repo.checkout(&at); - let install_args: Vec<_> = ic_repo.encode_args_batch(&canisters, args); - let canister_ids = ic_repo.parse_canister_id_batch(&canisters); - let compressed_wasm_hashes = ic_repo.build_canister_artifact_batch(&canisters); - - for (index, canister) in canisters.into_iter().enumerate() { - let output_dir = output_dir.join(canister.to_string()).join(at.to_string()); - - let proposal = InstallProposalTemplate { - canister, - at: at.clone(), - compressed_wasm_hash: compressed_wasm_hashes[index].clone(), - canister_id: canister_ids[index], - install_args: install_args[index].clone(), - }; - - write_to_disk(output_dir, proposal, submit.clone(), &ic_repo); + let canister_per_git_repo = canisters_per_git_repo(canisters); + + for git_repo_url in canister_per_git_repo.keys() { + let canisters: Vec<_> = canister_per_git_repo + .get(git_repo_url) + .unwrap() + .iter() + .cloned() + .collect(); + + let mut git_repo = GitRepository::clone(git_repo_url); + git_repo.checkout(&at); + let install_args: Vec<_> = git_repo.encode_args_batch(&canisters, args.clone()); + let canister_ids = git_repo.parse_canister_id_batch(&canisters); + let compressed_wasm_hashes = git_repo.build_canister_artifact_batch(&canisters); + + for (index, canister) in canisters.into_iter().enumerate() { + let output_dir = output_dir.join(canister.to_string()).join(at.to_string()); + + let proposal = InstallProposalTemplate { + canister: canister.clone(), + at: at.clone(), + compressed_wasm_hash: compressed_wasm_hashes[index].clone(), + canister_id: canister_ids[index], + install_args: install_args[index].clone(), + build_artifact_command: canister.build_artifact_as_str(), + }; + + write_to_disk(output_dir, proposal, submit.clone(), &git_repo); + } } } } @@ -283,3 +304,15 @@ fn check_dir_has_required_permissions(output_dir: &Path) -> Result<(), String> { } Ok(()) } + +fn canisters_per_git_repo( + canisters: Vec, +) -> BTreeMap> { + canisters + .into_iter() + .fold(BTreeMap::new(), |mut acc, canister| { + let git_repo = canister.git_repository_url().to_string(); + acc.entry(git_repo).or_default().insert(canister); + acc + }) +} diff --git a/rs/cross-chain/proposal-cli/src/proposal/mod.rs b/rs/cross-chain/proposal-cli/src/proposal/mod.rs index 34e95a33f37..b42eff55d00 100644 --- a/rs/cross-chain/proposal-cli/src/proposal/mod.rs +++ b/rs/cross-chain/proposal-cli/src/proposal/mod.rs @@ -15,6 +15,7 @@ pub struct UpgradeProposalTemplate { pub last_upgrade_proposal_id: Option, pub upgrade_args: UpgradeArgs, pub release_notes: ReleaseNotes, + pub build_artifact_command: String, } impl UpgradeProposalTemplate { @@ -33,6 +34,7 @@ pub struct InstallProposalTemplate { pub compressed_wasm_hash: CompressedWasmHash, pub canister_id: Principal, pub install_args: UpgradeArgs, + pub build_artifact_command: String, } pub enum ProposalTemplate { diff --git a/rs/cross-chain/proposal-cli/templates/install.md b/rs/cross-chain/proposal-cli/templates/install.md index 80e51aba86e..7aa540b2a74 100644 --- a/rs/cross-chain/proposal-cli/templates/install.md +++ b/rs/cross-chain/proposal-cli/templates/install.md @@ -1,5 +1,7 @@ # Proposal to install the {{canister}} canister +Repository: `{{canister.git_repository_url()}}` + Git hash: `{{at}}` New compressed Wasm hash: `{{compressed_wasm_hash}}` @@ -17,7 +19,9 @@ TODO: THIS MUST BE FILLED OUT ``` git fetch git checkout {{at}} -cd {{canister.repo_dir().as_path().display()}} +{% if let Some(dir) = canister.repo_dir() -%} +cd {{dir.as_path().display()}} +{% endif -%} {{install_args.didc_encode_cmd()}} | xxd -r -p | sha256sum ``` @@ -28,6 +32,6 @@ Verify that the hash of the gzipped WASM matches the proposed hash. ``` git fetch git checkout {{at}} -./ci/container/build-ic.sh -c +{{build_artifact_command}} sha256sum ./{{canister.artifact().as_path().display()}} ``` diff --git a/rs/cross-chain/proposal-cli/templates/upgrade.md b/rs/cross-chain/proposal-cli/templates/upgrade.md index 635e680a271..8487a10dda6 100644 --- a/rs/cross-chain/proposal-cli/templates/upgrade.md +++ b/rs/cross-chain/proposal-cli/templates/upgrade.md @@ -1,5 +1,7 @@ # Proposal to upgrade the {{canister}} canister +Repository: `{{canister.git_repository_url()}}` + Git hash: `{{to}}` New compressed Wasm hash: `{{compressed_wasm_hash}}` @@ -19,7 +21,9 @@ TODO: THIS MUST BE FILLED OUT ``` git fetch git checkout {{to}} -cd {{canister.repo_dir().as_path().display()}} +{% if let Some(dir) = canister.repo_dir() -%} +cd {{dir.as_path().display()}} +{% endif -%} {{upgrade_args.didc_encode_cmd()}} | xxd -r -p | sha256sum ``` @@ -36,6 +40,6 @@ Verify that the hash of the gzipped WASM matches the proposed hash. ``` git fetch git checkout {{to}} -./ci/container/build-ic.sh -c +{{build_artifact_command}} sha256sum ./{{canister.artifact().as_path().display()}} ```