diff --git a/Cargo.lock b/Cargo.lock index e54f2f68..e8b2c917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,7 @@ dependencies = [ "moss", "nix 0.27.1", "regex", + "reqwest", "serde", "serde_json", "serde_yaml", @@ -1131,6 +1132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -2292,6 +2294,7 @@ dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", diff --git a/Cargo.toml b/Cargo.toml index b9e2e245..4c1eb371 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,6 @@ [workspace] -members = [ - "boulder", - "moss", - "crates/*", -] -default-members = [ - "moss" -] +members = ["boulder", "moss", "crates/*"] +default-members = ["moss"] resolver = "2" [workspace.package] @@ -23,9 +17,17 @@ clap_complete = "4.5.37" clap_mangen = "0.2.24" criterion = { version = "0.5.1", features = ["html_reports"] } crossterm = "0.28.1" -derive_more = { version = "1.0.0", features = ["as_ref", "display", "from", "into"] } +derive_more = { version = "1.0.0", features = [ + "as_ref", + "display", + "from", + "into", +] } dialoguer = "0.11.0" -diesel = { version = "2.2.1", features = ["sqlite", "returning_clauses_for_sqlite_3_35"] } +diesel = { version = "2.2.1", features = [ + "sqlite", + "returning_clauses_for_sqlite_3_35", +] } diesel_migrations = "2.2.0" dirs = "5.0.1" elf = "0.7.4" @@ -39,11 +41,31 @@ indextree = "4.6.1" libsqlite3-sys = { version = "0.30.1", features = ["bundled"] } log = "0.4.22" nom = "7.1.3" -nix = { version = "0.27.1", features = ["user", "fs", "sched", "process", "mount", "hostname", "signal", "term"] } +nix = { version = "0.27.1", features = [ + "user", + "fs", + "sched", + "process", + "mount", + "hostname", + "signal", + "term", +] } petgraph = "0.6.5" rayon = "1.10.0" regex = "1.10.5" -reqwest = { version = "0.12.5", default-features = false, features = ["brotli", "charset", "deflate", "gzip", "http2", "rustls-tls", "stream", "zstd"] } +reqwest = { version = "0.12.5", default-features = false, features = [ + "brotli", + "charset", + "deflate", + "gzip", + "http2", + "rustls-tls", + "stream", + "zstd", + "blocking", + "json", +] } serde = { version = "1.0.204", features = ["derive"] } serde_json = "1.0.120" serde_yaml = "0.9.34" diff --git a/boulder/Cargo.toml b/boulder/Cargo.toml index c203cda1..01089df9 100644 --- a/boulder/Cargo.toml +++ b/boulder/Cargo.toml @@ -38,6 +38,7 @@ hex.workspace = true itertools.workspace = true nix.workspace = true regex.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true diff --git a/boulder/src/cli/recipe.rs b/boulder/src/cli/recipe.rs index 33694758..0e1f5d4a 100644 --- a/boulder/src/cli/recipe.rs +++ b/boulder/src/cli/recipe.rs @@ -54,12 +54,7 @@ pub enum Subcommand { }, #[command(about = "Create skeletal stone.yaml recipe from source archive URIs")] New { - #[arg( - short, - long, - default_value = "./stone.yaml", - help = "Location to output generated build recipe" - )] + #[arg(short, long, default_value = ".", help = "Location to output generated files")] output: PathBuf, #[arg(required = true, value_name = "URI", help = "Source archive URIs")] upstreams: Vec, @@ -154,12 +149,20 @@ fn new(output: PathBuf, upstreams: Vec) -> Result<(), Error> { // We use async to fetch upstreams let _guard = runtime::init(); + const RECIPE_FILE: &str = "stone.yaml"; + const MONITORING_FILE: &str = "monitoring.yaml"; + let drafter = Drafter::new(upstreams); - let recipe = drafter.run()?; + let draft = drafter.run()?; + + if !output.is_dir() { + fs::create_dir_all(&output).map_err(Error::CreateDir)?; + } - fs::write(&output, recipe).map_err(Error::Write)?; + fs::write(PathBuf::from(&output).join(RECIPE_FILE), draft.stone).map_err(Error::Write)?; + fs::write(PathBuf::from(&output).join(MONITORING_FILE), draft.monitoring).map_err(Error::Write)?; - println!("Saved recipe to {output:?}"); + println!("Saved {RECIPE_FILE} & {MONITORING_FILE} to {output:?}"); Ok(()) } @@ -418,6 +421,8 @@ pub enum Error { Read(#[source] io::Error), #[error("writing recipe")] Write(#[source] io::Error), + #[error("creating output directory")] + CreateDir(#[source] io::Error), #[error("deserializing recipe")] Deser(#[from] serde_yaml::Error), #[error("fetch upstream")] diff --git a/boulder/src/draft.rs b/boulder/src/draft.rs index 5172b183..1d7cb65b 100644 --- a/boulder/src/draft.rs +++ b/boulder/src/draft.rs @@ -15,22 +15,29 @@ use url::Url; use crate::util; use self::metadata::Metadata; +use self::monitoring::Monitoring; use self::upstream::Upstream; mod build; mod metadata; +mod monitoring; mod upstream; pub struct Drafter { upstreams: Vec, } +pub struct Draft { + pub stone: String, + pub monitoring: String, +} + impl Drafter { pub fn new(upstreams: Vec) -> Self { Self { upstreams } } - pub fn run(&self) -> Result { + pub fn run(&self) -> Result { // TODO: Use tempdir let extract_root = PathBuf::from("/tmp/boulder-new"); @@ -40,6 +47,9 @@ impl Drafter { // Build metadata from extracted upstreams let metadata = Metadata::new(extracted); + let monitoring = Monitoring::new(&metadata.source.name, &metadata.source.homepage); + let monitoring_result = monitoring.run()?; + // Enumerate all extracted files let files = util::enumerate_files(&extract_root, |_| true)? .into_iter() @@ -96,7 +106,10 @@ license : UPDATE LICENSE metadata.upstreams(), ); - Ok(template) + Ok(Draft { + stone: template, + monitoring: monitoring_result, + }) } } @@ -135,6 +148,8 @@ pub enum Error { AnalyzeBuildSystem(#[source] build::Error), #[error("upstream")] Upstream(#[from] upstream::Error), + #[error("monitoring")] + Monitoring(#[from] monitoring::Error), #[error("io")] Io(#[from] io::Error), } diff --git a/boulder/src/draft/monitoring.rs b/boulder/src/draft/monitoring.rs new file mode 100644 index 00000000..781301d4 --- /dev/null +++ b/boulder/src/draft/monitoring.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright © 2020-2024 Serpent OS Developers +// +// SPDX-License-Identifier: MPL-2.0 + +use std::io; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use thiserror::Error; +use tui::Styled; + +#[derive(Serialize)] +struct MonitoringTemplate { + releases: Releases, + security: Security, +} + +#[derive(Serialize)] +struct Releases { + id: Option, + rss: Option, +} + +#[derive(Serialize)] +struct Security { + cpe: Vec, +} + +#[derive(Serialize)] +struct Cpe { + vendor: Option, + product: Option, +} + +#[derive(Debug, Deserialize)] +struct Item { + id: u32, + name: String, +} + +#[derive(Debug, Deserialize)] +struct Response { + items: Vec, + total_items: u32, +} + +#[derive(Serialize)] +pub struct Monitoring<'a> { + name: &'a String, + homepage: &'a String, +} + +impl<'a> Monitoring<'a> { + pub fn new(name: &'a String, homepage: &'a String) -> Self { + Self { name, homepage } + } + + pub fn run(&self) -> Result { + let client = reqwest::blocking::Client::new(); + + let id = self.find_monitoring_id(self.name, &client)?; + let cpes = self.find_security_cpe(self.name, &client)?; + let rss = self.guess_rss(self.homepage); + + let output = self.format_monitoring(id, cpes, rss)?; + + Ok(output) + } + + fn find_monitoring_id(&self, name: &String, client: &reqwest::blocking::Client) -> Result, Error> { + let url = format!("https://release-monitoring.org/api/v2/projects/?name={name}"); + + let resp = client.get(&url).send()?; + + match resp.error_for_status_ref() { + Ok(_res) => (), + Err(err) => return Err(Error::StatusCode(err)), + } + + let body: Response = resp.json()?; + + if body.total_items == 1 { + if let Some(result) = body.items.first() { + println!( + "{} | Matched id {} from {}", + "Monitoring".green(), + result.id, + result.name + ); + Ok(Some(result.id)) + } else { + Ok(None) + } + } else if body.total_items > 1 && body.total_items < 10 { + println!("{} | Multiple potential IDs matched, find the correct ID for the project at https://release-monitoring.org/", "Warning".yellow()); + for i in body.items { + println!( + "ID {} Name {} URL https://release-monitoring.org/project/{}/", + i.id, i.name, i.id + ); + } + println!(); + Ok(None) + } else { + println!( + "{} | Find the correct ID for the project at https://release-monitoring.org/", + "Warning".yellow() + ); + Ok(None) + } + } + + fn find_security_cpe(&self, name: &String, client: &reqwest::blocking::Client) -> Result, Error> { + const URL: &str = "https://cpe-guesser.cve-search.org/search"; + + let mut query = HashMap::new(); + query.insert("query", [name]); + + let resp = client.post(URL).json(&query).send()?; + + match resp.error_for_status_ref() { + Ok(_res) => (), + Err(err) => return Err(Error::StatusCode(err)), + } + + let json: Vec> = serde_json::from_str(&resp.text()?).unwrap_or_default(); + + // Extract CPEs into a Vec + let cpes: Vec = json + .iter() + .map(|item| { + if let Some(Value::String(cpe_string)) = item.get(1) { + // Split the CPE string and extract the desired parts + let parts: Vec<&str> = cpe_string.split(':').collect(); + if parts.len() > 4 { + let vendor = parts[3].to_owned(); + let product = parts[4].to_owned(); + println!( + "{} | Matched CPE Vendor: {vendor} Product: {product}", + "Security".green() + ); + return Cpe { + vendor: Some(vendor), + product: Some(product), + }; + } + } + Cpe { + vendor: None, + product: None, + } + }) + .collect(); + println!(); + + if cpes.len() > 1 { + println!( + "{} | Multiple CPEs matched, please verify and remove any superfluous", + "Warning".yellow() + ); + } + + Ok(cpes) + } + + fn guess_rss(&self, homepage: &String) -> Option { + match homepage { + _ if homepage.starts_with("https://github.com") => Some(format!("{homepage}/releases.atom")), + _ => None, + } + } + + fn format_monitoring(&self, id: Option, cpes: Vec, rss: Option) -> Result { + let monitoring_template = MonitoringTemplate { + releases: Releases { + id: Some(id.unwrap_or_default()), + rss: Some(rss.unwrap_or_default()), + }, + security: Security { cpe: cpes }, + }; + + let mut yaml_string = serde_yaml::to_string(&monitoring_template).expect("Failed to serialize to YAML"); + + // We may not have matched any ID or CPE which is fine + // Unwrap the default value then mangle it into a YAML ~ (null) value + if monitoring_template.releases.id.unwrap_or_default() == 0 { + let id_string = "id: 0"; + let id_marker = yaml_string.find(id_string).expect("releases id marker not found"); + yaml_string = yaml_string.replace(id_string, "id: ~"); + const ID_HELP_TEXT: &str = + " # https://release-monitoring.org/ and use the numeric id in the url of project"; + yaml_string.insert_str(id_marker + id_string.len(), ID_HELP_TEXT); + } + + if monitoring_template.releases.rss.unwrap_or_default().is_empty() { + yaml_string = yaml_string.replace("rss: ''", "rss: ~"); + } + + if monitoring_template.security.cpe.is_empty() { + let cpe_string = "cpe: []"; + let cpe_marker = yaml_string.find(cpe_string).expect("security cpe marker not found"); + yaml_string = yaml_string.replace(cpe_string, "cpe: ~"); + let cpe_help_text = format!( + " # Last checked {}", + chrono::Local::now().date_naive().format("%Y-%m-%d") + ); + yaml_string.insert_str(cpe_marker + cpe_string.len() - 1, &cpe_help_text); + } + + Ok(yaml_string) + } +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("io")] + Io(#[from] io::Error), + #[error("statuscode")] + StatusCode(#[from] reqwest::Error), +}