WIP build artifacts copy, getting empty dl tar

Signed-off-by: Robert Detjens <[email protected]>

Cargo.toml

@@ -16,7 +16,7 @@ tera = "1.19.1"
 simplelog = { version = "0.12.2", features = ["paris"] }
 fully_pub = "0.1.4"
 void = "1"
-futures-util = "0.3.30"
+futures = "0.3.30"
 figment = { version = "0.10.19", features = ["env", "yaml", "test"] }
 
 # kubernetes:

src/access_handlers/docker.rs

@@ -4,7 +4,7 @@ use bollard::{
     image::{CreateImageOptions, PushImageOptions, TagImageOptions},
     Docker,
 };
-use futures_util::{StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use itertools::Itertools;
 use simplelog::*;
 use tokio;

src/builder/artifacts.rs

@@ -0,0 +1,78 @@
+use anyhow::{anyhow, Context, Error, Result};
+use futures::future::try_join_all;
+use itertools::Itertools;
+use simplelog::{debug, trace};
+use std::path::PathBuf;
+
+use crate::builder::docker::{client, copy_file, create_container};
+use crate::configparser::challenge::ProvideConfig;
+
+use super::docker;
+
+/// extract assets from given container name and provide config to challenge directory, return file path(s) extracted
+#[tokio::main(flavor = "current_thread")] // make this a sync function
+pub async fn extract_asset(provide: &ProvideConfig, container: &str) -> Result<Vec<String>> {
+    debug!("extracting assets from container {}", container);
+    // This needs to handle three cases:
+    // - single or multiple files without renaming (no as: field)
+    // - single file with rename (one item with as:)
+    // - multiple files as archive (multiple items with as:)
+
+    // TODO: since this puts artifacts in the repo source folder, this should
+    // try to not overwrite any existing files.
+
+    match &provide.as_file {
+        // no renaming, copy out all as-is
+        None => extract_files(container, &provide.include).await,
+        // (as is keyword, so add underscore)
+        Some(as_) => {
+            if provide.include.len() == 1 {
+                // single file, rename
+                extract_rename(container, &provide.include[0], as_).await
+            } else {
+                // multiple files, zip as archive
+                extract_archive(container, &provide.include, as_).await
+            }
+        }
+    }
+}
+
+/// Extract multiple files from container
+async fn extract_files(container: &str, files: &Vec<String>) -> Result<Vec<String>> {
+    trace!(
+        "extracting {} files without renaming: {:?}",
+        files.len(),
+        files
+    );
+
+    try_join_all(
+        files
+            .iter()
+            .enumerate() // need index to avoid copy collisions
+            .map(|(i, f)| docker::copy_file(container, f, None)),
+    )
+    .await
+
+    // files
+    //     .iter()
+    //     .map(|f| docker::copy_file(container, f, None))
+    //     .collect::<Result<Vec<_>>>()
+}
+
+/// Extract one file from container and rename
+async fn extract_rename(container: &str, file: &str, new_name: &str) -> Result<Vec<String>> {
+    trace!("extracting file and renaming it");
+
+    Ok(vec!["todo rename".to_string()])
+}
+
+/// Extract one or more file from container as archive
+async fn extract_archive(
+    container: &str,
+    files: &Vec<String>,
+    archive_name: &str,
+) -> Result<Vec<String>> {
+    trace!("extracting mutliple files into archive");
+
+    Ok(vec!["todo archive".to_string()])
+}

src/builder/docker.rs

@@ -1,14 +1,21 @@
 use anyhow::{anyhow, bail, Context, Error, Result};
 use bollard::auth::DockerCredentials;
+use bollard::container::{
+    Config, CreateContainerOptions, DownloadFromContainerOptions, RemoveContainerOptions,
+};
 use bollard::errors::Error as DockerError;
 use bollard::image::{BuildImageOptions, PushImageOptions};
 use bollard::Docker;
 use core::fmt;
-use futures_util::{StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use simplelog::*;
+use std::fs::File;
+use std::io::{Seek, Write};
+use std::sync::LazyLock;
+use std::{fs, io};
 use std::{io::Read, path::Path};
 use tar;
-use tempfile::tempfile;
+use tempfile::{tempdir_in, tempfile};
 use tokio;
 
 use crate::configparser::challenge::BuildObject;
@@ -17,10 +24,7 @@ use crate::configparser::UserPass;
 #[tokio::main(flavor = "current_thread")] // make this a sync function
 pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Result<String> {
     trace!("building image in directory {context:?} to tag {tag:?}");
-    let client = client()
-        .await
-        // truncate error chain with new error (returned error is way too verbose)
-        .map_err(|_| anyhow!("could not talk to Docker daemon (is DOCKER_HOST correct?)"))?;
+    let client = client().await?;
 
     let build_opts = BuildImageOptions {
         dockerfile: options.dockerfile.clone(),
@@ -76,10 +80,7 @@ pub async fn build_image(context: &Path, options: &BuildObject, tag: &str) -> Re
 #[tokio::main(flavor = "current_thread")] // make this a sync function
 pub async fn push_image(image_tag: &str, creds: &UserPass) -> Result<String> {
     info!("pushing image {image_tag:?} to registry");
-    let client = client()
-        .await
-        // truncate error chain with new error (returned error is way too verbose)
-        .map_err(|_| anyhow!("could not talk to Docker daemon (is DOCKER_HOST correct?)"))?;
+    let client = client().await?;
 
     let (image, tag) = image_tag
         .rsplit_once(":")
@@ -114,15 +115,104 @@ pub async fn push_image(image_tag: &str, creds: &UserPass) -> Result<String> {
     Ok(tag.to_string())
 }
 
+#[tokio::main(flavor = "current_thread")] // make this a sync function
+pub async fn create_container(image_tag: &str, name: &str) -> Result<String> {
+    debug!("creating container {name:?} from image {image_tag:?}");
+    let client = client().await?;
+
+    let opts = CreateContainerOptions {
+        name: name.to_string(),
+        ..Default::default()
+    };
+    let config = Config {
+        image: Some(image_tag),
+        ..Default::default()
+    };
+
+    let container = client.create_container(Some(opts), config).await?;
+    Ok(container.id)
+}
+
+#[tokio::main(flavor = "current_thread")] // make this a sync function
+pub async fn remove_container(name: &str) -> Result<()> {
+    debug!("removing container {name:?}");
+    let client = client().await?;
+
+    let opts = RemoveContainerOptions {
+        force: true,
+        ..Default::default()
+    };
+    client.remove_container(name, Some(opts)).await?;
+
+    Ok(())
+}
+
+pub async fn copy_file(
+    container_id: &str,
+    from_path: &str,
+    rename_to: Option<&str>,
+) -> Result<String> {
+    let client = client().await?;
+
+    // if no rename is given, use basename of `from` as target path
+    let target_path = match rename_to {
+        Some(to) => to,
+        None => Path::new(from_path).file_name().unwrap().to_str().unwrap(),
+    };
+
+    info!("copying {container_id}:{from_path} to {target_path}");
+
+    // Download single file from container in an archive
+    let opts = DownloadFromContainerOptions { path: from_path };
+    let mut dl_stream = client.download_from_container(container_id, Some(opts));
+
+    // scratch dir in chal repo (two vars for scoping reasons)
+    // let mut tempdir_full = tempdir_in(".")?;
+    // let tempdir = tempdir_full.path();
+
+    fs::create_dir("./.tempdir");
+    let tempdir = Path::new("./.tempdir");
+
+    // collect byte stream chunks into full file
+    let mut tarfile = File::create(tempdir.join(format!("download_{target_path}.tar")))?;
+    while let Some(chunk) = dl_stream.next().await {
+        tarfile.write_all(&chunk?)?;
+    }
+    tarfile.rewind();
+
+    // unpack file retrieved to temp dir
+    trace!("extracting download tar to {:?}", tempdir);
+    let mut tar = tar::Archive::new(tarfile);
+
+    // extract single file from archive to disk
+    // we only copied out one file, so this tar should only have one file
+    if let Some(Ok(mut entry)) = tar.entries()?.next() {
+        let mut target = File::create_new(target_path)?;
+        io::copy(&mut entry, &mut target);
+    } else {
+        bail!("downloaded archive for {container_id}:{from_path} has no files in it!");
+    }
+
+    Ok(target_path.to_string())
+}
+
 //
 // helper functions
 //
+
+// connect to Docker/Podman daemon once and share client
+static CLIENT: LazyLock<std::result::Result<Docker, bollard::errors::Error>> =
+    LazyLock::new(|| {
+        debug!("connecting to docker...");
+        Docker::connect_with_defaults()
+    });
 pub async fn client() -> Result<Docker> {
-    debug!("connecting to docker...");
-    let client = Docker::connect_with_defaults()?;
-    client.ping().await?;
+    let c = CLIENT
+        .as_ref()
+        .map_err(|_| anyhow!("could not talk to Docker daemon (is DOCKER_HOST correct?)"))?;
+    c.ping().await?;
 
-    Ok(client)
+    Ok(c.clone())
 }
 
 #[derive(Debug)]