WIP: Add a new container/store module

The initial scope of this project was just "encapsulating" ostree
commits in containers.

However, when doing that a very, very natural question arises:
Why not support *deriving* from that base image container, and
have the tooling natively support importing it?

This initial prototype code implements that.  Here, we still use
the `tar::import` path for the base image - we expect it to have
a pre-generated ostree commit.

This new `container::store` module processes layered images and
generates (client side) ostree commits from the tar layers.

There's a whole lot of new infrastructure we need around mapping
ostree refs to blobs and images, etc.

lib/src/cli.rs

@@ -8,11 +8,12 @@
 use anyhow::Result;
 use ostree::gio;
 use std::collections::BTreeMap;
-use std::convert::TryInto;
+use std::convert::{TryFrom, TryInto};
 use std::ffi::OsString;
 use structopt::StructOpt;
 
-use crate::container::{Config, ImportOptions};
+use crate::container::store::{LayeredImageImporter, PrepareResult};
+use crate::container::{Config, ImportOptions, OstreeImageReference};
 
 #[derive(Debug, StructOpt)]
 struct BuildOpts {
@@ -107,6 +108,63 @@ enum ContainerOpts {
         #[structopt(long)]
         cmd: Option<Vec<String>>,
     },
+
+    /// Commands for working with (possibly layered, non-encapsulated) container images.
+    Image(ContainerImageOpts),
+}
+
+/// Options for import/export to tar archives.
+#[derive(Debug, StructOpt)]
+enum ContainerImageOpts {
+    /// List container images
+    List {
+        /// Path to the repository
+        #[structopt(long)]
+        repo: String,
+    },
+
+    /// Pull (or update) a container image.
+    Pull {
+        /// Path to the repository
+        #[structopt(long)]
+        repo: String,
+
+        /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest
+        imgref: String,
+    },
+
+    /// Copy a pulled container image from one repo to another.
+    Copy {
+        /// Path to the source repository
+        #[structopt(long)]
+        src_repo: String,
+
+        /// Path to the destination repository
+        #[structopt(long)]
+        dest_repo: String,
+
+        /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest
+        imgref: String,
+    },
+
+    /// Perform initial deployment for a container image
+    Deploy {
+        /// Path to the system root
+        #[structopt(long)]
+        sysroot: String,
+
+        /// Name for the state directory, also known as "osname".
+        #[structopt(long)]
+        stateroot: String,
+
+        /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest
+        #[structopt(long)]
+        imgref: String,
+
+        #[structopt(long)]
+        /// Add a kernel argument
+        karg: Option<Vec<String>>,
+    },
 }
 
 /// Options for the Integrity Measurement Architecture (IMA).
@@ -251,6 +309,52 @@ async fn container_info(imgref: &str) -> Result<()> {
     Ok(())
 }
 
+/// Write a layered container image into an OSTree commit.
+async fn container_store(repo: &str, imgref: &str) -> Result<()> {
+    let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?;
+    let imgref = imgref.try_into()?;
+    let mut imp = LayeredImageImporter::new(&repo, &imgref).await?;
+    let prep = match imp.prepare().await? {
+        PrepareResult::AlreadyPresent(c) => {
+            println!("No changes in {} => {}", imgref, c);
+            return Ok(());
+        }
+        PrepareResult::Ready(r) => r,
+    };
+    if prep.base_layer.commit.is_none() {
+        let size = crate::glib::format_size(prep.base_layer.size());
+        println!(
+            "Downloading base layer: {} ({})",
+            prep.base_layer.digest(),
+            size
+        );
+    } else {
+        println!("Using base: {}", prep.base_layer.digest());
+    }
+    for layer in prep.layers.iter() {
+        if layer.commit.is_some() {
+            println!("Using layer: {}", layer.digest());
+        } else {
+            let size = crate::glib::format_size(layer.size());
+            println!("Downloading layer: {} ({})", layer.digest(), size);
+        }
+    }
+    let import = imp.import(prep).await?;
+    if !import.layer_filtered_content.is_empty() {
+        for (layerid, filtered) in import.layer_filtered_content {
+            eprintln!("Unsupported paths filtered from {}:", layerid);
+            for (prefix, count) in filtered {
+                eprintln!("  {}: {}", prefix, count);
+            }
+        }
+    }
+    println!(
+        "Wrote: {} => {} => {}",
+        imgref, import.ostree_ref, import.commit
+    );
+    Ok(())
+}
+
 /// Add IMA signatures to an ostree commit, generating a new commit.
 fn ima_sign(cmdopts: &ImaSignOpts) -> Result<()> {
     let repo =
@@ -309,6 +413,48 @@ where
                     .collect();
                 container_export(&repo, &rev, &imgref, labels?, cmd).await
             }
+            ContainerOpts::Image(opts) => match opts {
+                ContainerImageOpts::List { repo } => {
+                    let repo =
+                        &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?;
+                    for image in crate::container::store::list_images(&repo)? {
+                        println!("{}", image);
+                    }
+                    Ok(())
+                }
+                ContainerImageOpts::Pull { repo, imgref } => container_store(&repo, &imgref).await,
+                ContainerImageOpts::Copy {
+                    src_repo,
+                    dest_repo,
+                    imgref,
+                } => {
+                    let src_repo =
+                        &ostree::Repo::open_at(libc::AT_FDCWD, &src_repo, gio::NONE_CANCELLABLE)?;
+                    let dest_repo =
+                        &ostree::Repo::open_at(libc::AT_FDCWD, &dest_repo, gio::NONE_CANCELLABLE)?;
+                    let imgref = OstreeImageReference::try_from(imgref.as_str())?;
+                    crate::container::store::copy(src_repo, dest_repo, &imgref).await
+                }
+                ContainerImageOpts::Deploy {
+                    sysroot,
+                    stateroot,
+                    imgref,
+                    karg,
+                } => {
+                    let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot)));
+                    let imgref = OstreeImageReference::try_from(imgref.as_str())?;
+                    let kargs = karg.as_deref();
+                    let kargs = kargs.map(|v| {
+                        let r: Vec<_> = v.iter().map(|s| s.as_str()).collect();
+                        r
+                    });
+                    let options = crate::container::deploy::DeployOpts {
+                        kargs: kargs.as_deref(),
+                    };
+                    crate::container::deploy::deploy(sysroot, &stateroot, &imgref, Some(options))
+                        .await
+                }
+            },
         },
         Opt::ImaSign(ref opts) => ima_sign(opts),
     }

lib/src/container/deploy.rs

@@ -0,0 +1,53 @@
+//! Perform initial setup for a container image based system root
+
+use super::OstreeImageReference;
+use crate::container::store::PrepareResult;
+use anyhow::Result;
+use ostree::glib;
+
+/// The key in the OSTree origin which holds a serialized [`super::OstreeImageReference`].
+pub const ORIGIN_CONTAINER: &str = "container";
+
+async fn pull_idempotent(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result<String> {
+    let mut imp = super::store::LayeredImageImporter::new(repo, imgref).await?;
+    match imp.prepare().await? {
+        PrepareResult::AlreadyPresent(r) => Ok(r),
+        PrepareResult::Ready(prep) => Ok(imp.import(prep).await?.commit),
+    }
+}
+
+/// Options configuring deployment.
+#[derive(Debug, Default)]
+pub struct DeployOpts<'a> {
+    /// Kernel arguments to use.
+    pub kargs: Option<&'a [&'a str]>,
+}
+
+/// Write a container image to an OSTree deployment.
+///
+/// This API is currently intended for only an initial deployment.
+pub async fn deploy<'opts>(
+    sysroot: &ostree::Sysroot,
+    stateroot: &str,
+    imgref: &OstreeImageReference,
+    options: Option<DeployOpts<'opts>>,
+) -> Result<()> {
+    let cancellable = ostree::gio::NONE_CANCELLABLE;
+    let options = options.unwrap_or_default();
+    let repo = &sysroot.repo().unwrap();
+    let commit = &pull_idempotent(repo, imgref).await?;
+    let origin = glib::KeyFile::new();
+    origin.set_string("ostree", ORIGIN_CONTAINER, &imgref.to_string());
+    let deployment = &sysroot.deploy_tree(
+        Some(stateroot),
+        commit,
+        Some(&origin),
+        None,
+        options.kargs.unwrap_or_default(),
+        cancellable,
+    )?;
+    let flags = ostree::SysrootSimpleWriteDeploymentFlags::NONE;
+    sysroot.simple_write_deployment(Some(stateroot), deployment, None, flags, cancellable)?;
+    sysroot.cleanup(cancellable)?;
+    Ok(())
+}

lib/src/container/mod.rs

@@ -223,13 +223,15 @@ impl std::fmt::Display for OstreeImageReference {
     }
 }
 
+pub mod deploy;
 mod export;
 pub use export::*;
 mod import;
 pub use import::*;
 mod imageproxy;
 mod oci;
 mod skopeo;
+pub mod store;
 
 #[cfg(test)]
 mod tests {