WIP: Add a new container/store module

The initial scope of this project was just "encapsulating" ostree
commits in containers.

However, when doing that a very, very natural question arises:
Why not support *deriving* from that base image container, and
have the tooling natively support importing it?

This initial prototype code implements that.  Here, we still use
the `tar::import` path for the base image - we expect it to have
a pre-generated ostree commit.

This new `container::store` module processes layered images and
generates (client side) ostree commits from the tar layers.

There's a whole lot of new infrastructure we need around mapping
ostree refs to blobs and images, etc.

lib/Cargo.toml

@@ -41,6 +41,7 @@ tokio = { features = ["full"], version = "1" }
 tokio-stream = "0.1.5"
 tokio-util = { features = ["io"], version = "0.6" }
 tracing = "0.1"
+scopeguard = "1.1.0"
 
 [dev-dependencies]
 clap = "2.33.3"

lib/src/cli.rs

@@ -12,6 +12,7 @@ use std::convert::TryInto;
 use std::ffi::OsString;
 use structopt::StructOpt;
 
+use crate::container::store::{LayeredImageImporter, PrepareResult};
 use crate::container::{Config, ImportOptions};
 
 #[derive(Debug, StructOpt)]
@@ -105,6 +106,16 @@ enum ContainerOpts {
         #[structopt(long)]
         cmd: Option<Vec<String>>,
     },
+
+    /// Store a (possibly) layered container image with an OSTree base
+    StoreLayered {
+        /// Path to the repository
+        #[structopt(long)]
+        repo: String,
+
+        /// Image reference, e.g. registry:quay.io/exampleos/exampleos:latest
+        imgref: String,
+    },
 }
 
 /// Options for the Integrity Measurement Architecture (IMA).
@@ -249,6 +260,23 @@ async fn container_info(imgref: &str) -> Result<()> {
     Ok(())
 }
 
+/// Write a layered container image into an OSTree commit.
+async fn container_store(repo: &str, imgref: &str) -> Result<()> {
+    let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?;
+    let imgref = imgref.try_into()?;
+    let mut imp = LayeredImageImporter::new(&repo, &imgref).await?;
+    let prep = match imp.prepare().await? {
+        PrepareResult::AlreadyPresent(c) => {
+            println!("No changes in {} => {}", imgref, c);
+            return Ok(());
+        }
+        PrepareResult::Ready(r) => r,
+    };
+    let import = imp.import(prep).await?;
+    println!("Wrote: {} => {}", imgref, import);
+    Ok(())
+}
+
 /// Add IMA signatures to an ostree commit, generating a new commit.
 fn ima_sign(cmdopts: &ImaSignOpts) -> Result<()> {
     let repo =
@@ -306,6 +334,9 @@ where
                 .collect();
             container_export(&repo, &rev, &imgref, labels?, cmd).await
         }
+        Opt::Container(ContainerOpts::StoreLayered { repo, imgref }) => {
+            container_store(&repo, &imgref).await
+        }
         Opt::ImaSign(ref opts) => ima_sign(opts),
     }
 }

lib/src/container/mod.rs

@@ -230,6 +230,7 @@ pub use import::*;
 mod imageproxy;
 mod oci;
 mod skopeo;
+pub mod store;
 
 #[cfg(test)]
 mod tests {

lib/src/container/store.rs

@@ -0,0 +1,204 @@
+//! APIs for generating OSTree commits from layered container images
+//!
+//! # Extension of import support
+//!
+//! This code supports ingesting arbitrary layered container images from an ostree-exported
+//! base.  See [`super::import`] for more information on encaspulation of images.
+
+use super::imageproxy::ImageProxy;
+use super::*;
+use anyhow::{anyhow, Context};
+use fn_error_context::context;
+use ostree::gio;
+use ostree::prelude::Cast;
+
+const LAYER_PREFIX: &str = "ostree/container/blob/";
+const IMAGE_PREFIX: &str = "ostree/container/image";
+
+/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`.
+fn ref_for_blob_digest(d: &str) -> Result<String> {
+    let escaped = crate::util::escape_for_ref(d)?;
+    Ok(format!("{}{}", LAYER_PREFIX, escaped))
+}
+
+/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`.
+fn ref_for_layer(l: &oci::ManifestLayer) -> Result<String> {
+    ref_for_blob_digest(l.digest.as_str())
+}
+
+/// Context for importing a container image.
+pub struct LayeredImageImporter {
+    repo: ostree::Repo,
+    proxy: ImageProxy,
+    imgref: OstreeImageReference,
+    ostree_ref: String,
+}
+
+/// Result of invoking [`LayeredImageImporter::prepare`].
+pub enum PrepareResult {
+    /// The image reference is already present; the contained string is the OSTree commit.
+    AlreadyPresent(String),
+    /// The image needs to be downloaded
+    Ready(PreparedImport),
+}
+
+/// Information about which layers need to be downloaded.
+pub struct PreparedImport {
+    /// The manifest digest that was found
+    pub manifest_digest: String,
+    manifest: oci::Manifest,
+}
+
+impl LayeredImageImporter {
+    /// Create a new importer.
+    pub async fn new(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result<Self> {
+        let proxy = ImageProxy::new(&imgref.imgref).await?;
+        let repo = repo.clone();
+        let ostree_ref = crate::util::escape_for_ref(&imgref.imgref.to_string())?;
+        Ok(LayeredImageImporter {
+            repo,
+            proxy,
+            ostree_ref,
+            imgref: imgref.clone(),
+        })
+    }
+
+    /// Determine if there is a new manifest, and if so return its digest.
+    #[context("Fetching manifest")]
+    pub async fn prepare(&mut self) -> Result<PrepareResult> {
+        match &self.imgref.sigverify {
+            SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure()? => {
+                return Err(anyhow!("containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage"));
+            }
+            SignatureSource::OstreeRemote(_) => {
+                return Err(anyhow!(
+                    "Cannot currently verify layered containers via ostree remote"
+                ));
+            }
+            _ => {}
+        }
+
+        // Do we already have this image?  If so, we're done.
+        if let Some(merge_commit) = self.repo.resolve_rev(&self.ostree_ref, true)? {
+            return Ok(PrepareResult::AlreadyPresent(merge_commit.to_string()));
+        }
+
+        let (manifest_digest, manifest_bytes) = self.proxy.fetch_manifest().await?;
+        let manifest: oci::Manifest = serde_json::from_slice(&manifest_bytes)?;
+        let imp = PreparedImport {
+            manifest,
+            manifest_digest,
+        };
+        Ok(PrepareResult::Ready(imp))
+    }
+
+    /// Import a layered container image
+    pub async fn import(mut self, import: PreparedImport) -> Result<String> {
+        // TODO hook up gcancellable + async https://github.com/gtk-rs/gtk-rs-core/issues/240
+        let cancellable = gio::NONE_CANCELLABLE;
+        let manifest = import.manifest;
+
+        // First download the base image - we need the SELinux policy
+        // there to label all following layers.
+        // Presence of at least one layer is validated by find_layer_blobids
+
+        let mut layers = manifest.layers.iter();
+        let base_layer = layers.next().ok_or_else(|| anyhow!("No layers found"))?;
+        let base_ref = ref_for_layer(base_layer)?;
+        let base_commit = if let Some(base_commit) = self.repo.resolve_rev(&base_ref, true)? {
+            base_commit.to_string()
+        } else {
+            let blob = self.proxy.fetch_layer_decompress(base_layer).await?;
+            let commit = crate::tar::import_tar(&self.repo, blob, None)
+                .await
+                .with_context(|| format!("Parsing blob {}", base_layer.digest))?;
+            // TODO support ref writing in tar import
+            self.repo
+                .set_ref_immediate(None, &base_ref, Some(commit.as_str()), cancellable)?;
+            commit
+        };
+
+        let mut layer_commits = Vec::new();
+        for layer in layers {
+            let layer_ref = ref_for_layer(layer)?;
+            let layer_commit = self.repo.resolve_rev(&layer_ref, true)?;
+
+            if let Some(c) = layer_commit {
+                layer_commits.push(c.to_string());
+            } else {
+                let blob = self.proxy.fetch_layer_decompress(layer).await?;
+                // An important aspect of this is that we SELinux label the derived layers using
+                // the base policy.
+                let opts = crate::tar::WriteTarOptions {
+                    base: Some(base_commit.as_str()),
+                    selinux: true,
+                };
+                let commit = crate::tar::write_tar(&self.repo, blob, &layer_ref, Some(opts))
+                    .await
+                    .with_context(|| format!("Parsing layer blob {}", layer.digest))?;
+                layer_commits.push(commit);
+            }
+        }
+
+        // We're done with the proxy, make sure it didn't have any errors.
+        self.proxy.finalize().await?;
+
+        // Destructure to transfer ownership to thread
+        let repo = self.repo;
+        let target_ref = self.ostree_ref;
+        tokio::task::spawn_blocking(move || -> Result<String> {
+            let repo = &repo;
+            scopeguard::defer! {
+                let _ = repo.abort_transaction(cancellable);
+            }
+            let (base_commit_tree, _) = repo.read_commit(&base_commit, gio::NONE_CANCELLABLE)?;
+            let base_commit_tree = base_commit_tree.downcast::<ostree::RepoFile>().unwrap();
+            let base_contents_obj = base_commit_tree.tree_get_contents_checksum().unwrap();
+            let base_metadata_obj = base_commit_tree.tree_get_metadata_checksum().unwrap();
+            let mt =
+                ostree::MutableTree::from_checksum(&repo, &base_contents_obj, &base_metadata_obj);
+            repo.prepare_transaction(cancellable)?;
+            // Layer all subsequent commits
+            for commit in layer_commits {
+                let (layer_tree, _) = repo.read_commit(&commit, gio::NONE_CANCELLABLE)?;
+                repo.write_directory_to_mtree(&layer_tree, &mt, None, gio::NONE_CANCELLABLE)?;
+            }
+
+            let merged_root = repo.write_mtree(&mt, gio::NONE_CANCELLABLE)?;
+            let merged_root = merged_root.downcast::<ostree::RepoFile>().unwrap();
+            let merged_commit =
+                repo.write_commit(None, None, None, None, &merged_root, gio::NONE_CANCELLABLE)?;
+            repo.transaction_set_ref(None, &target_ref, Some(merged_commit.as_str()));
+            repo.commit_transaction(cancellable)?;
+            Ok(merged_commit.to_string())
+        })
+        .await?
+    }
+}
+
+/// List all images stored
+pub fn list_images(repo: &ostree::Repo) -> Result<Vec<String>> {
+    let cancellable = gio::NONE_CANCELLABLE;
+    let refs = repo.list_refs_ext(
+        Some(IMAGE_PREFIX),
+        ostree::RepoListRefsExtFlags::empty(),
+        cancellable,
+    )?;
+    let r: Result<Vec<_>> = refs
+        .keys()
+        .map(|imgname| {
+            let img = imgname.strip_prefix(IMAGE_PREFIX).unwrap();
+            crate::util::unescape_for_ref(img)
+        })
+        .collect();
+    Ok(r?)
+}
+
+/// Remove the specified images and their corresponding blobs.
+pub fn prune_images(_repo: &ostree::Repo, _imgs: &[&str]) -> Result<()> {
+    // Most robust approach is to iterate over all known images, load the
+    // manifest and build the set of reachable blobs, then compute the set
+    // Set(unreachable) = Set(all) - Set(reachable)
+    // And remove the unreachable ones.
+    unimplemented!()
+}

lib/src/lib.rs

@@ -35,3 +35,5 @@ pub mod prelude {
     #[doc(hidden)]
     pub use ostree::prelude::*;
 }
+
+mod util;