cmd: add applydiff-using-staging-dir

add a new command to exercise the ApplyDiff from a staging directory.

Signed-off-by: Giuseppe Scrivano <[email protected]>

cmd/containers-storage/diff.go

@@ -1,12 +1,17 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
 
 	"github.com/containers/storage"
+	graphdriver "github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/archive"
+	"github.com/containers/storage/pkg/chunked"
+	"github.com/containers/storage/pkg/chunked/compressor"
+	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/mflag"
 )
 
@@ -96,6 +101,93 @@ func diff(flags *mflag.FlagSet, action string, m storage.Store, args []string) (
 	return 0, nil
 }
 
+type fileFetcher struct {
+	file *os.File
+}
+
+func sendFileParts(f *fileFetcher, chunks []chunked.ImageSourceChunk, streams chan io.ReadCloser, errors chan error) {
+	defer close(streams)
+	defer close(errors)
+
+	for _, chunk := range chunks {
+		l := io.NewSectionReader(f.file, int64(chunk.Offset), int64(chunk.Length))
+		streams <- ioutils.NewReadCloserWrapper(l, func() error {
+			return nil
+		})
+	}
+}
+
+func (f fileFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
+	streams := make(chan io.ReadCloser)
+	errs := make(chan error)
+	go sendFileParts(&f, chunks, streams, errs)
+	return streams, errs, nil
+}
+
+func applyDiffUsingStagingDirectory(flags *mflag.FlagSet, action string, m storage.Store, args []string) (int, error) {
+	if len(args) < 2 {
+		return 2, nil
+	}
+
+	layer := args[0]
+	sourceDirectory := args[1]
+
+	tOptions := archive.TarOptions{}
+	tr, err := archive.TarWithOptions(sourceDirectory, &tOptions)
+	if err != nil {
+		return 1, err
+	}
+	defer tr.Close()
+
+	tar, err := os.CreateTemp("", "layer-diff-tar-")
+	if err != nil {
+		return 1, err
+	}
+	defer os.Remove(tar.Name())
+	defer tar.Close()
+
+	// we go through the zstd:chunked compressor first so that it generates the metadata required to mount
+	// a composefs image.
+
+	metadata := make(map[string]string)
+	compressor, err := compressor.ZstdCompressor(tar, metadata, nil)
+	if err != nil {
+		return 1, err
+	}
+
+	if _, err := io.Copy(compressor, tr); err != nil {
+		return 1, err
+	}
+	if err := compressor.Close(); err != nil {
+		return 1, err
+	}
+
+	size, err := tar.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return 1, err
+	}
+
+	fetcher := fileFetcher{
+		file: tar,
+	}
+
+	differ, err := chunked.GetDiffer(context.Background(), m, size, metadata, &fetcher)
+	if err != nil {
+		return 1, err
+	}
+
+	var options graphdriver.ApplyDiffOpts
+	out, err := m.ApplyDiffWithDiffer("", &options, differ)
+	if err != nil {
+		return 1, err
+	}
+	if err := m.ApplyDiffFromStagingDirectory(layer, out.Target, out, &options); err != nil {
+		m.CleanupStagingDirectory(out.Target)
+		return 1, err
+	}
+	return 0, nil
+}
+
 func applyDiff(flags *mflag.FlagSet, action string, m storage.Store, args []string) (int, error) {
 	if len(args) < 1 {
 		return 1, nil
@@ -179,4 +271,12 @@ func init() {
 			flags.StringVar(&applyDiffFile, []string{"-file", "f"}, "", "Read from file instead of stdin")
 		},
 	})
+	commands = append(commands, command{
+		names:       []string{"applydiff-using-staging-dir"},
+		optionsHelp: "layerNameOrID directory",
+		usage:       "Apply a diff to a layer using a staging directory",
+		minArgs:     2,
+		maxArgs:     2,
+		action:      applyDiffUsingStagingDirectory,
+	})
 }

docs/containers-storage-applydiff-using-staging-dir.md

@@ -0,0 +1,27 @@
+## containers-storage-applydiff-using-staging-dir 1 "September 2023"
+
+## NAME
+containers-storage applydiff-using-staging-dir - Apply a layer diff to a layer using a staging directory
+
+## SYNOPSIS
+**containers-storage** **applydiff-using-staging-dir** *layerNameOrID* *source*
+
+## DESCRIPTION
+When a layer is first created, it contains no changes relative to its parent
+layer.  The layer can either be mounted read-write and its contents modified
+directly, or contents can be added (or removed) by applying a layer diff.  A
+layer diff takes the form of a (possibly compressed) tar archive with
+additional information present in its headers, and can be produced by running
+*containers-storage diff* or an equivalent.
+
+Differently than **apply-diff**, the command **applydiff-using-staging-dir**
+first creates a staging directory and then moves the final result to the destination.
+
+## EXAMPLE
+**containers-storage applydiff-using-staging-dir 5891b5b522 /path/to/diff**
+
+## SEE ALSO
+containers-storage-apply-diff(1)
+containers-storage-changes(1)
+containers-storage-diff(1)
+containers-storage-diffsize(1)