From 4d808ac52a8bf981c217c3df562ab20a052ee86e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Sun, 7 Apr 2024 18:58:10 +0800 Subject: [PATCH] feat: support `--format` for `oras pull` (#1293) Signed-off-by: Billy Zha --- cmd/oras/internal/display/handler.go | 23 +++- .../internal/display/metadata/interface.go | 10 ++ .../internal/display/metadata/json/pull.go | 55 ++++++++ .../internal/display/metadata/model/pull.go | 95 +++++++++++++ .../display/metadata/template/pull.go | 57 ++++++++ .../internal/display/metadata/text/pull.go | 61 +++++++++ cmd/oras/internal/display/status/discard.go | 36 ++++- cmd/oras/internal/display/status/interface.go | 23 +++- cmd/oras/internal/display/status/text.go | 48 ++++++- cmd/oras/internal/display/status/tty.go | 54 +++++++- cmd/oras/internal/display/status/tty_test.go | 62 ++++++++- cmd/oras/internal/display/status/utils.go | 34 +++++ cmd/oras/internal/display/utils/utils.go | 33 +++++ cmd/oras/root/attach.go | 4 +- cmd/oras/root/pull.go | 129 +++++------------- cmd/oras/root/pull_test.go | 63 --------- cmd/oras/root/push.go | 12 +- test/e2e/suite/command/pull.go | 66 ++++++++- 18 files changed, 684 insertions(+), 181 deletions(-) create mode 100644 cmd/oras/internal/display/metadata/json/pull.go create mode 100644 cmd/oras/internal/display/metadata/model/pull.go create mode 100644 cmd/oras/internal/display/metadata/template/pull.go create mode 100644 cmd/oras/internal/display/metadata/text/pull.go create mode 100644 cmd/oras/internal/display/status/utils.go create mode 100644 cmd/oras/internal/display/utils/utils.go delete mode 100644 cmd/oras/root/pull_test.go diff --git a/cmd/oras/internal/display/handler.go b/cmd/oras/internal/display/handler.go index c3c931609..66aa468a4 100644 --- a/cmd/oras/internal/display/handler.go +++ b/cmd/oras/internal/display/handler.go @@ -46,7 +46,6 @@ func NewPushHandler(format string, tty *os.File, out io.Writer, verbose bool) (s default: metadataHandler = template.NewPushHandler(out, format) } - return statusHandler, metadataHandler } @@ -70,6 +69,28 @@ func NewAttachHandler(format string, tty *os.File, out io.Writer, verbose bool) default: metadataHandler = template.NewAttachHandler(out, format) } + return statusHandler, metadataHandler +} + +// NewPullHandler returns status and metadata handlers for pull command. +func NewPullHandler(format string, path string, tty *os.File, out io.Writer, verbose bool) (status.PullHandler, metadata.PullHandler) { + var statusHandler status.PullHandler + if tty != nil { + statusHandler = status.NewTTYPullHandler(tty) + } else if format == "" { + statusHandler = status.NewTextPullHandler(out, verbose) + } else { + statusHandler = status.NewDiscardHandler() + } + var metadataHandler metadata.PullHandler + switch format { + case "": + metadataHandler = text.NewPullHandler(out) + case "json": + metadataHandler = json.NewPullHandler(out, path) + default: + metadataHandler = template.NewPullHandler(out, path, format) + } return statusHandler, metadataHandler } diff --git a/cmd/oras/internal/display/metadata/interface.go b/cmd/oras/internal/display/metadata/interface.go index 577f9eb2e..0709a0578 100644 --- a/cmd/oras/internal/display/metadata/interface.go +++ b/cmd/oras/internal/display/metadata/interface.go @@ -30,3 +30,13 @@ type PushHandler interface { type AttachHandler interface { OnCompleted(opts *option.Target, root, subject ocispec.Descriptor) error } + +// PullHandler handles metadata output for pull events. +type PullHandler interface { + // OnLayerSkipped is called when a layer is skipped. + OnLayerSkipped(ocispec.Descriptor) error + // OnFilePulled is called after a file is pulled. + OnFilePulled(name string, outputDir string, desc ocispec.Descriptor, descPath string) error + // OnCompleted is called when the pull cmd execution is completed. + OnCompleted(opts *option.Target, desc ocispec.Descriptor) error +} diff --git a/cmd/oras/internal/display/metadata/json/pull.go b/cmd/oras/internal/display/metadata/json/pull.go new file mode 100644 index 000000000..67a2b15fb --- /dev/null +++ b/cmd/oras/internal/display/metadata/json/pull.go @@ -0,0 +1,55 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "io" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/cmd/oras/internal/display/metadata" + "oras.land/oras/cmd/oras/internal/display/metadata/model" + "oras.land/oras/cmd/oras/internal/option" +) + +// PullHandler handles JSON metadata output for pull events. +type PullHandler struct { + path string + pulled model.Pulled + out io.Writer +} + +// OnLayerSkipped implements metadata.PullHandler. +func (ph *PullHandler) OnLayerSkipped(ocispec.Descriptor) error { + return nil +} + +// NewPullHandler returns a new handler for Pull events. +func NewPullHandler(out io.Writer, path string) metadata.PullHandler { + return &PullHandler{ + out: out, + path: path, + } +} + +// OnFilePulled implements metadata.PullHandler. +func (ph *PullHandler) OnFilePulled(name string, outputDir string, desc ocispec.Descriptor, descPath string) error { + return ph.pulled.Add(name, outputDir, desc, descPath) +} + +// OnCompleted implements metadata.PullHandler. +func (ph *PullHandler) OnCompleted(opts *option.Target, desc ocispec.Descriptor) error { + return printJSON(ph.out, model.NewPull(ph.path+"@"+desc.Digest.String(), ph.pulled.Files())) +} diff --git a/cmd/oras/internal/display/metadata/model/pull.go b/cmd/oras/internal/display/metadata/model/pull.go new file mode 100644 index 000000000..cc2f5b1fd --- /dev/null +++ b/cmd/oras/internal/display/metadata/model/pull.go @@ -0,0 +1,95 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "path/filepath" + "slices" + "sync" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content/file" +) + +// File records metadata of a pulled file. +type File struct { + // Path is the absolute path of the pulled file. + Path string + Descriptor +} + +// newFile creates a new file metadata. +func newFile(name string, outputDir string, desc ocispec.Descriptor, descPath string) (File, error) { + path := name + if !filepath.IsAbs(name) { + var err error + path, err = filepath.Abs(filepath.Join(outputDir, name)) + // not likely to go wrong since the file has already be written to file store + if err != nil { + return File{}, fmt.Errorf("failed to get absolute path of pulled file %s: %w", name, err) + } + } else { + path = filepath.Clean(path) + } + if desc.Annotations[file.AnnotationUnpack] == "true" { + path += string(filepath.Separator) + } + return File{ + Path: path, + Descriptor: FromDescriptor(descPath, desc), + }, nil +} + +type pull struct { + DigestReference + Files []File `json:"Files"` +} + +// NewPull creates a new metadata struct for pull command. +func NewPull(digestReference string, files []File) any { + return pull{ + DigestReference: DigestReference{ + Ref: digestReference, + }, + Files: files, + } +} + +// Pulled records all pulled files. +type Pulled struct { + lock sync.Mutex + files []File +} + +// Files returns all pulled files. +func (p *Pulled) Files() []File { + p.lock.Lock() + defer p.lock.Unlock() + return slices.Clone(p.files) +} + +// Add adds a pulled file. +func (p *Pulled) Add(name string, outputDir string, desc ocispec.Descriptor, descPath string) error { + p.lock.Lock() + defer p.lock.Unlock() + file, err := newFile(name, outputDir, desc, descPath) + if err != nil { + return err + } + p.files = append(p.files, file) + return nil +} diff --git a/cmd/oras/internal/display/metadata/template/pull.go b/cmd/oras/internal/display/metadata/template/pull.go new file mode 100644 index 000000000..fa940f25a --- /dev/null +++ b/cmd/oras/internal/display/metadata/template/pull.go @@ -0,0 +1,57 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package template + +import ( + "io" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/cmd/oras/internal/display/metadata" + "oras.land/oras/cmd/oras/internal/display/metadata/model" + "oras.land/oras/cmd/oras/internal/option" +) + +// PullHandler handles text metadata output for pull events. +type PullHandler struct { + template string + path string + out io.Writer + pulled model.Pulled +} + +// OnCompleted implements metadata.PullHandler. +func (ph *PullHandler) OnCompleted(opts *option.Target, desc ocispec.Descriptor) error { + return parseAndWrite(ph.out, model.NewPull(ph.path+"@"+desc.Digest.String(), ph.pulled.Files()), ph.template) +} + +// OnFilePulled implements metadata.PullHandler. +func (ph *PullHandler) OnFilePulled(name string, outputDir string, desc ocispec.Descriptor, descPath string) error { + return ph.pulled.Add(name, outputDir, desc, descPath) +} + +// OnLayerSkipped implements metadata.PullHandler. +func (ph *PullHandler) OnLayerSkipped(ocispec.Descriptor) error { + return nil +} + +// NewPullHandler returns a new handler for pull events. +func NewPullHandler(out io.Writer, path string, template string) metadata.PullHandler { + return &PullHandler{ + path: path, + template: template, + out: out, + } +} diff --git a/cmd/oras/internal/display/metadata/text/pull.go b/cmd/oras/internal/display/metadata/text/pull.go new file mode 100644 index 000000000..2413b5957 --- /dev/null +++ b/cmd/oras/internal/display/metadata/text/pull.go @@ -0,0 +1,61 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package text + +import ( + "fmt" + "io" + "sync/atomic" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras/cmd/oras/internal/display/metadata" + "oras.land/oras/cmd/oras/internal/option" +) + +// PullHandler handles text metadata output for pull events. +type PullHandler struct { + out io.Writer + layerSkipped atomic.Bool +} + +// OnCompleted implements metadata.PullHandler. +func (p *PullHandler) OnCompleted(opts *option.Target, desc ocispec.Descriptor) error { + if p.layerSkipped.Load() { + _, _ = fmt.Fprintf(p.out, "Skipped pulling layers without file name in %q\n", ocispec.AnnotationTitle) + _, _ = fmt.Fprintf(p.out, "Use 'oras copy %s --to-oci-layout ' to pull all layers.\n", opts.RawReference) + } else { + _, _ = fmt.Fprintln(p.out, "Pulled", opts.AnnotatedReference()) + _, _ = fmt.Fprintln(p.out, "Digest:", desc.Digest) + } + return nil +} + +func (p *PullHandler) OnFilePulled(name string, outputDir string, desc ocispec.Descriptor, descPath string) error { + return nil +} + +// OnLayerSkipped implements metadata.PullHandler. +func (ph *PullHandler) OnLayerSkipped(ocispec.Descriptor) error { + ph.layerSkipped.Store(true) + return nil +} + +// NewPullHandler returns a new handler for Pull events. +func NewPullHandler(out io.Writer) metadata.PullHandler { + return &PullHandler{ + out: out, + } +} diff --git a/cmd/oras/internal/display/status/discard.go b/cmd/oras/internal/display/status/discard.go index e2cf35a31..91fe6229c 100644 --- a/cmd/oras/internal/display/status/discard.go +++ b/cmd/oras/internal/display/status/discard.go @@ -16,10 +16,15 @@ limitations under the License. package status import ( + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" ) +func discardStopTrack() error { + return nil +} + // DiscardHandler is a no-op handler that discards all status updates. type DiscardHandler struct{} @@ -38,10 +43,35 @@ func (DiscardHandler) OnEmptyArtifact() error { return nil } -// TrackTarget returns a target with status tracking -func (DiscardHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, error) { - return gt, nil +// TrackTarget returns a target with status tracking. +func (DiscardHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) { + return gt, discardStopTrack, nil } // UpdateCopyOptions updates the copy options for the artifact push. func (DiscardHandler) UpdateCopyOptions(opts *oras.CopyGraphOptions, fetcher content.Fetcher) {} + +// OnNodeDownloading implements PullHandler. +func (DiscardHandler) OnNodeDownloading(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeDownloaded implements PullHandler. +func (DiscardHandler) OnNodeDownloaded(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeRestored implements PullHandler. +func (DiscardHandler) OnNodeRestored(_ ocispec.Descriptor) error { + return nil +} + +// OnNodeProcessing implements PullHandler. +func (DiscardHandler) OnNodeProcessing(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeProcessing implements PullHandler. +func (DiscardHandler) OnNodeSkipped(desc ocispec.Descriptor) error { + return nil +} diff --git a/cmd/oras/internal/display/status/interface.go b/cmd/oras/internal/display/status/interface.go index d5b884245..4527bcc97 100644 --- a/cmd/oras/internal/display/status/interface.go +++ b/cmd/oras/internal/display/status/interface.go @@ -16,17 +16,38 @@ limitations under the License. package status import ( + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" ) +// StopTrackTargetFunc is the function type to stop tracking a target. +type StopTrackTargetFunc func() error + // PushHandler handles status output for push command. type PushHandler interface { OnFileLoading(name string) error OnEmptyArtifact() error - TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, error) + TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) UpdateCopyOptions(opts *oras.CopyGraphOptions, fetcher content.Fetcher) } // AttachHandler handles text status output for attach command. type AttachHandler PushHandler + +// PullHandler handles status output for pull command. +type PullHandler interface { + // TrackTarget returns a tracked target. + // If no TTY is available, it returns the original target. + TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) + // OnNodeProcessing is called when processing a manifest. + OnNodeProcessing(desc ocispec.Descriptor) error + // OnNodeDownloading is called before downloading a node. + OnNodeDownloading(desc ocispec.Descriptor) error + // OnNodeDownloaded is called after a node is downloaded. + OnNodeDownloaded(desc ocispec.Descriptor) error + // OnNodeRestored is called after a deduplicated node is restored. + OnNodeRestored(desc ocispec.Descriptor) error + // OnNodeSkipped is called when a node is skipped. + OnNodeSkipped(desc ocispec.Descriptor) error +} diff --git a/cmd/oras/internal/display/status/text.go b/cmd/oras/internal/display/status/text.go index 954f215ae..56aa886b9 100644 --- a/cmd/oras/internal/display/status/text.go +++ b/cmd/oras/internal/display/status/text.go @@ -53,8 +53,8 @@ func (ph *TextPushHandler) OnEmptyArtifact() error { } // TrackTarget returns a tracked target. -func (ph *TextPushHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, error) { - return gt, nil +func (ph *TextPushHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) { + return gt, discardStopTrack, nil } // UpdateCopyOptions adds status update to the copy options. @@ -86,3 +86,47 @@ func (ph *TextPushHandler) UpdateCopyOptions(opts *oras.CopyGraphOptions, fetche func NewTextAttachHandler(out io.Writer, verbose bool) AttachHandler { return NewTextPushHandler(out, verbose) } + +// TextPullHandler handles text status output for pull events. +type TextPullHandler struct { + verbose bool + printer *Printer +} + +// TrackTarget implements PullHander. +func (ph *TextPullHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) { + return gt, discardStopTrack, nil +} + +// OnNodeDownloading implements PullHandler. +func (ph *TextPullHandler) OnNodeDownloading(desc ocispec.Descriptor) error { + return PrintStatus(desc, PullPromptDownloading, ph.verbose) +} + +// OnNodeDownloaded implements PullHandler. +func (ph *TextPullHandler) OnNodeDownloaded(desc ocispec.Descriptor) error { + return PrintStatus(desc, PullPromptDownloaded, ph.verbose) +} + +// OnNodeRestored implements PullHandler. +func (ph *TextPullHandler) OnNodeRestored(desc ocispec.Descriptor) error { + return PrintStatus(desc, PullPromptRestored, ph.verbose) +} + +// OnNodeProcessing implements PullHandler. +func (ph *TextPullHandler) OnNodeProcessing(desc ocispec.Descriptor) error { + return PrintStatus(desc, PullPromptProcessing, ph.verbose) +} + +// OnNodeProcessing implements PullHandler. +func (ph *TextPullHandler) OnNodeSkipped(desc ocispec.Descriptor) error { + return PrintStatus(desc, PullPromptSkipped, ph.verbose) +} + +// NewTextPullHandler returns a new handler for pull command. +func NewTextPullHandler(out io.Writer, verbose bool) PullHandler { + return &TextPullHandler{ + verbose: verbose, + printer: NewPrinter(out), + } +} diff --git a/cmd/oras/internal/display/status/tty.go b/cmd/oras/internal/display/status/tty.go index ac9797467..19acab537 100644 --- a/cmd/oras/internal/display/status/tty.go +++ b/cmd/oras/internal/display/status/tty.go @@ -50,17 +50,17 @@ func (ph *TTYPushHandler) OnEmptyArtifact() error { } // TrackTarget returns a tracked target. -func (ph *TTYPushHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, error) { +func (ph *TTYPushHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) { const ( promptUploaded = "Uploaded " promptUploading = "Uploading" ) tracked, err := track.NewTarget(gt, promptUploading, promptUploaded, ph.tty) if err != nil { - return nil, err + return nil, nil, err } ph.tracked = tracked - return tracked, nil + return tracked, tracked.Close, nil } // UpdateCopyOptions adds TTY status output to the copy options. @@ -86,3 +86,51 @@ func (ph *TTYPushHandler) UpdateCopyOptions(opts *oras.CopyGraphOptions, fetcher func NewTTYAttachHandler(tty *os.File) AttachHandler { return NewTTYPushHandler(tty) } + +// TTYPullHandler handles TTY status output for pull events. +type TTYPullHandler struct { + tty *os.File + tracked track.GraphTarget +} + +// NewTTYPullHandler returns a new handler for Pull status events. +func NewTTYPullHandler(tty *os.File) PullHandler { + return &TTYPullHandler{ + tty: tty, + } +} + +// OnNodeDownloading implements PullHandler. +func (ph *TTYPullHandler) OnNodeDownloading(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeDownloaded implements PullHandler. +func (ph *TTYPullHandler) OnNodeDownloaded(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeProcessing implements PullHandler. +func (ph *TTYPullHandler) OnNodeProcessing(desc ocispec.Descriptor) error { + return nil +} + +// OnNodeRestored implements PullHandler. +func (ph *TTYPullHandler) OnNodeRestored(desc ocispec.Descriptor) error { + return ph.tracked.Prompt(desc, PullPromptRestored) +} + +// OnNodeProcessing implements PullHandler. +func (ph *TTYPullHandler) OnNodeSkipped(desc ocispec.Descriptor) error { + return ph.tracked.Prompt(desc, PullPromptSkipped) +} + +// TrackTarget returns a tracked target. +func (ph *TTYPullHandler) TrackTarget(gt oras.GraphTarget) (oras.GraphTarget, StopTrackTargetFunc, error) { + tracked, err := track.NewTarget(gt, PullPromptDownloading, PullPromptPulled, ph.tty) + if err != nil { + return nil, nil, err + } + ph.tracked = tracked + return tracked, tracked.Close, nil +} diff --git a/cmd/oras/internal/display/status/tty_test.go b/cmd/oras/internal/display/status/tty_test.go index 89183ef3f..a29e4545e 100644 --- a/cmd/oras/internal/display/status/tty_test.go +++ b/cmd/oras/internal/display/status/tty_test.go @@ -82,10 +82,15 @@ func TestTTYPushHandler_TrackTarget(t *testing.T) { ph := NewTTYPushHandler(slave) store := memory.New() // test - _, err = ph.TrackTarget(store) + _, fn, err := ph.TrackTarget(store) if err != nil { t.Error("TrackTarget() should not return an error") } + defer func() { + if err := fn(); err != nil { + t.Fatal(err) + } + }() if ttyPushHandler, ok := ph.(*TTYPushHandler); !ok { t.Errorf("TrackTarget() should return a *TTYPushHandler, got %T", ttyPushHandler) } else if ttyPushHandler.tracked.Inner() != store { @@ -101,7 +106,7 @@ func TestTTYPushHandler_UpdateCopyOptions(t *testing.T) { } defer slave.Close() ph := NewTTYPushHandler(slave) - gt, err := ph.TrackTarget(memory.New()) + gt, _, err := ph.TrackTarget(memory.New()) if err != nil { t.Errorf("TrackTarget() should not return an error: %v", err) } @@ -124,3 +129,56 @@ func TestTTYPushHandler_UpdateCopyOptions(t *testing.T) { t.Fatal(err) } } + +func Test_TTYPullHandler_TrackTarget(t *testing.T) { + src := memory.New() + t.Run("has TTY", func(t *testing.T) { + _, device, err := testutils.NewPty() + if err != nil { + t.Fatal(err) + } + defer device.Close() + ph := NewTTYPullHandler(device) + got, fn, err := ph.TrackTarget(src) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := fn(); err != nil { + t.Fatal(err) + } + }() + if got == src { + t.Fatal("GraphTarget not be modified on TTY") + } + }) + + t.Run("invalid TTY", func(t *testing.T) { + ph := NewTTYPullHandler(nil) + + if _, _, err := ph.TrackTarget(src); err == nil { + t.Fatal("expected error for no tty but got nil") + } + }) +} + +func TestTTYPullHandler_OnNodeDownloading(t *testing.T) { + ph := NewTTYPullHandler(nil) + if err := ph.OnNodeDownloading(ocispec.Descriptor{}); err != nil { + t.Error("OnNodeDownloading() should not return an error") + } +} + +func TestTTYPullHandler_OnNodeDownloaded(t *testing.T) { + ph := NewTTYPullHandler(nil) + if err := ph.OnNodeDownloaded(ocispec.Descriptor{}); err != nil { + t.Error("OnNodeDownloaded() should not return an error") + } +} + +func TestTTYPullHandler_OnNodeProcessing(t *testing.T) { + ph := NewTTYPullHandler(nil) + if err := ph.OnNodeProcessing(ocispec.Descriptor{}); err != nil { + t.Error("OnNodeProcessing() should not return an error") + } +} diff --git a/cmd/oras/internal/display/status/utils.go b/cmd/oras/internal/display/status/utils.go new file mode 100644 index 000000000..5fc3af81b --- /dev/null +++ b/cmd/oras/internal/display/status/utils.go @@ -0,0 +1,34 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status + +import ocispec "github.com/opencontainers/image-spec/specs-go/v1" + +// GenerateContentKey generates a unique key for each content descriptor, using +// its digest and name if applicable. +func GenerateContentKey(desc ocispec.Descriptor) string { + return desc.Digest.String() + desc.Annotations[ocispec.AnnotationTitle] +} + +// Prompts for pull events. +const ( + PullPromptDownloading = "Downloading" + PullPromptPulled = "Pulled " + PullPromptProcessing = "Processing " + PullPromptSkipped = "Skipped " + PullPromptRestored = "Restored " + PullPromptDownloaded = "Downloaded " +) diff --git a/cmd/oras/internal/display/utils/utils.go b/cmd/oras/internal/display/utils/utils.go new file mode 100644 index 000000000..6a88c7c02 --- /dev/null +++ b/cmd/oras/internal/display/utils/utils.go @@ -0,0 +1,33 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import v1 "github.com/opencontainers/image-spec/specs-go/v1" + +// GenerateContentKey generates a unique key for each content descriptor, using +// its digest and name if applicable. +func GenerateContentKey(desc v1.Descriptor) string { + return desc.Digest.String() + desc.Annotations[v1.AnnotationTitle] +} + +const ( + PullPromptDownloading = "Downloading" + PullPromptPulled = "Pulled " + PullPromptProcessing = "Processing " + PullPromptSkipped = "Skipped " + PullPromptRestored = "Restored " + PullPromptDownloaded = "Downloaded " +) diff --git a/cmd/oras/root/attach.go b/cmd/oras/root/attach.go index fbc7cead3..2c9c6c08c 100644 --- a/cmd/oras/root/attach.go +++ b/cmd/oras/root/attach.go @@ -142,7 +142,7 @@ func runAttach(cmd *cobra.Command, opts *attachOptions) error { } // prepare push - dst, err = displayStatus.TrackTarget(dst) + dst, stopTrack, err := displayStatus.TrackTarget(dst) if err != nil { return err } @@ -178,7 +178,7 @@ func runAttach(cmd *cobra.Command, opts *attachOptions) error { } // Attach - root, err := doPush(dst, pack, copy) + root, err := doPush(dst, stopTrack, pack, copy) if err != nil { return err } diff --git a/cmd/oras/root/pull.go b/cmd/oras/root/pull.go index 3e4402eed..6a2e10b98 100644 --- a/cmd/oras/root/pull.go +++ b/cmd/oras/root/pull.go @@ -20,9 +20,7 @@ import ( "errors" "fmt" "io" - "os" "sync" - "sync/atomic" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" @@ -30,8 +28,9 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" "oras.land/oras/cmd/oras/internal/argument" + "oras.land/oras/cmd/oras/internal/display" + "oras.land/oras/cmd/oras/internal/display/metadata" "oras.land/oras/cmd/oras/internal/display/status" - "oras.land/oras/cmd/oras/internal/display/status/track" oerrors "oras.land/oras/cmd/oras/internal/errors" "oras.land/oras/cmd/oras/internal/fileref" "oras.land/oras/cmd/oras/internal/option" @@ -43,6 +42,7 @@ type pullOptions struct { option.Common option.Platform option.Target + option.Format concurrency int KeepOldFiles bool @@ -134,7 +134,8 @@ func runPull(cmd *cobra.Command, opts *pullOptions) error { dst.AllowPathTraversalOnWrite = opts.PathTraversal dst.DisableOverwrite = opts.KeepOldFiles - desc, layerSkipped, err := doPull(ctx, src, dst, copyOptions, opts) + statusHandler, metadataHandler := display.NewPullHandler(opts.Template, opts.Path, opts.TTY, cmd.OutOrStdout(), opts.Verbose) + desc, err := doPull(ctx, src, dst, copyOptions, metadataHandler, statusHandler, opts) if err != nil { if errors.Is(err, file.ErrPathTraversalDisallowed) { err = fmt.Errorf("%s: %w", "use flag --allow-path-traversal to allow insecurely pulling files outside of working directory", err) @@ -142,57 +143,35 @@ func runPull(cmd *cobra.Command, opts *pullOptions) error { return err } - // suggest oras copy for pulling layers without annotation - outWriter := cmd.OutOrStdout() - if layerSkipped { - fmt.Fprintf(outWriter, "Skipped pulling layers without file name in %q\n", ocispec.AnnotationTitle) - fmt.Fprintf(outWriter, "Use 'oras copy %s --to-oci-layout ' to pull all layers.\n", opts.RawReference) - } else { - fmt.Fprintln(outWriter, "Pulled", opts.AnnotatedReference()) - fmt.Fprintln(outWriter, "Digest:", desc.Digest) - } - return nil + return metadataHandler.OnCompleted(&opts.Target, desc) } -func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, opts oras.CopyOptions, po *pullOptions) (ocispec.Descriptor, bool, error) { +func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, opts oras.CopyOptions, metadataHandler metadata.PullHandler, statusHandler status.PullHandler, po *pullOptions) (ocispec.Descriptor, error) { var configPath, configMediaType string var err error + if po.ManifestConfigRef != "" { configPath, configMediaType, err = fileref.Parse(po.ManifestConfigRef, "") if err != nil { - return ocispec.Descriptor{}, false, err + return ocispec.Descriptor{}, err } } - - const ( - promptDownloading = "Downloading" - promptPulled = "Pulled " - promptProcessing = "Processing " - promptSkipped = "Skipped " - promptRestored = "Restored " - promptDownloaded = "Downloaded " - ) - - dst, err = getTrackedTarget(dst, po.TTY, "Downloading", promptPulled) + dst, stopTrack, err := statusHandler.TrackTarget(dst) if err != nil { - return ocispec.Descriptor{}, false, err - } - if tracked, ok := dst.(track.GraphTarget); ok { - defer tracked.Close() + return ocispec.Descriptor{}, err } - var layerSkipped atomic.Bool + defer func() { + _ = stopTrack() + }() var printed sync.Map var getConfigOnce sync.Once opts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { statusFetcher := content.FetcherFunc(func(ctx context.Context, target ocispec.Descriptor) (fetched io.ReadCloser, fetchErr error) { - if _, ok := printed.LoadOrStore(generateContentKey(target), true); ok { + if _, ok := printed.LoadOrStore(status.GenerateContentKey(target), true); ok { return fetcher.Fetch(ctx, target) } - if po.TTY == nil { - // none TTY, print status log for first-time fetching - if err := status.PrintStatus(target, promptDownloading, po.Verbose); err != nil { - return nil, err - } + if err := statusHandler.OnNodeDownloading(target); err != nil { + return nil, err } rc, err := fetcher.Fetch(ctx, target) if err != nil { @@ -203,11 +182,7 @@ func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, rc.Close() } }() - if po.TTY == nil { - // none TTY, add logs for processing manifest - return rc, status.PrintStatus(target, promptProcessing, po.Verbose) - } - return rc, nil + return rc, statusHandler.OnNodeProcessing(target) }) nodes, subject, config, err := graph.Successors(ctx, statusFetcher, desc) @@ -240,7 +215,9 @@ func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, } if s.Annotations[ocispec.AnnotationTitle] == "" { // unnamed layers are skipped - layerSkipped.Store(true) + if err = metadataHandler.OnLayerSkipped(s); err != nil { + return nil, err + } } ss, err := content.Successors(ctx, fetcher, s) if err != nil { @@ -248,7 +225,7 @@ func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, } if len(ss) == 0 { // skip s if it is unnamed AND has no successors. - if err := printOnce(&printed, s, promptSkipped, po.Verbose, dst); err != nil { + if err := notifyOnce(&printed, s, statusHandler.OnNodeSkipped); err != nil { return nil, err } continue @@ -256,20 +233,11 @@ func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, } ret = append(ret, s) } - return ret, nil } opts.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - if _, ok := printed.LoadOrStore(generateContentKey(desc), true); ok { - return nil - } - if po.TTY == nil { - // none TTY, print status log for downloading - return status.PrintStatus(desc, promptDownloading, po.Verbose) - } - // TTY - return nil + return notifyOnce(&printed, desc, statusHandler.OnNodeDownloading) } opts.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { // restore named but deduplicated successor nodes @@ -278,54 +246,27 @@ func doPull(ctx context.Context, src oras.ReadOnlyTarget, dst oras.GraphTarget, return err } for _, s := range successors { - if _, ok := s.Annotations[ocispec.AnnotationTitle]; ok { - if err := printOnce(&printed, s, promptRestored, po.Verbose, dst); err != nil { + if name, ok := s.Annotations[ocispec.AnnotationTitle]; ok { + if err = metadataHandler.OnFilePulled(name, po.Output, s, po.Path); err != nil { + return err + } + if err = notifyOnce(&printed, s, statusHandler.OnNodeRestored); err != nil { return err } } } - name, ok := desc.Annotations[ocispec.AnnotationTitle] - if !ok { - if !po.Verbose { - return nil - } - name = desc.MediaType - } - printed.Store(generateContentKey(desc), true) - return status.Print(promptDownloaded, status.ShortDigest(desc), name) + printed.Store(status.GenerateContentKey(desc), true) + return statusHandler.OnNodeDownloaded(desc) } // Copy desc, err := oras.Copy(ctx, src, po.Reference, dst, po.Reference, opts) - return desc, layerSkipped.Load(), err -} - -// generateContentKey generates a unique key for each content descriptor, using -// its digest and name if applicable. -func generateContentKey(desc ocispec.Descriptor) string { - return desc.Digest.String() + desc.Annotations[ocispec.AnnotationTitle] -} - -func printOnce(printed *sync.Map, s ocispec.Descriptor, msg string, verbose bool, dst any) error { - if _, loaded := printed.LoadOrStore(generateContentKey(s), true); loaded { - return nil - } - if tracked, ok := dst.(track.GraphTarget); ok { - // TTY - return tracked.Prompt(s, msg) - - } - // none TTY - return status.PrintStatus(s, msg, verbose) + return desc, err } -func getTrackedTarget(gt oras.GraphTarget, tty *os.File, actionPrompt, doneprompt string) (oras.GraphTarget, error) { - if tty == nil { - return gt, nil +func notifyOnce(notified *sync.Map, s ocispec.Descriptor, notify func(ocispec.Descriptor) error) error { + if _, loaded := notified.LoadOrStore(status.GenerateContentKey(s), true); !loaded { + return notify(s) } - tracked, err := track.NewTarget(gt, actionPrompt, doneprompt, tty) - if err != nil { - return nil, err - } - return tracked, nil + return nil } diff --git a/cmd/oras/root/pull_test.go b/cmd/oras/root/pull_test.go deleted file mode 100644 index 7e325c84b..000000000 --- a/cmd/oras/root/pull_test.go +++ /dev/null @@ -1,63 +0,0 @@ -//go:build freebsd || linux || netbsd || openbsd || solaris - -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package root - -import ( - "os" - "testing" - - "oras.land/oras-go/v2/content/memory" - "oras.land/oras/cmd/oras/internal/display/status/console/testutils" -) - -func Test_getTrackedTarget(t *testing.T) { - _, device, err := testutils.NewPty() - if err != nil { - t.Fatal(err) - } - defer device.Close() - src := memory.New() - actionPrompt := "action" - donePrompt := "done" - - t.Run("no TTY", func(t *testing.T) { - got, err := getTrackedTarget(src, nil, actionPrompt, donePrompt) - if err != nil { - t.Fatal(err) - } - if got != src { - t.Fatal("GraphTarget should not be modified if no TTY") - } - }) - - t.Run("has TTY", func(t *testing.T) { - got, err := getTrackedTarget(src, device, actionPrompt, donePrompt) - if err != nil { - t.Fatal(err) - } - if got == src { - t.Fatal("GraphTarget not be modified on TTY") - } - }) - - t.Run("invalid TTY", func(t *testing.T) { - if _, err := getTrackedTarget(src, os.Stdin, actionPrompt, donePrompt); err == nil { - t.Fatal("expected error for no tty but got nil") - } - }) -} diff --git a/cmd/oras/root/push.go b/cmd/oras/root/push.go index fefd62d4a..3710f0b41 100644 --- a/cmd/oras/root/push.go +++ b/cmd/oras/root/push.go @@ -184,7 +184,7 @@ func runPush(cmd *cobra.Command, opts *pushOptions) error { if err != nil { return err } - dst, err = displayStatus.TrackTarget(dst) + dst, stopTrack, err := displayStatus.TrackTarget(dst) if err != nil { return err } @@ -206,7 +206,7 @@ func runPush(cmd *cobra.Command, opts *pushOptions) error { } // Push - root, err := doPush(dst, pack, copy) + root, err := doPush(dst, stopTrack, pack, copy) if err != nil { return err } @@ -240,10 +240,10 @@ func runPush(cmd *cobra.Command, opts *pushOptions) error { return opts.ExportManifest(ctx, memoryStore, root) } -func doPush(dst oras.Target, pack packFunc, copy copyFunc) (ocispec.Descriptor, error) { - if tracked, ok := dst.(track.GraphTarget); ok { - defer tracked.Close() - } +func doPush(dst oras.Target, stopTrack status.StopTrackTargetFunc, pack packFunc, copy copyFunc) (ocispec.Descriptor, error) { + defer func() { + _ = stopTrack() + }() // Push return pushArtifact(dst, pack, copy) } diff --git a/test/e2e/suite/command/pull.go b/test/e2e/suite/command/pull.go index ebd5cf61b..230671648 100644 --- a/test/e2e/suite/command/pull.go +++ b/test/e2e/suite/command/pull.go @@ -16,6 +16,7 @@ limitations under the License. package command import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -67,12 +68,26 @@ var _ = Describe("ORAS beginners:", func() { gomega.Expect(out).ShouldNot(gbytes.Say(hintMsg(ref))) }) + It("should not show hint for json output", func() { + tempDir := PrepareTempFiles() + ref := RegistryRef(ZOTHost, ArtifactRepo, unnamed.Tag) + out := ORAS("pull", ref, "--format", "json").WithWorkDir(tempDir).Exec().Out + gomega.Expect(out).ShouldNot(gbytes.Say(hintMsg(ref))) + }) + + It("should not show hint for go template output", func() { + tempDir := PrepareTempFiles() + ref := RegistryRef(ZOTHost, ArtifactRepo, unnamed.Tag) + out := ORAS("pull", ref, "--format", "{{.}}").WithWorkDir(tempDir).Exec().Out + gomega.Expect(out).ShouldNot(gbytes.Say(hintMsg(ref))) + }) + It("should fail and show detailed error description if no argument provided", func() { err := ORAS("pull").ExpectFailure().Exec().Err - gomega.Expect(err).Should(gbytes.Say("Error")) - gomega.Expect(err).Should(gbytes.Say("\nUsage: oras pull")) - gomega.Expect(err).Should(gbytes.Say("\n")) - gomega.Expect(err).Should(gbytes.Say(`Run "oras pull -h"`)) + Expect(err).Should(gbytes.Say("Error")) + Expect(err).Should(gbytes.Say("\nUsage: oras pull")) + Expect(err).Should(gbytes.Say("\n")) + Expect(err).Should(gbytes.Say(`Run "oras pull -h"`)) }) It("should fail if password is wrong with registry error prefix", func() { @@ -90,6 +105,26 @@ var _ = Describe("ORAS beginners:", func() { ORAS("pull", Flags.Layout, LayoutRef(root, InvalidTag)). MatchErrKeyWords("Error: ").ExpectFailure().Exec() }) + + It("should fail if manifest config reference is invalid", func() { + root := PrepareTempOCI(ImageRepo) + ORAS("pull", Flags.Layout, LayoutRef(root, foobar.Tag), "--config", ":"). + MatchErrKeyWords("Error: ").ExpectFailure().Exec() + }) + + It("should fail to pull layers outside of working directory", func() { + // prepare + pushRoot := GinkgoT().TempDir() + Expect(CopyTestFiles(pushRoot)).ShouldNot(HaveOccurred()) + tag := "pushed" + ORAS("push", Flags.Layout, LayoutRef(pushRoot, tag), filepath.Join(pushRoot, foobar.FileConfigName), "--disable-path-validation").WithWorkDir(pushRoot).Exec() + // test + pullRoot := GinkgoT().TempDir() + ORAS("pull", Flags.Layout, LayoutRef(pushRoot, tag), "-o", "pulled"). + WithDescription(pullRoot). + ExpectFailure(). + MatchErrKeyWords("Error: ", "--allow-path-traversal").Exec() + }) }) }) @@ -153,6 +188,29 @@ var _ = Describe("OCI spec 1.1 registry users:", func() { WithWorkDir(tempDir).Exec() }) + It("should pull and output downloaded file paths", func() { + tempDir := GinkgoT().TempDir() + var paths []string + for _, p := range foobar.ImageLayerNames { + paths = append(paths, filepath.Join(tempDir, p)) + } + ORAS("pull", RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag), "--format", "{{range .Files}}{{println .Path}}{{end}}"). + WithWorkDir(tempDir).MatchKeyWords(paths...).Exec() + }) + + It("should pull and output path in json", func() { + tempDir := GinkgoT().TempDir() + var paths []string + for _, p := range foobar.ImageLayerNames { + paths = append(paths, filepath.Join(tempDir, p)) + } + out := ORAS("pull", RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag), "--format", "json"). + WithWorkDir(tempDir).MatchKeyWords(paths...).Exec().Out.Contents() + var parsed struct{} + err := json.Unmarshal(out, &parsed) + Expect(err).ShouldNot(HaveOccurred()) + }) + It("should pull specific platform", func() { ORAS("pull", RegistryRef(ZOTHost, ImageRepo, "multi"), "--platform", "linux/amd64", "-v", "-o", GinkgoT().TempDir()). MatchStatus(multi_arch.LinuxAMD64StateKeys, true, len(multi_arch.LinuxAMD64StateKeys)).Exec()