From a0efecf1ec329554a3d2630b652dc00167f080ff Mon Sep 17 00:00:00 2001 From: Anthony Nandaa Date: Mon, 22 Apr 2024 11:52:42 +0300 Subject: [PATCH] vendor: github.com/tonistiigi/fsutil @ 497d33b Full diff: https://github.com/tonistiigi/fsutil/compare/7525a1af2bb5..497d33b Summary changes: - https://github.com/tonistiigi/fsutil/pull/195 receive: ensure callback errors are propagated - https://github.com/tonistiigi/fsutil/pull/196 Fix file transfers from windows to linux fixes #4741 - https://github.com/tonistiigi/fsutil/pull/197 recv: translate linkname to wire format Signed-off-by: Anthony Nandaa (cherry picked from commit 64ea9da1c4c948558fc4abe377c042285e6bbf7b) --- go.mod | 2 +- go.sum | 4 +- .../tonistiigi/fsutil/diff_containerd.go | 2 +- .../tonistiigi/fsutil/diskwriter.go | 20 +++++---- .../github.com/tonistiigi/fsutil/receive.go | 45 ++++++++++++++++++- vendor/github.com/tonistiigi/fsutil/send.go | 1 + vendor/modules.txt | 2 +- 7 files changed, 61 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 5d3bef888f1d..84bd13c9efd7 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spdx/tools-golang v0.5.3 github.com/stretchr/testify v1.8.4 - github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 + github.com/tonistiigi/fsutil v0.0.0-20240418180507-497d33b008ef github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 github.com/tonistiigi/go-archvariant v1.0.0 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea diff --git a/go.sum b/go.sum index 20ef32d21f12..2285937ff6d3 100644 --- a/go.sum +++ b/go.sum @@ -405,8 +405,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 h1:oZS8KCqAg62sxJkEq/Ppzqrb6EooqzWtL8Oaex7bc5c= -github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= +github.com/tonistiigi/fsutil v0.0.0-20240418180507-497d33b008ef h1:1rshiFn5ka7/H9oGYXvRnV1BzhtWls2WSQZDrNwVsCA= +github.com/tonistiigi/fsutil v0.0.0-20240418180507-497d33b008ef/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598 h1:DA/NDC0YbMdnfcOSUzAnbUZE6dSM54d+0hrBqG+bOfs= github.com/tonistiigi/go-actions-cache v0.0.0-20240227172821-a0b64f338598/go.mod h1:anhKd3mnC1shAbQj1Q4IJ+w6xqezxnyDYlx/yKa7IXM= github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= diff --git a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go index 84fdc89dc5bf..86d64602737b 100644 --- a/vendor/github.com/tonistiigi/fsutil/diff_containerd.go +++ b/vendor/github.com/tonistiigi/fsutil/diff_containerd.go @@ -111,7 +111,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil if filter != nil { filter(f2.path, &statCopy) } - f2copy = ¤tPath{path: filepath.FromSlash(f2.path), stat: &statCopy} + f2copy = ¤tPath{path: f2.path, stat: &statCopy} } k, p := pathChange(f1, f2copy) switch k { diff --git a/vendor/github.com/tonistiigi/fsutil/diskwriter.go b/vendor/github.com/tonistiigi/fsutil/diskwriter.go index 10b60851381b..a62c6b0c917b 100644 --- a/vendor/github.com/tonistiigi/fsutil/diskwriter.go +++ b/vendor/github.com/tonistiigi/fsutil/diskwriter.go @@ -37,6 +37,7 @@ type DiskWriter struct { ctx context.Context cancel func() eg *errgroup.Group + egCtx context.Context filter FilterFunc dirModTimes map[string]int64 } @@ -50,13 +51,14 @@ func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWr } ctx, cancel := context.WithCancel(ctx) - eg, ctx := errgroup.WithContext(ctx) + eg, egCtx := errgroup.WithContext(ctx) return &DiskWriter{ opt: opt, dest: dest, eg: eg, ctx: ctx, + egCtx: egCtx, cancel: cancel, filter: opt.Filter, dirModTimes: map[string]int64{}, @@ -98,7 +100,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } }() - destPath := filepath.Join(dw.dest, filepath.FromSlash(p)) + destPath := filepath.Join(dw.dest, p) if kind == ChangeKindDelete { if dw.filter != nil { @@ -183,12 +185,12 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } default: isRegularFile = true - file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) //todo: windows + file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) if err != nil { return errors.Wrapf(err, "failed to create %s", newPath) } if dw.opt.SyncDataCb != nil { - if err := dw.processChange(ChangeKindAdd, p, fi, file); err != nil { + if err := dw.processChange(dw.ctx, ChangeKindAdd, p, fi, file); err != nil { file.Close() return err } @@ -219,7 +221,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er dw.requestAsyncFileData(p, destPath, fi, &statCopy) } } else { - return dw.processChange(kind, p, fi, nil) + return dw.processChange(dw.ctx, kind, p, fi, nil) } return nil @@ -228,7 +230,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *types.Stat) { // todo: limit worker threads dw.eg.Go(func() error { - if err := dw.processChange(ChangeKindAdd, p, fi, &lazyFileWriter{ + if err := dw.processChange(dw.egCtx, ChangeKindAdd, p, fi, &lazyFileWriter{ dest: dest, }); err != nil { return err @@ -237,7 +239,7 @@ func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *t }) } -func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error { +func (dw *DiskWriter) processChange(ctx context.Context, kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error { origw := w var hw *hashedWriter if dw.opt.NotifyCb != nil { @@ -252,7 +254,7 @@ func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w if fn == nil && dw.opt.AsyncDataCb != nil { fn = dw.opt.AsyncDataCb } - if err := fn(dw.ctx, p, w); err != nil { + if err := fn(ctx, p, w); err != nil { return err } } else { @@ -313,7 +315,7 @@ type lazyFileWriter struct { func (lfw *lazyFileWriter) Write(dt []byte) (int, error) { if lfw.f == nil { - file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows + file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) if os.IsPermission(err) { // retry after chmod fi, er := os.Stat(lfw.dest) diff --git a/vendor/github.com/tonistiigi/fsutil/receive.go b/vendor/github.com/tonistiigi/fsutil/receive.go index 209d1d2fafa2..6a82d205b19f 100644 --- a/vendor/github.com/tonistiigi/fsutil/receive.go +++ b/vendor/github.com/tonistiigi/fsutil/receive.go @@ -1,10 +1,42 @@ +// send.go and receive.go describe the fsutil file-transfer protocol, which +// allows transferring file trees across a network connection. +// +// The protocol operates as follows: +// - The client (the receiver) connects to the server (the sender). +// - The sender walks the target tree lexicographically and sends a series of +// STAT packets that describe each file (an empty stat indicates EOF). +// - The receiver sends a REQ packet for each file it requires the contents for, +// using the ID for the file (determined as its index in the STAT sequence). +// - The sender sends a DATA packet with byte arrays for the contents of the +// file, associated with an ID (an empty array indicates EOF). +// - Once the receiver has received all files it wants, it sends a FIN packet, +// and the file transfer is complete. +// If an error is encountered on either side, an ERR packet is sent containing +// a human-readable error. +// +// All paths transferred over the protocol are normalized to unix-style paths, +// regardless of which platforms are present on either side. These path +// conversions are performed right before sending a STAT packet (for the +// sender) or right after receiving the corresponding STAT packet (for the +// receiver); this abstraction doesn't leak into the rest of fsutil, which +// operates on native platform-specific paths. +// +// Note that in the case of cross-platform file transfers, the transfer is +// best-effort. Some filenames that are valid on a unix sender would not be +// valid on a windows receiver, so these paths are rejected as they are +// received. Additionally, file metadata, like user/group owners and xattrs do +// not have an exact correspondence on windows, and so would be discarded by +// a windows receiver. + package fsutil import ( "context" "io" "os" + "path/filepath" "sync" + "syscall" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" @@ -184,13 +216,24 @@ func (r *receiver) run(ctx context.Context) error { } break } + + // normalize unix wire-specific paths to platform-specific paths + path := filepath.FromSlash(p.Stat.Path) + if filepath.ToSlash(path) != p.Stat.Path { + // e.g. a linux path foo/bar\baz cannot be represented on windows + return errors.WithStack(&os.PathError{Path: p.Stat.Path, Err: syscall.EINVAL, Op: "unrepresentable path"}) + } + p.Stat.Path = path + p.Stat.Linkname = filepath.FromSlash(p.Stat.Linkname) + if fileCanRequestData(os.FileMode(p.Stat.Mode)) { r.mu.Lock() r.files[p.Stat.Path] = i r.mu.Unlock() } i++ - cp := ¤tPath{path: p.Stat.Path, stat: p.Stat} + + cp := ¤tPath{path: path, stat: p.Stat} if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } diff --git a/vendor/github.com/tonistiigi/fsutil/send.go b/vendor/github.com/tonistiigi/fsutil/send.go index ba97ef7ad0ed..43bf1717609a 100644 --- a/vendor/github.com/tonistiigi/fsutil/send.go +++ b/vendor/github.com/tonistiigi/fsutil/send.go @@ -161,6 +161,7 @@ func (s *sender) walk(ctx context.Context) error { return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) } stat.Path = filepath.ToSlash(stat.Path) + stat.Linkname = filepath.ToSlash(stat.Linkname) p := &types.Packet{ Type: types.PACKET_STAT, Stat: stat, diff --git a/vendor/modules.txt b/vendor/modules.txt index 5f390917fd7c..b39a813546f6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -765,7 +765,7 @@ github.com/spdx/tools-golang/spdx/v2/v2_3 ## explicit; go 1.20 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 +# github.com/tonistiigi/fsutil v0.0.0-20240418180507-497d33b008ef ## explicit; go 1.20 github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/copy