Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use fixed fileutils matching functions #2319

Merged
merged 1 commit into from
Aug 23, 2021
Merged

Use fixed fileutils matching functions #2319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 56 additions & 20 deletions cache/contenthash/checksum.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,11 @@ type Hashed interface {
Digest() digest.Digest
}

type IncludedPath struct {
Path string
Record *CacheRecord
type includedPath struct {
path string
record *CacheRecord
matchedIncludePattern bool
matchedExcludePattern bool
}

type cacheManager struct {
Expand Down Expand Up @@ -404,30 +406,30 @@ func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable,

if opts.FollowLinks {
for i, w := range includedPaths {
if w.Record.Type == CacheRecordTypeSymlink {
dgst, err := cc.checksumFollow(ctx, m, w.Path, opts.FollowLinks)
if w.record.Type == CacheRecordTypeSymlink {
dgst, err := cc.checksumFollow(ctx, m, w.path, opts.FollowLinks)
if err != nil {
return "", err
}
includedPaths[i].Record = &CacheRecord{Digest: dgst}
includedPaths[i].record = &CacheRecord{Digest: dgst}
}
}
}
if len(includedPaths) == 0 {
return digest.FromBytes([]byte{}), nil
}

if len(includedPaths) == 1 && path.Base(p) == path.Base(includedPaths[0].Path) {
return includedPaths[0].Record.Digest, nil
if len(includedPaths) == 1 && path.Base(p) == path.Base(includedPaths[0].path) {
return includedPaths[0].record.Digest, nil
}

digester := digest.Canonical.Digester()
for i, w := range includedPaths {
if i != 0 {
digester.Hash().Write([]byte{0})
}
digester.Hash().Write([]byte(path.Base(w.Path)))
digester.Hash().Write([]byte(w.Record.Digest))
digester.Hash().Write([]byte(path.Base(w.path)))
digester.Hash().Write([]byte(w.record.Digest))
}
return digester.Digest(), nil
}
Expand Down Expand Up @@ -456,7 +458,7 @@ func (cc *cacheContext) checksumFollow(ctx context.Context, m *mount, p string,
}
}

func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, opts ChecksumOpts) ([]*IncludedPath, error) {
func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, opts ChecksumOpts) ([]*includedPath, error) {
cc.mu.Lock()
defer cc.mu.Unlock()

Expand Down Expand Up @@ -501,7 +503,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
}
}

includedPaths := make([]*IncludedPath, 0, 2)
includedPaths := make([]*includedPath, 0, 2)

txn := cc.tree.Txn()
root = txn.Root()
Expand All @@ -524,7 +526,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
}

var (
parentDirHeaders []*IncludedPath
parentDirHeaders []*includedPath
lastMatchedDir string
)

Expand All @@ -533,11 +535,15 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o

for len(parentDirHeaders) != 0 {
lastParentDir := parentDirHeaders[len(parentDirHeaders)-1]
if strings.HasPrefix(fn, lastParentDir.Path+"/") {
if strings.HasPrefix(fn, lastParentDir.path+"/") {
break
}
parentDirHeaders = parentDirHeaders[:len(parentDirHeaders)-1]
}
var parentDir *includedPath
if len(parentDirHeaders) != 0 {
parentDir = parentDirHeaders[len(parentDirHeaders)-1]
}

dirHeader := false
if len(k) > 0 && k[len(k)-1] == byte(0) {
Expand All @@ -550,6 +556,7 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
}
}

maybeIncludedPath := &includedPath{path: fn}
var shouldInclude bool
if opts.Wildcard {
if p != "" && (lastMatchedDir == "" || !strings.HasPrefix(fn, lastMatchedDir+"/")) {
Expand All @@ -564,7 +571,13 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
lastMatchedDir = fn
}

shouldInclude, err = shouldIncludePath(strings.TrimSuffix(strings.TrimPrefix(fn+"/", lastMatchedDir+"/"), "/"), includePatternMatcher, excludePatternMatcher)
shouldInclude, err = shouldIncludePath(
strings.TrimSuffix(strings.TrimPrefix(fn+"/", lastMatchedDir+"/"), "/"),
includePatternMatcher,
excludePatternMatcher,
maybeIncludedPath,
parentDir,
)
if err != nil {
return nil, err
}
Expand All @@ -574,7 +587,13 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
continue
}

shouldInclude, err = shouldIncludePath(strings.TrimSuffix(strings.TrimPrefix(fn+"/", p+"/"), "/"), includePatternMatcher, excludePatternMatcher)
shouldInclude, err = shouldIncludePath(
strings.TrimSuffix(strings.TrimPrefix(fn+"/", p+"/"), "/"),
includePatternMatcher,
excludePatternMatcher,
maybeIncludedPath,
parentDir,
)
if err != nil {
return nil, err
}
Expand All @@ -599,17 +618,18 @@ func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, o
// dir.
shouldInclude = false
}
maybeIncludedPath.record = cr

if !shouldInclude {
if cr.Type == CacheRecordTypeDirHeader {
// We keep track of non-included parent dir headers in case an
// include pattern matches a file inside one of these dirs.
parentDirHeaders = append(parentDirHeaders, &IncludedPath{Path: fn, Record: cr})
parentDirHeaders = append(parentDirHeaders, maybeIncludedPath)
}
} else {
includedPaths = append(includedPaths, parentDirHeaders...)
parentDirHeaders = nil
includedPaths = append(includedPaths, &IncludedPath{Path: fn, Record: cr})
includedPaths = append(includedPaths, maybeIncludedPath)
}
k, _, kOk = iter.Next()
}
Expand All @@ -624,22 +644,38 @@ func shouldIncludePath(
candidate string,
includePatternMatcher *fileutils.PatternMatcher,
excludePatternMatcher *fileutils.PatternMatcher,
maybeIncludedPath *includedPath,
parentDir *includedPath,
) (bool, error) {
var (
m bool
err error
)
if includePatternMatcher != nil {
m, err := includePatternMatcher.Matches(candidate)
if parentDir != nil {
m, err = includePatternMatcher.MatchesUsingParentResult(candidate, parentDir.matchedIncludePattern)
} else {
m, err = includePatternMatcher.MatchesOrParentMatches(candidate)
}
if err != nil {
return false, errors.Wrap(err, "failed to match includepatterns")
}
maybeIncludedPath.matchedIncludePattern = m
if !m {
return false, nil
}
}

if excludePatternMatcher != nil {
m, err := excludePatternMatcher.Matches(candidate)
if parentDir != nil {
m, err = excludePatternMatcher.MatchesUsingParentResult(candidate, parentDir.matchedExcludePattern)
} else {
m, err = excludePatternMatcher.MatchesOrParentMatches(candidate)
}
if err != nil {
return false, errors.Wrap(err, "failed to match excludepatterns")
}
maybeIncludedPath.matchedExcludePattern = m
if m {
return false, nil
}
Expand Down
10 changes: 10 additions & 0 deletions cache/contenthash/checksum_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,16 @@ func TestChecksumIncludeDoubleStar(t *testing.T) {
require.NoError(t, err)
require.Equal(t, dgstDoubleStar, dgst)

// **/... pattern (https://github.com/moby/moby/issues/41433)
dgst, err = cc.Checksum(context.TODO(), ref, "prefix/a", ChecksumOpts{IncludePatterns: []string{"**/foo", "**/report"}}, nil)
require.NoError(t, err)
require.Equal(t, dgstDoubleStar, dgst)

// Same, with Wildcard = true
dgst, err = cc.Checksum(context.TODO(), ref, "prefix/a", ChecksumOpts{IncludePatterns: []string{"**/foo", "**/report"}, Wildcard: true}, nil)
require.NoError(t, err)
require.Equal(t, dgstDoubleStar, dgst)

}

func TestChecksumIncludeSymlink(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ require (
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf
github.com/tonistiigi/fsutil v0.0.0-20210818161904-4442383b5028
github.com/tonistiigi/go-actions-cache v0.0.0-20210714033416-b93d7f1b2e70
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f
Expand Down Expand Up @@ -118,7 +118,7 @@ require (
)

replace (
github.com/docker/docker => github.com/docker/docker v20.10.3-0.20210609100121-ef4d47340142+incompatible
github.com/docker/docker => github.com/docker/docker v20.10.3-0.20210817025855-ba2adeebdb8d+incompatible
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.0.0-20210714055410-d010b05b4939
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace => github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/net/http/httptrace/otelhttptrace v0.0.0-20210714055410-d010b05b4939
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/net/http/otelhttp v0.0.0-20210714055410-d010b05b4939
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,8 +385,8 @@ github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompa
github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v20.10.3-0.20210609100121-ef4d47340142+incompatible h1:CKSQs5KedtaAdusBPAJQS7cN1PibFX4RuThbwHgJrJE=
github.com/docker/docker v20.10.3-0.20210609100121-ef4d47340142+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v20.10.3-0.20210817025855-ba2adeebdb8d+incompatible h1:tSd7TeZCH0j9m4P14bfe/eO1KYawrt3DztHI8gZAmLM=
github.com/docker/docker v20.10.3-0.20210817025855-ba2adeebdb8d+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ=
github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
Expand Down Expand Up @@ -1078,8 +1078,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo=
github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf h1:L0ixhsTk9j+dVnIvF6aiVCxPiaFvwTOyJxqimPq44p8=
github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf/go.mod h1:lJAxK//iyZ3yGbQswdrPTxugZIDM7sd4bEsD0x3XMHk=
github.com/tonistiigi/fsutil v0.0.0-20210818161904-4442383b5028 h1:uEkkUFMCPtzz1HVOa42u15OHems1ugiRt172tSRTWSk=
github.com/tonistiigi/fsutil v0.0.0-20210818161904-4442383b5028/go.mod h1:E6osHKls9ix67jofYQ61RQKwlJhqJOZM2hintp+49iI=
github.com/tonistiigi/go-actions-cache v0.0.0-20210714033416-b93d7f1b2e70 h1:+ZlFs3Tl5qYZJvX2PxfZxGlVXz847LsOJGyNVU5pCHo=
github.com/tonistiigi/go-actions-cache v0.0.0-20210714033416-b93d7f1b2e70/go.mod h1:dNS+PPTqGnSl80x3wEyWWCHeON5xiBGtcM0uD6CgHNU=
github.com/tonistiigi/opentelemetry-go-contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.0.0-20210714055410-d010b05b4939 h1:s6wDMZYNyWt8KvkjhrMpOthFPgI3JB8ipJS+eCV/psg=
Expand Down
2 changes: 1 addition & 1 deletion vendor/github.com/docker/docker/api/common.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

50 changes: 44 additions & 6 deletions vendor/github.com/docker/docker/api/swagger.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 12 additions & 2 deletions vendor/github.com/docker/docker/api/types/client.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading