From 86663cedcc82fc16699ba6d2e1fac301af21f8c0 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 30 Nov 2022 13:24:35 -0500 Subject: [PATCH 1/8] fix: exclude packages that have overlap by file ownership relationship as child Signed-off-by: Keith Zantow --- grype/pkg/package.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/grype/pkg/package.go b/grype/pkg/package.go index 96a39999c3c..3713f103c2b 100644 --- a/grype/pkg/package.go +++ b/grype/pkg/package.go @@ -59,9 +59,16 @@ func New(p pkg.Package) Package { } func FromCatalog(catalog *pkg.Catalog, config ProviderConfig) []Package { + overlapRelationships := pkg.RelationshipsByFileOwnership(catalog) result := make([]Package, 0, catalog.PackageCount()) missingCPEs := false +nextPackage: for _, p := range catalog.Sorted() { + for _, overlapRelationship := range overlapRelationships { + if overlapRelationship.To.ID() == p.ID() { + continue nextPackage + } + } if len(p.CPEs) == 0 { // For SPDX (or any format, really) we may have no CPEs if config.GenerateMissingCPEs { From 100904b198b8b76df1d446c9343c154e932ebaaa Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 30 Nov 2022 15:23:39 -0500 Subject: [PATCH 2/8] chore: refactor overlap exclusion, add tests Signed-off-by: Keith Zantow --- grype/pkg/package.go | 28 +++++++++---- grype/pkg/package_test.go | 71 +++++++++++++++++++++++++++++++++ grype/pkg/syft_provider.go | 4 +- grype/pkg/syft_sbom_provider.go | 5 ++- 4 files changed, 99 insertions(+), 9 deletions(-) diff --git a/grype/pkg/package.go b/grype/pkg/package.go index 3713f103c2b..fd811f4041e 100644 --- a/grype/pkg/package.go +++ b/grype/pkg/package.go @@ -2,6 +2,7 @@ package pkg import ( "fmt" + "github.com/anchore/syft/syft/artifact" "regexp" "github.com/anchore/grype/internal" @@ -58,17 +59,30 @@ func New(p pkg.Package) Package { } } +func RemovePackagesByOverlap(catalog *pkg.Catalog, relationships []artifact.Relationship) *pkg.Catalog { + byOverlap := map[artifact.ID]artifact.Identifiable{} + for _, r := range relationships { + if r.Type == artifact.OwnershipByFileOverlapRelationship { + byOverlap[r.To.ID()] = r.To + } + } + + out := pkg.NewCatalog() + for p := range catalog.Enumerate() { + if _, ok := byOverlap[p.ID()]; ok { + continue + } + out.Add(p) + } + + return out +} + func FromCatalog(catalog *pkg.Catalog, config ProviderConfig) []Package { - overlapRelationships := pkg.RelationshipsByFileOwnership(catalog) result := make([]Package, 0, catalog.PackageCount()) missingCPEs := false -nextPackage: + for _, p := range catalog.Sorted() { - for _, overlapRelationship := range overlapRelationships { - if overlapRelationship.To.ID() == p.ID() { - continue nextPackage - } - } if len(p.CPEs) == 0 { // For SPDX (or any format, really) we may have no CPEs if config.GenerateMissingCPEs { diff --git a/grype/pkg/package_test.go b/grype/pkg/package_test.go index 329a71aaba6..5f84ddf1a54 100644 --- a/grype/pkg/package_test.go +++ b/grype/pkg/package_test.go @@ -1,6 +1,7 @@ package pkg import ( + "github.com/anchore/syft/syft/artifact" "testing" "github.com/scylladb/go-set" @@ -486,3 +487,73 @@ func Test_getNameAndELVersion(t *testing.T) { func intRef(i int) *int { return &i } + +func Test_RemovePackagesByOverlap(t *testing.T) { + tests := []struct { + name string + sbom catalogRelationships + expectedPackages []string + }{ + { + name: "includes all packages without overlap", + sbom: catalogWithOverlaps([]string{"go"}, []string{}), + expectedPackages: []string{"go"}, + }, + { + name: "excludes single package by overlap", + sbom: catalogWithOverlaps([]string{"go", "node"}, []string{"node"}), + expectedPackages: []string{"go"}, + }, + { + name: "excludes multiple package by overlap", + sbom: catalogWithOverlaps([]string{"go", "node", "python", "george"}, []string{"node", "george"}), + expectedPackages: []string{"go", "python"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + catalog := RemovePackagesByOverlap(test.sbom.catalog, test.sbom.relationships) + pkgs := FromCatalog(catalog, ProviderConfig{}) + var pkgNames []string + for _, p := range pkgs { + pkgNames = append(pkgNames, p.Name) + } + assert.EqualValues(t, test.expectedPackages, pkgNames) + }) + } +} + +type catalogRelationships struct { + catalog *syftPkg.Catalog + relationships []artifact.Relationship +} + +func catalogWithOverlaps(packages []string, overlaps []string) catalogRelationships { + var pkgs []syftPkg.Package + var relationships []artifact.Relationship + + for _, name := range packages { + p := syftPkg.Package{ + Name: name, + } + p.SetID() + + for _, overlap := range overlaps { + if overlap == name { + relationships = append(relationships, artifact.Relationship{ + From: p, + To: p, + Type: artifact.OwnershipByFileOverlapRelationship, + }) + } + } + + pkgs = append(pkgs, p) + } + catalog := syftPkg.NewCatalog(pkgs...) + + return catalogRelationships{ + catalog: catalog, + relationships: relationships, + } +} diff --git a/grype/pkg/syft_provider.go b/grype/pkg/syft_provider.go index e61fa09926c..500c0c111cf 100644 --- a/grype/pkg/syft_provider.go +++ b/grype/pkg/syft_provider.go @@ -21,11 +21,13 @@ func syftProvider(userInput string, config ProviderConfig) ([]Package, Context, } defer cleanup() - catalog, _, theDistro, err := syft.CatalogPackages(src, config.CatalogingOptions) + catalog, relationships, theDistro, err := syft.CatalogPackages(src, config.CatalogingOptions) if err != nil { return nil, Context{}, err } + catalog = RemovePackagesByOverlap(catalog, relationships) + return FromCatalog(catalog, config), Context{ Source: &src.Metadata, Distro: theDistro, diff --git a/grype/pkg/syft_sbom_provider.go b/grype/pkg/syft_sbom_provider.go index a9135884378..c4b757729f7 100644 --- a/grype/pkg/syft_sbom_provider.go +++ b/grype/pkg/syft_sbom_provider.go @@ -41,7 +41,10 @@ func syftSBOMProvider(userInput string, config ProviderConfig) ([]Package, Conte return nil, Context{}, err } - return FromCatalog(s.Artifacts.PackageCatalog, config), Context{ + catalog := s.Artifacts.PackageCatalog + catalog = RemovePackagesByOverlap(catalog, s.Relationships) + + return FromCatalog(catalog, config), Context{ Source: &s.Source, Distro: s.Artifacts.LinuxDistribution, }, nil From d20b5410e05e723f32593904d013ce95aa04ce84 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Wed, 30 Nov 2022 15:25:54 -0500 Subject: [PATCH 3/8] chore: lint fix Signed-off-by: Keith Zantow --- grype/pkg/package.go | 2 +- grype/pkg/package_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grype/pkg/package.go b/grype/pkg/package.go index fd811f4041e..fde429d1f86 100644 --- a/grype/pkg/package.go +++ b/grype/pkg/package.go @@ -2,11 +2,11 @@ package pkg import ( "fmt" - "github.com/anchore/syft/syft/artifact" "regexp" "github.com/anchore/grype/internal" "github.com/anchore/grype/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" "github.com/anchore/syft/syft/source" diff --git a/grype/pkg/package_test.go b/grype/pkg/package_test.go index 5f84ddf1a54..9a78f580be4 100644 --- a/grype/pkg/package_test.go +++ b/grype/pkg/package_test.go @@ -1,13 +1,13 @@ package pkg import ( - "github.com/anchore/syft/syft/artifact" "testing" "github.com/scylladb/go-set" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" syftFile "github.com/anchore/syft/syft/file" syftPkg "github.com/anchore/syft/syft/pkg" From 59f0a091d7b1a3fce976f08d665acf170bd8e09b Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 8 Dec 2022 11:54:55 -0500 Subject: [PATCH 4/8] fix: tweak binary package overlap removal Signed-off-by: Keith Zantow --- grype/pkg/package.go | 4 ++-- grype/pkg/package_test.go | 2 +- grype/pkg/syft_provider.go | 2 +- grype/pkg/syft_sbom_provider.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/grype/pkg/package.go b/grype/pkg/package.go index fde429d1f86..3e694976bd0 100644 --- a/grype/pkg/package.go +++ b/grype/pkg/package.go @@ -59,7 +59,7 @@ func New(p pkg.Package) Package { } } -func RemovePackagesByOverlap(catalog *pkg.Catalog, relationships []artifact.Relationship) *pkg.Catalog { +func RemoveBinaryPackagesByOverlap(catalog *pkg.Catalog, relationships []artifact.Relationship) *pkg.Catalog { byOverlap := map[artifact.ID]artifact.Identifiable{} for _, r := range relationships { if r.Type == artifact.OwnershipByFileOverlapRelationship { @@ -69,7 +69,7 @@ func RemovePackagesByOverlap(catalog *pkg.Catalog, relationships []artifact.Rela out := pkg.NewCatalog() for p := range catalog.Enumerate() { - if _, ok := byOverlap[p.ID()]; ok { + if _, ok := byOverlap[p.ID()]; p.Type == pkg.BinaryPkg && ok { continue } out.Add(p) diff --git a/grype/pkg/package_test.go b/grype/pkg/package_test.go index 9a78f580be4..775b174b174 100644 --- a/grype/pkg/package_test.go +++ b/grype/pkg/package_test.go @@ -512,7 +512,7 @@ func Test_RemovePackagesByOverlap(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - catalog := RemovePackagesByOverlap(test.sbom.catalog, test.sbom.relationships) + catalog := RemoveBinaryPackagesByOverlap(test.sbom.catalog, test.sbom.relationships) pkgs := FromCatalog(catalog, ProviderConfig{}) var pkgNames []string for _, p := range pkgs { diff --git a/grype/pkg/syft_provider.go b/grype/pkg/syft_provider.go index 500c0c111cf..f2978378bef 100644 --- a/grype/pkg/syft_provider.go +++ b/grype/pkg/syft_provider.go @@ -26,7 +26,7 @@ func syftProvider(userInput string, config ProviderConfig) ([]Package, Context, return nil, Context{}, err } - catalog = RemovePackagesByOverlap(catalog, relationships) + catalog = RemoveBinaryPackagesByOverlap(catalog, relationships) return FromCatalog(catalog, config), Context{ Source: &src.Metadata, diff --git a/grype/pkg/syft_sbom_provider.go b/grype/pkg/syft_sbom_provider.go index c4b757729f7..46fa5baef14 100644 --- a/grype/pkg/syft_sbom_provider.go +++ b/grype/pkg/syft_sbom_provider.go @@ -42,7 +42,7 @@ func syftSBOMProvider(userInput string, config ProviderConfig) ([]Package, Conte } catalog := s.Artifacts.PackageCatalog - catalog = RemovePackagesByOverlap(catalog, s.Relationships) + catalog = RemoveBinaryPackagesByOverlap(catalog, s.Relationships) return FromCatalog(catalog, config), Context{ Source: &s.Source, From 5df0171f67854b26adc73d8f8938e5e1df5bdd88 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 8 Dec 2022 13:46:39 -0500 Subject: [PATCH 5/8] chore: update tests Signed-off-by: Keith Zantow --- grype/pkg/package_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/grype/pkg/package_test.go b/grype/pkg/package_test.go index 775b174b174..4b98faff0d4 100644 --- a/grype/pkg/package_test.go +++ b/grype/pkg/package_test.go @@ -488,7 +488,7 @@ func intRef(i int) *int { return &i } -func Test_RemovePackagesByOverlap(t *testing.T) { +func Test_RemoveBinaryPackagesByOverlap(t *testing.T) { tests := []struct { name string sbom catalogRelationships @@ -496,17 +496,17 @@ func Test_RemovePackagesByOverlap(t *testing.T) { }{ { name: "includes all packages without overlap", - sbom: catalogWithOverlaps([]string{"go"}, []string{}), + sbom: catalogWithBinaryOverlaps([]string{"go"}, []string{}), expectedPackages: []string{"go"}, }, { name: "excludes single package by overlap", - sbom: catalogWithOverlaps([]string{"go", "node"}, []string{"node"}), + sbom: catalogWithBinaryOverlaps([]string{"go", "node"}, []string{"node"}), expectedPackages: []string{"go"}, }, { name: "excludes multiple package by overlap", - sbom: catalogWithOverlaps([]string{"go", "node", "python", "george"}, []string{"node", "george"}), + sbom: catalogWithBinaryOverlaps([]string{"go", "node", "python", "george"}, []string{"node", "george"}), expectedPackages: []string{"go", "python"}, }, } @@ -528,13 +528,14 @@ type catalogRelationships struct { relationships []artifact.Relationship } -func catalogWithOverlaps(packages []string, overlaps []string) catalogRelationships { +func catalogWithBinaryOverlaps(packages []string, overlaps []string) catalogRelationships { var pkgs []syftPkg.Package var relationships []artifact.Relationship for _, name := range packages { p := syftPkg.Package{ Name: name, + Type: syftPkg.BinaryPkg, } p.SetID() From ff55f27766a90559efb7cbee8a07403a47ba1710 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Fri, 9 Dec 2022 11:48:48 -0500 Subject: [PATCH 6/8] chore: update vulnerability-match-labels Signed-off-by: Keith Zantow --- .gitignore | 3 +++ test/quality/.python-version | 1 + test/quality/vulnerability-match-labels | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 test/quality/.python-version diff --git a/.gitignore b/.gitignore index f2d42c4e4fb..a4e51f36a27 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ CHANGELOG.md *.tmp coverage.txt +# OS files +.DS_Store + # Binaries for programs and plugins *.exe *.exe~ diff --git a/test/quality/.python-version b/test/quality/.python-version new file mode 100644 index 00000000000..1281604a491 --- /dev/null +++ b/test/quality/.python-version @@ -0,0 +1 @@ +3.10.7 diff --git a/test/quality/vulnerability-match-labels b/test/quality/vulnerability-match-labels index d0140484105..c2876a52c5f 160000 --- a/test/quality/vulnerability-match-labels +++ b/test/quality/vulnerability-match-labels @@ -1 +1 @@ -Subproject commit d01404841050b2215a78ba4bbc9d996abb290a9a +Subproject commit c2876a52c5f876bdf05439b46541f8a7a67b1f7c From 88d131c66354afc134b6b5b13104d91a511534b3 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Fri, 9 Dec 2022 18:23:37 -0500 Subject: [PATCH 7/8] chore: lint-fix Signed-off-by: Keith Zantow --- grype/pkg/package.go | 1 - 1 file changed, 1 deletion(-) diff --git a/grype/pkg/package.go b/grype/pkg/package.go index 9b1d9312e96..5662a833227 100644 --- a/grype/pkg/package.go +++ b/grype/pkg/package.go @@ -89,7 +89,6 @@ func (p Package) String() string { return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s, upstreams=%d)", p.Type, p.Name, p.Version, len(p.Upstreams)) } - func RemoveBinaryPackagesByOverlap(catalog *pkg.Catalog, relationships []artifact.Relationship) *pkg.Catalog { byOverlap := map[artifact.ID]artifact.Identifiable{} for _, r := range relationships { From ec03a97cb7a66994b96131ed0a1ae3ba9a67f3f2 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Fri, 9 Dec 2022 19:10:55 -0500 Subject: [PATCH 8/8] chore: fix broken tests Signed-off-by: Keith Zantow --- grype/pkg/package_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grype/pkg/package_test.go b/grype/pkg/package_test.go index f559e8153e4..46f77f67ffc 100644 --- a/grype/pkg/package_test.go +++ b/grype/pkg/package_test.go @@ -513,7 +513,7 @@ func Test_RemoveBinaryPackagesByOverlap(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { catalog := RemoveBinaryPackagesByOverlap(test.sbom.catalog, test.sbom.relationships) - pkgs := FromCatalog(catalog, ProviderConfig{}) + pkgs := FromCatalog(catalog, SynthesisConfig{}) var pkgNames []string for _, p := range pkgs { pkgNames = append(pkgNames, p.Name)