diff --git a/internal/clients/clientimpl/localmatcher/localmatcher.go b/internal/clients/clientimpl/localmatcher/localmatcher.go index 4f0ce460ef2..727f7b0026f 100644 --- a/internal/clients/clientimpl/localmatcher/localmatcher.go +++ b/internal/clients/clientimpl/localmatcher/localmatcher.go @@ -54,8 +54,8 @@ func (matcher *LocalMatcher) MatchVulnerabilities(ctx context.Context, invs []*e } pkg := imodels.FromInventory(inv) - if pkg.Ecosystem.IsEmpty() { - if pkg.Commit == "" { + if pkg.Ecosystem().IsEmpty() { + if pkg.Commit() == "" { // This should never happen, as those results will be filtered out before matching return nil, errors.New("ecosystem is empty and there is no commit hash") } @@ -63,12 +63,12 @@ func (matcher *LocalMatcher) MatchVulnerabilities(ctx context.Context, invs []*e // Is a commit based query, skip local scanning results = append(results, []*models.Vulnerability{}) // TODO (V2 logging): - matcher.r.Infof("Skipping commit scanning for: %s\n", pkg.Commit) + matcher.r.Infof("Skipping commit scanning for: %s\n", pkg.Commit()) continue } - db, err := matcher.loadDBFromCache(ctx, pkg.Ecosystem) + db, err := matcher.loadDBFromCache(ctx, pkg.Ecosystem()) if err != nil { continue diff --git a/internal/clients/clientimpl/localmatcher/zip.go b/internal/clients/clientimpl/localmatcher/zip.go index fc3951b4011..0e7be230057 100644 --- a/internal/clients/clientimpl/localmatcher/zip.go +++ b/internal/clients/clientimpl/localmatcher/zip.go @@ -238,12 +238,12 @@ func (db *ZipDB) VulnerabilitiesAffectingPackage(pkg imodels.PackageInfo) []*mod // TODO (V2 Models): remove this once PackageDetails has been migrated mappedPackageDetails := lockfile.PackageDetails{ - Name: pkg.Name, - Version: pkg.Version, - Commit: pkg.Commit, - Ecosystem: lockfile.Ecosystem(pkg.Ecosystem.String()), - CompareAs: lockfile.Ecosystem(pkg.Ecosystem.String()), - DepGroups: pkg.DepGroups, + Name: pkg.Name(), + Version: pkg.Version(), + Commit: pkg.Commit(), + Ecosystem: lockfile.Ecosystem(pkg.Ecosystem().String()), + CompareAs: lockfile.Ecosystem(pkg.Ecosystem().String()), + DepGroups: pkg.DepGroups(), } for _, vulnerability := range db.Vulnerabilities(false) { diff --git a/internal/clients/clientimpl/osvmatcher/osvmatcher.go b/internal/clients/clientimpl/osvmatcher/osvmatcher.go index 731c7c73eaf..81258c1f5c5 100644 --- a/internal/clients/clientimpl/osvmatcher/osvmatcher.go +++ b/internal/clients/clientimpl/osvmatcher/osvmatcher.go @@ -9,6 +9,7 @@ import ( "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/log" "github.com/google/osv-scanner/internal/imodels" + "github.com/google/osv-scanner/internal/imodels/ecosystem" "github.com/google/osv-scanner/internal/osvdev" "github.com/google/osv-scanner/internal/semantic" "github.com/google/osv-scanner/pkg/models" @@ -155,19 +156,19 @@ func queryForBatchWithPaging(ctx context.Context, c *osvdev.OSVClient, queries [ } func pkgToQuery(pkg imodels.PackageInfo) *osvdev.Query { - if pkg.Name != "" && !pkg.Ecosystem.IsEmpty() && pkg.Version != "" { + if pkg.Name() != "" && !pkg.Ecosystem().IsEmpty() && pkg.Version() != "" { return &osvdev.Query{ Package: osvdev.Package{ - Name: pkg.Name, - Ecosystem: pkg.Ecosystem.String(), + Name: pkg.Name(), + Ecosystem: pkg.Ecosystem().String(), }, - Version: pkg.Version, + Version: pkg.Version(), } } - if pkg.Commit != "" { + if pkg.Commit() != "" { return &osvdev.Query{ - Commit: pkg.Commit, + Commit: pkg.Commit(), } } @@ -184,16 +185,16 @@ func invsToQueries(invs []*extractor.Inventory) []*osvdev.Query { for i, inv := range invs { pkg := imodels.FromInventory(inv) - pkg = patchPackageForRequest(pkg) queries[i] = pkgToQuery(pkg) + patchQueryForRequest(queries[i]) } return queries } -// patchPackageForRequest modifies packages before they are sent to osv.dev to +// patchQueryForRequest modifies packages before they are sent to osv.dev to // account for edge cases. -func patchPackageForRequest(pkg imodels.PackageInfo) imodels.PackageInfo { +func patchQueryForRequest(queryToPatch *osvdev.Query) { // Assume Go stdlib patch version as the latest version // // This is done because go1.20 and earlier do not support patch @@ -202,10 +203,12 @@ func patchPackageForRequest(pkg imodels.PackageInfo) imodels.PackageInfo { // However, if we assume patch version as .0, this will cause a lot of // false positives. This compromise still allows osv-scanner to pick up // when the user is using a minor version that is out-of-support. - if pkg.Name == "stdlib" && pkg.Ecosystem.Ecosystem == osvschema.EcosystemGo { - v := semantic.ParseSemverLikeVersion(pkg.Version, 3) + // + // MustParse works here because this query is converted from a valid ecosystem in the first place + if queryToPatch.Package.Name == "stdlib" && ecosystem.MustParse(queryToPatch.Package.Ecosystem).Ecosystem == osvschema.EcosystemGo { + v := semantic.ParseSemverLikeVersion(queryToPatch.Version, 3) if len(v.Components) == 2 { - pkg.Version = fmt.Sprintf( + queryToPatch.Version = fmt.Sprintf( "%d.%d.%d", v.Components.Fetch(0), v.Components.Fetch(1), @@ -213,6 +216,4 @@ func patchPackageForRequest(pkg imodels.PackageInfo) imodels.PackageInfo { ) } } - - return pkg } diff --git a/internal/config/config.go b/internal/config/config.go index 13db57ee01c..c889b37dc7a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,19 +54,19 @@ type PackageOverrideEntry struct { } func (e PackageOverrideEntry) matches(pkg imodels.PackageInfo) bool { - if e.Name != "" && e.Name != pkg.Name { + if e.Name != "" && e.Name != pkg.Name() { return false } - if e.Version != "" && e.Version != pkg.Version { + if e.Version != "" && e.Version != pkg.Version() { return false } // If there is an ecosystem filter, the filter must not match both the: // - Full ecosystem + suffix // - The base ecosystem - if e.Ecosystem != "" && (e.Ecosystem != pkg.Ecosystem.String() && e.Ecosystem != string(pkg.Ecosystem.Ecosystem)) { + if e.Ecosystem != "" && (e.Ecosystem != pkg.Ecosystem().String() && e.Ecosystem != string(pkg.Ecosystem().Ecosystem)) { return false } - if e.Group != "" && !slices.Contains(pkg.DepGroups, e.Group) { + if e.Group != "" && !slices.Contains(pkg.DepGroups(), e.Group) { return false } diff --git a/internal/config/config_internal_test.go b/internal/config/config_internal_test.go index d62cf1cdde0..61d4cacefbb 100644 --- a/internal/config/config_internal_test.go +++ b/internal/config/config_internal_test.go @@ -8,8 +8,11 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scanner/internal/imodels" - "github.com/google/osv-scanner/internal/imodels/ecosystem" + "github.com/google/osv-scanner/internal/scalibrextract/ecosystemmock" "github.com/google/osv-scanner/pkg/reporter" ) @@ -378,10 +381,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -404,10 +413,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -430,10 +445,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("npm"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib2", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "npm", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -452,9 +473,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "bin1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Alpine:3.20"), + Inventory: &extractor.Inventory{ + Name: "bin1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Alpine:3.20", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -477,9 +502,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "bin2", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Alpine:3.19"), + Inventory: &extractor.Inventory{ + Name: "bin2", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Alpine:3.19", + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -497,9 +526,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "bin1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Alpine:3.20"), + Inventory: &extractor.Inventory{ + Name: "bin1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Alpine:3.20", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -523,10 +556,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -549,10 +588,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("npm"), - DepGroups: []string{"optional"}, + Inventory: &extractor.Inventory{ + Name: "lib2", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "npm", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"optional"}, + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -570,9 +615,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("npm"), + Inventory: &extractor.Inventory{ + Name: "lib2", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "npm", + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -591,10 +640,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -617,10 +672,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -639,10 +700,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -665,10 +732,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib2", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("npm"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib2", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "npm", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -689,9 +762,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -717,9 +794,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -745,10 +826,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"dev"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev"}, + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -775,10 +862,16 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), - DepGroups: []string{"prod"}, + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"prod"}, + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -806,9 +899,13 @@ func TestConfig_ShouldIgnorePackage(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "2.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "2.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -855,9 +952,13 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, }, @@ -877,9 +978,13 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: false, }, @@ -898,9 +1003,13 @@ func TestConfig_ShouldIgnorePackageVulnerabilities(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, }, @@ -944,9 +1053,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -975,9 +1088,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.0", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.0", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -1006,9 +1123,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -1029,9 +1150,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: false, wantEntry: PackageOverrideEntry{}, @@ -1051,9 +1176,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ @@ -1080,9 +1209,13 @@ func TestConfig_ShouldOverridePackageLicense(t *testing.T) { }, }, args: imodels.PackageInfo{ - Name: "lib1", - Version: "1.0.1", - Ecosystem: ecosystem.MustParse("Go"), + Inventory: &extractor.Inventory{ + Name: "lib1", + Version: "1.0.1", + Extractor: ecosystemmock.Extractor{ + MockEcosystem: "Go", + }, + }, }, wantOk: true, wantEntry: PackageOverrideEntry{ diff --git a/internal/imodels/ecosystem/ecosystem.go b/internal/imodels/ecosystem/ecosystem.go index 677ef70dc0b..2de04fcaf98 100644 --- a/internal/imodels/ecosystem/ecosystem.go +++ b/internal/imodels/ecosystem/ecosystem.go @@ -39,7 +39,7 @@ type Parsed struct { Suffix string } -func (p *Parsed) IsEmpty() bool { +func (p Parsed) IsEmpty() bool { return p.Ecosystem == "" } @@ -70,8 +70,7 @@ func (p Parsed) MarshalJSON() ([]byte, error) { return []byte(`"` + p.String() + `"`), nil } -//goland:noinspection GoMixedReceiverTypes -func (p *Parsed) String() string { +func (p Parsed) String() string { str := string(p.Ecosystem) if p.Suffix != "" { diff --git a/internal/imodels/imodels.go b/internal/imodels/imodels.go index eafaf493561..bcca6c0b246 100644 --- a/internal/imodels/imodels.go +++ b/internal/imodels/imodels.go @@ -32,123 +32,131 @@ var osExtractors = map[string]struct{}{ rpm.Extractor{}.Name(): {}, } -// PackageInfo represents a package found during a scan. This is generally -// converted directly from the extractor.Inventory type, with some restructuring -// for easier use within osv-scanner itself. +// PackageInfo provides getter functions for commonly used fields of inventory +// and applies transformations when required for use in osv-scanner type PackageInfo struct { - Name string // Name will be SourceName matching the osv-schema - Version string - Ecosystem ecosystem.Parsed - - Location string // Contains Inventory.Locations[0] - SourceType SourceType - - Commit string - Repository string + // purlCache is used to cache the special case for SBOMs where we convert Name, Version, and Ecosystem from purls + // extracted from the SBOM + purlCache *models.PackageInfo + *extractor.Inventory +} - // For package sources - DepGroups []string +func (pkg *PackageInfo) Name() string { + // TODO(v2): SBOM special case, to be removed after PURL to ESI conversion within each extractor is complete + if pkg.purlCache != nil { + return pkg.purlCache.Name + } - // For OS packages - // This is usually the BinaryName, while Name is the SourceName - OSPackageName string + if pkg.Ecosystem().Ecosystem == osvschema.EcosystemGo && pkg.Inventory.Name == "go" { + return "stdlib" + } - AdditionalLocations []string // Contains Inventory.Locations[1..] + if metadata, ok := pkg.Inventory.Metadata.(*dpkg.Metadata); ok { + // Debian uses source name on osv.dev + // (fallback to using the normal name if source name is empty) + if metadata.SourceName != "" { + return metadata.SourceName + } + } - OriginalInventory *extractor.Inventory + return pkg.Inventory.Name } -// FromInventory converts an extractor.Inventory into a PackageInfo. -// -// For ease of use, this function does not return an error, but will log -// warnings when encountering unexpected inventory entries -func FromInventory(inventory *extractor.Inventory) PackageInfo { - pkgInfo := PackageInfo{ - Name: inventory.Name, - Version: inventory.Version, - Location: inventory.Locations[0], - AdditionalLocations: inventory.Locations[1:], - OriginalInventory: inventory, +func (pkg *PackageInfo) Ecosystem() ecosystem.Parsed { + ecosystemStr := pkg.Inventory.Ecosystem() + + // TODO(v2): SBOM special case, to be removed after PURL to ESI conversion within each extractor is complete + if pkg.purlCache != nil { + ecosystemStr = pkg.purlCache.Ecosystem } - // Set Ecosystem - eco, err := ecosystem.Parse(inventory.Ecosystem()) + // TODO: Maybe cache this parse result + eco, err := ecosystem.Parse(ecosystemStr) if err != nil { // Ignore this error for now as we can't do too much about an unknown ecosystem // TODO(v2): Replace with slog log.Printf("Warning: %s\n", err.Error()) } - pkgInfo.Ecosystem = eco - // Set Commit and Repository - if inventory.SourceCode != nil { - pkgInfo.Commit = inventory.SourceCode.Commit - pkgInfo.Repository = inventory.SourceCode.Repo + return eco +} + +func (pkg *PackageInfo) Version() string { + // TODO(v2): SBOM special case, to be removed after PURL to ESI conversion within each extractor is complete + if pkg.purlCache != nil { + return pkg.purlCache.Version } - // Set DepGroups - if dg, ok := inventory.Metadata.(scalibrosv.DepGroups); ok { - pkgInfo.DepGroups = dg.DepGroups() + return pkg.Inventory.Version +} + +func (pkg *PackageInfo) Location() string { + if len(pkg.Inventory.Locations) > 0 { + return pkg.Inventory.Locations[0] } - // Set SourceType - if inventory.Extractor != nil { - extractorName := inventory.Extractor.Name() - if _, ok := osExtractors[extractorName]; ok { - pkgInfo.SourceType = SourceTypeOSPackage - } else if _, ok := sbomExtractors[extractorName]; ok { - pkgInfo.SourceType = SourceTypeSBOM - } else if _, ok := gitExtractors[extractorName]; ok { - pkgInfo.SourceType = SourceTypeGit - } else { - pkgInfo.SourceType = SourceTypeProjectPackage - } + return "" +} + +func (pkg *PackageInfo) Commit() string { + if pkg.Inventory.SourceCode != nil { + return pkg.Inventory.SourceCode.Commit } - // --- General Patching --- + return "" +} - // Scalibr uses go to indicate go compiler version - // We specifically cares about the stdlib version inside the package - // so convert the package name from go to stdlib - if eco.Ecosystem == osvschema.EcosystemGo && inventory.Name == "go" { - pkgInfo.Name = "stdlib" +func (pkg *PackageInfo) SourceType() SourceType { + if pkg.Inventory.Extractor == nil { + return SourceTypeUnknown } - // TODO (V2): SBOMs have a special case where we manually convert the PURL here - // instead while PURL to ESI conversion is not complete - if pkgInfo.SourceType == SourceTypeSBOM { - purl := inventory.Extractor.ToPURL(inventory) - if purl != nil { - // Error should never happen here since the PURL is from an already parsed purl - pi, _ := models.PURLToPackage(purl.String()) - pkgInfo.Name = pi.Name - pkgInfo.Version = pi.Version - parsed, err := ecosystem.Parse(pi.Ecosystem) - if err != nil { - // TODO: Replace with slog - log.Printf("Warning, found unexpected ecosystem in purl %q, likely will not return any results for this package.\n", purl.String()) - } - pkgInfo.Ecosystem = parsed - } + extractorName := pkg.Inventory.Extractor.Name() + if _, ok := osExtractors[extractorName]; ok { + return SourceTypeOSPackage + } else if _, ok := sbomExtractors[extractorName]; ok { + return SourceTypeSBOM + } else if _, ok := gitExtractors[extractorName]; ok { + return SourceTypeGit } - // --- Metadata Patching --- + return SourceTypeProjectPackage +} - // Depending on the specific metadata types set fields in package info. - if metadata, ok := inventory.Metadata.(*apk.Metadata); ok { - pkgInfo.OSPackageName = metadata.PackageName - } else if metadata, ok := inventory.Metadata.(*dpkg.Metadata); ok { - pkgInfo.OSPackageName = metadata.PackageName - // Debian uses source name on osv.dev - // (fallback to using the normal name if source name is empty) - if metadata.SourceName != "" { - pkgInfo.Name = metadata.SourceName +func (pkg *PackageInfo) DepGroups() []string { + if dg, ok := pkg.Inventory.Metadata.(scalibrosv.DepGroups); ok { + return dg.DepGroups() + } + + return []string{} +} + +func (pkg *PackageInfo) OSPackageName() string { + if metadata, ok := pkg.Inventory.Metadata.(*apk.Metadata); ok { + return metadata.PackageName + } + if metadata, ok := pkg.Inventory.Metadata.(*dpkg.Metadata); ok { + return metadata.PackageName + } + if metadata, ok := pkg.Inventory.Metadata.(*rpm.Metadata); ok { + return metadata.PackageName + } + + return "" +} + +// FromInventory converts an extractor.Inventory into a PackageInfo. +func FromInventory(inventory *extractor.Inventory) PackageInfo { + pi := PackageInfo{Inventory: inventory} + if pi.SourceType() == SourceTypeSBOM { + purl := pi.Inventory.Extractor.ToPURL(pi.Inventory) + if purl != nil { + purlCache, _ := models.PURLToPackage(purl.String()) + pi.purlCache = &purlCache } - } else if metadata, ok := inventory.Metadata.(*rpm.Metadata); ok { - pkgInfo.OSPackageName = metadata.PackageName } - return pkgInfo + return pi } // PackageScanResult represents a package and its associated vulnerabilities and licenses. diff --git a/internal/scalibrextract/ecosystemmock/extractor.go b/internal/scalibrextract/ecosystemmock/extractor.go new file mode 100644 index 00000000000..2b0b8df6f04 --- /dev/null +++ b/internal/scalibrextract/ecosystemmock/extractor.go @@ -0,0 +1,46 @@ +// ecosystemmock extractor just returns the passed in Ecosystem string from Ecosystem() +// This is useful when manually creating an inventory so that inv.Ecosystem() returns the ecosystem you want +package ecosystemmock + +import ( + "context" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" +) + +type Extractor struct { + MockEcosystem string +} + +var _ filesystem.Extractor = Extractor{} + +func (e Extractor) Name() string { return "ecosystemmock" } + +func (e Extractor) Version() int { return 0 } + +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +func (e Extractor) FileRequired(_ filesystem.FileAPI) bool { + return false +} + +func (e Extractor) Extract(_ context.Context, _ *filesystem.ScanInput) ([]*extractor.Inventory, error) { + panic("this is not a real extractor and should not be called") +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(_ *extractor.Inventory) *purl.PackageURL { + return nil +} + +// Ecosystem returns the OSV ecosystem ('npm') of the software extracted by this extractor. +func (e Extractor) Ecosystem(_ *extractor.Inventory) string { + return e.MockEcosystem +} + +var _ filesystem.Extractor = Extractor{} diff --git a/pkg/osv/osv.go b/pkg/osv/osv.go index 6705daa2603..56fdd74b6a9 100644 --- a/pkg/osv/osv.go +++ b/pkg/osv/osv.go @@ -125,13 +125,13 @@ func MakePURLRequest(purl string) *Query { func MakePkgRequest(pkgInfo imodels.PackageInfo) *Query { return &Query{ - Version: pkgInfo.Version, + Version: pkgInfo.Version(), Package: Package{ - Name: pkgInfo.Name, - Ecosystem: pkgInfo.Ecosystem.String(), + Name: pkgInfo.Name(), + Ecosystem: pkgInfo.Ecosystem().String(), }, Metadata: models.Metadata{ - DepGroups: pkgInfo.DepGroups, + DepGroups: pkgInfo.DepGroups(), }, } } diff --git a/pkg/osvscanner/filter.go b/pkg/osvscanner/filter.go index b9ff05b1b14..ad4856975cf 100644 --- a/pkg/osvscanner/filter.go +++ b/pkg/osvscanner/filter.go @@ -19,8 +19,8 @@ func filterUnscannablePackages(r reporter.Reporter, scanResults *results.ScanRes switch { // If none of the cases match, skip this package since it's not scannable - case !p.Ecosystem.IsEmpty() && p.Name != "" && p.Version != "": - case p.Commit != "": + case !p.Ecosystem().IsEmpty() && p.Name() != "" && p.Version() != "": + case p.Commit() != "": default: continue } @@ -42,10 +42,10 @@ func filterIgnoredPackages(r reporter.Reporter, scanResults *results.ScanResults out := make([]imodels.PackageScanResult, 0, len(scanResults.PackageScanResults)) for _, psr := range scanResults.PackageScanResults { p := psr.PackageInfo - configToUse := configManager.Get(r, p.Location) + configToUse := configManager.Get(r, p.Location()) if ignore, ignoreLine := configToUse.ShouldIgnorePackage(p); ignore { - pkgString := fmt.Sprintf("%s/%s/%s", p.Ecosystem.String(), p.Name, p.Version) + pkgString := fmt.Sprintf("%s/%s/%s", p.Ecosystem().String(), p.Name(), p.Version()) reason := ignoreLine.Reason if reason == "" { diff --git a/pkg/osvscanner/osvscanner.go b/pkg/osvscanner/osvscanner.go index 85477485c88..b2846679a14 100644 --- a/pkg/osvscanner/osvscanner.go +++ b/pkg/osvscanner/osvscanner.go @@ -204,7 +204,7 @@ func makeRequestWithMatcher( matcher clientinterfaces.VulnerabilityMatcher) error { invs := make([]*extractor.Inventory, 0, len(packages)) for _, pkgs := range packages { - invs = append(invs, pkgs.PackageInfo.OriginalInventory) + invs = append(invs, pkgs.PackageInfo.Inventory) } res, err := matcher.MatchVulnerabilities(context.Background(), invs) @@ -228,11 +228,11 @@ func makeLicensesRequests(packages []imodels.PackageScanResult) error { queries := make([]*depsdevpb.GetVersionRequest, len(packages)) for i, psr := range packages { pkg := psr.PackageInfo - system, ok := depsdev.System[psr.PackageInfo.Ecosystem.Ecosystem] - if !ok || pkg.Name == "" || pkg.Version == "" { + system, ok := depsdev.System[psr.PackageInfo.Ecosystem().Ecosystem] + if !ok || pkg.Name() == "" || pkg.Version() == "" { continue } - queries[i] = depsdev.VersionQuery(system, pkg.Name, pkg.Version) + queries[i] = depsdev.VersionQuery(system, pkg.Name(), pkg.Version()) } licenses, err := depsdev.MakeVersionRequests(queries) if err != nil { @@ -250,12 +250,10 @@ func makeLicensesRequests(packages []imodels.PackageScanResult) error { func overrideGoVersion(r reporter.Reporter, scanResults *results.ScanResults) { for i, psr := range scanResults.PackageScanResults { pkg := psr.PackageInfo - if pkg.Name == "stdlib" && pkg.Ecosystem.Ecosystem == osvschema.EcosystemGo { - configToUse := scanResults.ConfigManager.Get(r, pkg.Location) + if pkg.Name() == "stdlib" && pkg.Ecosystem().Ecosystem == osvschema.EcosystemGo { + configToUse := scanResults.ConfigManager.Get(r, pkg.Location()) if configToUse.GoVersionOverride != "" { - scanResults.PackageScanResults[i].PackageInfo.Version = configToUse.GoVersionOverride - // Also patch it in the inventory, as we have to use the original inventory to make requests - scanResults.PackageScanResults[i].PackageInfo.OriginalInventory.Version = configToUse.GoVersionOverride + scanResults.PackageScanResults[i].PackageInfo.Inventory.Version = configToUse.GoVersionOverride } continue diff --git a/pkg/osvscanner/scan.go b/pkg/osvscanner/scan.go index c8f8c591b38..610d6301b03 100644 --- a/pkg/osvscanner/scan.go +++ b/pkg/osvscanner/scan.go @@ -7,6 +7,7 @@ import ( "github.com/google/osv-scanner/internal/imodels" "github.com/google/osv-scanner/internal/resolution/client" "github.com/google/osv-scanner/internal/resolution/datasource" + "github.com/google/osv-scanner/internal/scalibrextract/ecosystemmock" "github.com/google/osv-scanner/internal/scalibrextract/language/java/pomxmlnet" "github.com/google/osv-scanner/pkg/osvscanner/internal/scanners" "github.com/google/osv-scanner/pkg/reporter" @@ -75,7 +76,10 @@ func scan(r reporter.Reporter, actions ScannerActions) ([]imodels.PackageScanRes // Add on additional direct dependencies passed straight from ScannerActions: for _, commit := range actions.GitCommits { pi := imodels.PackageInfo{ - Commit: commit, + Inventory: &extractor.Inventory{ + SourceCode: &extractor.SourceCodeIdentifier{Commit: commit}, + Extractor: ecosystemmock.Extractor{}, // Empty ecosystem + }, } packages = append(packages, imodels.PackageScanResult{ diff --git a/pkg/osvscanner/vulnerability_result.go b/pkg/osvscanner/vulnerability_result.go index 0129aa24a98..f3160ffd592 100644 --- a/pkg/osvscanner/vulnerability_result.go +++ b/pkg/osvscanner/vulnerability_result.go @@ -33,22 +33,22 @@ func buildVulnerabilityResults( includePackage := actions.ShowAllPackages var pkg models.PackageVulns - if p.Commit != "" { - pkg.Package.Commit = p.Commit - pkg.Package.Name = p.Name + if p.Commit() != "" { + pkg.Package.Commit = p.Commit() + pkg.Package.Name = p.Name() } - if p.Version != "" && !p.Ecosystem.IsEmpty() { + if p.Version() != "" && !p.Ecosystem().IsEmpty() { pkg.Package = models.PackageInfo{ - Name: p.Name, - Version: p.Version, - Ecosystem: p.Ecosystem.String(), + Name: p.Name(), + Version: p.Version(), + Ecosystem: p.Ecosystem().String(), // ImageOrigin: p.ImageOrigin, } } - pkg.DepGroups = p.DepGroups - configToUse := scanResults.ConfigManager.Get(r, p.Location) + pkg.DepGroups = p.DepGroups() + configToUse := scanResults.ConfigManager.Get(r, p.Location()) if len(psr.Vulnerabilities) > 0 { if !configToUse.ShouldIgnorePackageVulnerabilities(p) { @@ -107,7 +107,7 @@ func buildVulnerabilityResults( } if includePackage { var sourceType string - switch p.SourceType { + switch p.SourceType() { case imodels.SourceTypeOSPackage: sourceType = "os" case imodels.SourceTypeProjectPackage: @@ -122,7 +122,7 @@ func buildVulnerabilityResults( sourceType = "unknown" } source := models.SourceInfo{ - Path: p.Location, + Path: p.Location(), Type: sourceType, } groupedBySource[source] = append(groupedBySource[source], pkg) diff --git a/pkg/osvscanner/vulnerability_result_internal_test.go b/pkg/osvscanner/vulnerability_result_internal_test.go index 00710c95b16..d390ba95bf0 100644 --- a/pkg/osvscanner/vulnerability_result_internal_test.go +++ b/pkg/osvscanner/vulnerability_result_internal_test.go @@ -3,10 +3,11 @@ package osvscanner import ( "testing" + "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scanner/internal/config" "github.com/google/osv-scanner/internal/imodels" - "github.com/google/osv-scanner/internal/imodels/ecosystem" "github.com/google/osv-scanner/internal/imodels/results" + "github.com/google/osv-scanner/internal/scalibrextract/ecosystemmock" "github.com/google/osv-scanner/internal/testutility" "github.com/google/osv-scanner/pkg/models" "github.com/google/osv-scanner/pkg/osv" @@ -23,23 +24,27 @@ func Test_assembleResult(t *testing.T) { } packages := []imodels.PackageInfo{ { - Name: "pkg-1", - Ecosystem: ecosystem.MustParse("npm"), - Version: "1.0.0", - Location: "dir/package-lock.json", - SourceType: imodels.SourceTypeProjectPackage, - }, { - Name: "pkg-2", - Ecosystem: ecosystem.MustParse("npm"), - Version: "1.0.0", - Location: "dir/package-lock.json", - SourceType: imodels.SourceTypeProjectPackage, + Inventory: &extractor.Inventory{ + Name: "pkg-1", + Extractor: ecosystemmock.Extractor{MockEcosystem: "npm"}, + Version: "1.0.0", + Locations: []string{"dir/package-lock.json"}, + }, + }, + { + Inventory: &extractor.Inventory{ + Name: "pkg-2", + Extractor: ecosystemmock.Extractor{MockEcosystem: "npm"}, + Version: "1.0.0", + Locations: []string{"dir/package-lock.json"}, + }, }, { - Name: "pkg-3", - Ecosystem: ecosystem.MustParse("npm"), - Version: "1.0.0", - Location: "other-dir/package-lock.json", - SourceType: imodels.SourceTypeProjectPackage, + Inventory: &extractor.Inventory{ + Name: "pkg-3", + Extractor: ecosystemmock.Extractor{MockEcosystem: "npm"}, + Version: "1.0.0", + Locations: []string{"other-dir/package-lock.json"}, + }, }, } vulnsResp := &osv.HydratedBatchedResponse{