Skip to content

Commit

Permalink
perf: ignored packages should be filtered out before scanning (#1206)
Browse files Browse the repository at this point in the history 
Currently, vulnerabilities in packages that are ignored in the config
file are filtered out after the scanning process.
To optimize the scanning process, these packages should be preemptively
ignored.

---------

Co-authored-by: Rex P <[email protected]>
  • Loading branch information
@giovanni-bozzano @another-rex
giovanni-bozzano and another-rex authored Aug 28, 2024
1 parent edaf998 commit 2735f9d
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 18 deletions.
25 changes: 23 additions & 2 deletions cmd/osv-scanner/__snapshots__/main_test.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ unsupported output format "unknown" - must be one of: table, vertical, json, mar
[TestRun/Empty_cyclonedx_1.4_output - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand All @@ -114,6 +115,7 @@ Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
[TestRun/Empty_cyclonedx_1.5_output - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand All @@ -124,6 +126,7 @@ Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
[TestRun/Empty_gh-annotations_output - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand Down Expand Up @@ -151,6 +154,7 @@ Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
[TestRun/Empty_sarif_output - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand Down Expand Up @@ -515,6 +519,7 @@ invalid verbosity level "unknown" - must be one of: error, warn, info, verbose
[TestRun/json_output_1 - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand All @@ -534,6 +539,7 @@ Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
[TestRun/json_output_2 - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml

---

Expand All @@ -552,6 +558,7 @@ No issues found
[TestRun/one_specific_supported_lockfile - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand Down Expand Up @@ -612,6 +619,7 @@ No issues found
[TestRun/verbosity_level_=_info - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand Down Expand Up @@ -1174,6 +1182,7 @@ Scanned <rootdir>/fixtures/locks-many/package-lock.json file and found 1 package
[TestRun_LocalDatabases/#00 - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
No issues found

Expand All @@ -1186,6 +1195,7 @@ No issues found
[TestRun_LocalDatabases/#00 - 3]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
No issues found

Expand Down Expand Up @@ -1390,11 +1400,11 @@ Scanned <rootdir>/fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Scanned <rootdir>/fixtures/locks-many/package-lock.json file and found 1 package
Scanned <rootdir>/fixtures/locks-many/yarn.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded RubyGems local db from <tempdir>/osv-scanner/RubyGems/all.zip
Loaded Alpine local db from <tempdir>/osv-scanner/Alpine/all.zip
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
Loaded npm local db from <tempdir>/osv-scanner/npm/all.zip
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: Test manifest file
Filtered 1 vulnerability from output
No issues found
Expand All @@ -1412,11 +1422,11 @@ Scanned <rootdir>/fixtures/locks-many/alpine.cdx.xml as CycloneDX SBOM and found
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Scanned <rootdir>/fixtures/locks-many/package-lock.json file and found 1 package
Scanned <rootdir>/fixtures/locks-many/yarn.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded RubyGems local db from <tempdir>/osv-scanner/RubyGems/all.zip
Loaded Alpine local db from <tempdir>/osv-scanner/Alpine/all.zip
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
Loaded npm local db from <tempdir>/osv-scanner/npm/all.zip
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
GHSA-whgm-jr23-g3j9 and 1 alias have been filtered out because: Test manifest file
Filtered 1 vulnerability from output
No issues found
Expand Down Expand Up @@ -1593,6 +1603,7 @@ No issues found
[TestRun_LocalDatabases/#09 - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip

---
Expand All @@ -1613,6 +1624,7 @@ Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
[TestRun_LocalDatabases/#09 - 4]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip

---
Expand All @@ -1633,6 +1645,7 @@ Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
[TestRun_LocalDatabases/#10 - 2]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip

---
Expand All @@ -1653,13 +1666,15 @@ Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
[TestRun_LocalDatabases/#10 - 4]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip

---

[TestRun_LocalDatabases/#11 - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
No issues found

Expand All @@ -1672,6 +1687,7 @@ No issues found
[TestRun_LocalDatabases/#11 - 3]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
Loaded Packagist local db from <tempdir>/osv-scanner/Packagist/all.zip
No issues found

Expand Down Expand Up @@ -1710,6 +1726,7 @@ could not determine extractor, requested my-file

[TestRun_LockfileWithExplicitParseAs/#01 - 1]
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand Down Expand Up @@ -1811,6 +1828,7 @@ Scanned <rootdir>/fixtures/locks-insecure/composer.lock file and found 1 package

[TestRun_LockfileWithExplicitParseAs/#09 - 1]
Scanned <rootdir>/fixtures/locks-many/installed file as a apk-installed and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand All @@ -1821,6 +1839,7 @@ No issues found

[TestRun_LockfileWithExplicitParseAs/#10 - 1]
Scanned <rootdir>/fixtures/locks-many/status file as a dpkg-status and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand Down Expand Up @@ -2033,6 +2052,7 @@ Warning: `scan` exists as both a subcommand of OSV-Scanner and as a file on the
[TestRun_SubCommands/with_no_subcommand - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand All @@ -2044,6 +2064,7 @@ No issues found
[TestRun_SubCommands/with_scan_subcommand - 1]
Scanning dir ./fixtures/locks-many/composer.lock
Scanned <rootdir>/fixtures/locks-many/composer.lock file and found 1 package
Loaded filter from: <rootdir>/fixtures/locks-many/osv-scanner.toml
No issues found

---
Expand Down
42 changes: 26 additions & 16 deletions pkg/osvscanner/osvscanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,19 +683,6 @@ func filterResults(r reporter.Reporter, results *models.VulnerabilityResults, co

// Filters package-grouped vulnerabilities according to config, preserving ordering. Returns filtered package vulnerabilities.
func filterPackageVulns(r reporter.Reporter, pkgVulns models.PackageVulns, configToUse config.Config, unimportantCount *int) models.PackageVulns {
if ignore, ignoreLine := configToUse.ShouldIgnorePackageVersion(pkgVulns.Package.Name, pkgVulns.Package.Version, pkgVulns.Package.Ecosystem); ignore {
pkgString := fmt.Sprintf("%s/%s/%s", pkgVulns.Package.Ecosystem, pkgVulns.Package.Name, pkgVulns.Package.Version)
switch len(pkgVulns.Vulnerabilities) {
case 1:
r.Infof("1 vulnerability for the package %s has been filtered out because: %s\n", pkgString, ignoreLine.Reason)
default:
r.Infof("%d vulnerabilities for the package %s have been filtered out because: %s\n", len(pkgVulns.Vulnerabilities), pkgString, ignoreLine.Reason)
}
pkgVulns.Groups = nil
pkgVulns.Vulnerabilities = nil

return pkgVulns
}
ignoredVulns := map[string]struct{}{}

// Ignores all unimportant vulnerabilities.
Expand Down Expand Up @@ -887,10 +874,16 @@ func DoScan(actions ScannerActions, r reporter.Reporter) (models.VulnerabilityRe
return models.VulnerabilityResults{}, NoPackagesFoundErr
}

filteredScannedPackages := filterUnscannablePackages(scannedPackages)
filteredScannedPackagesWithoutUnscannable := filterUnscannablePackages(scannedPackages)

if len(filteredScannedPackages) != len(scannedPackages) {
r.Infof("Filtered %d local package/s from the scan.\n", len(scannedPackages)-len(filteredScannedPackages))
if len(filteredScannedPackagesWithoutUnscannable) != len(scannedPackages) {
r.Infof("Filtered %d local package/s from the scan.\n", len(scannedPackages)-len(filteredScannedPackagesWithoutUnscannable))
}

filteredScannedPackages := filterIgnoredPackages(r, filteredScannedPackagesWithoutUnscannable, &configManager)

if len(filteredScannedPackages) != len(filteredScannedPackagesWithoutUnscannable) {
r.Infof("Filtered %d ignored package/s from the scan.\n", len(filteredScannedPackagesWithoutUnscannable)-len(filteredScannedPackages))
}

overrideGoVersion(r, filteredScannedPackages, &configManager)
Expand Down Expand Up @@ -969,6 +962,23 @@ func filterUnscannablePackages(packages []scannedPackage) []scannedPackage {
return out
}

// filterIgnoredPackages removes ignore scanned packages according to config. Returns filtered scanned packages.
func filterIgnoredPackages(r reporter.Reporter, packages []scannedPackage, configManager *config.ConfigManager) []scannedPackage {
out := make([]scannedPackage, 0, len(packages))
for _, p := range packages {
configToUse := configManager.Get(r, p.Source.Path)
if ignore, ignoreLine := configToUse.ShouldIgnorePackageVersion(p.Name, p.Version, string(p.Ecosystem)); ignore {
pkgString := fmt.Sprintf("%s/%s/%s", p.Ecosystem, p.Name, p.Version)
r.Infof("Package %s has been filtered out because: %s\n", pkgString, ignoreLine.Reason)

continue
}
out = append(out, p)
}

return out
}

// patchPackageForRequest modifies packages before they are sent to osv.dev to
// account for edge cases.
func patchPackageForRequest(pkg scannedPackage) scannedPackage {
Expand Down

0 comments on commit 2735f9d

Please sign in to comment.