refactor: Licence matcher (#1491)

Also partially removes the global user agent variable to be specificly
passed in for every client.

Majority of the code in licencematcher.go is migrated from the deleted
license.go file.

cmd/osv-scanner/fix/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/osv-scanner/internal/resolution/client"
 	"github.com/google/osv-scanner/internal/resolution/lockfile"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
+	"github.com/google/osv-scanner/internal/version"
 	"github.com/google/osv-scanner/pkg/reporter"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/term"
@@ -330,7 +331,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 
 	switch ctx.String("data-source") {
 	case "deps.dev":
-		cl, err := client.NewDepsDevClient(depsdev.DepsdevAPI)
+		cl, err := client.NewDepsDevClient(depsdev.DepsdevAPI, "osv-scanner_fix/"+version.OSVVersion)
 		if err != nil {
 			return nil, err
 		}

cmd/osv-scanner/update/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/google/osv-scanner/internal/remediation/suggest"
 	"github.com/google/osv-scanner/internal/resolution/client"
 	"github.com/google/osv-scanner/internal/resolution/manifest"
+	"github.com/google/osv-scanner/internal/version"
 	"github.com/google/osv-scanner/pkg/lockfile"
 	"github.com/google/osv-scanner/pkg/reporter"
 	"github.com/urfave/cli/v2"
@@ -74,7 +75,7 @@ func action(ctx *cli.Context, stdout, stderr io.Writer) (reporter.Reporter, erro
 	}
 
 	var err error
-	options.Client.DependencyClient, err = client.NewDepsDevClient(depsdev.DepsdevAPI)
+	options.Client.DependencyClient, err = client.NewDepsDevClient(depsdev.DepsdevAPI, "osv-scanner_update/"+version.OSVVersion)
 	if err != nil {
 		return nil, err
 	}

internal/clients/clientimpl/licensematcher/licensematcher.go

@@ -0,0 +1,109 @@
+package licensematcher
+
+import (
+	"context"
+
+	depsdevpb "deps.dev/api/v3"
+	"github.com/google/osv-scanner/internal/depsdev"
+	"github.com/google/osv-scanner/internal/imodels"
+	"github.com/google/osv-scanner/internal/resolution/datasource"
+	"github.com/google/osv-scanner/pkg/models"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	maxConcurrentRequests = 1000
+)
+
+// DepsDevLicenseMatcher implements the LicenseMatcher interface with a deps.dev client.
+// It sends out requests for every package version and does not perform caching.
+type DepsDevLicenseMatcher struct {
+	Client *datasource.DepsDevAPIClient
+}
+
+func (matcher *DepsDevLicenseMatcher) MatchLicenses(ctx context.Context, packages []imodels.PackageScanResult) error {
+	queries := make([]*depsdevpb.GetVersionRequest, len(packages))
+
+	for i, psr := range packages {
+		pkg := psr.PackageInfo
+		system, ok := depsdev.System[psr.PackageInfo.Ecosystem().Ecosystem]
+		if !ok || pkg.Name() == "" || pkg.Version() == "" {
+			continue
+		}
+		queries[i] = versionQuery(system, pkg.Name(), pkg.Version())
+	}
+
+	licenses, err := matcher.makeVersionRequest(ctx, queries)
+	if err != nil {
+		return err
+	}
+
+	for i, license := range licenses {
+		packages[i].Licenses = license
+	}
+
+	return nil
+}
+
+// makeVersionRequest calls the deps.dev GetVersion gRPC API endpoint for each
+// query. It makes these requests concurrently, sharing the single HTTP/2
+// connection. The order in which the requests are specified should correspond
+// to the order of licenses returned by this function.
+func (matcher *DepsDevLicenseMatcher) makeVersionRequest(ctx context.Context, queries []*depsdevpb.GetVersionRequest) ([][]models.License, error) {
+	licenses := make([][]models.License, len(queries))
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(maxConcurrentRequests)
+
+	for i := range queries {
+		if queries[i] == nil {
+			// This may be a private package.
+			licenses[i] = []models.License{models.License("UNKNOWN")}
+			continue
+		}
+		g.Go(func() error {
+			resp, err := matcher.Client.GetVersion(ctx, queries[i])
+			if err != nil {
+				if status.Code(err) == codes.NotFound {
+					licenses[i] = append(licenses[i], "UNKNOWN")
+					return nil
+				}
+
+				return err
+			}
+			ls := make([]models.License, len(resp.GetLicenses()))
+			for j, license := range resp.GetLicenses() {
+				ls[j] = models.License(license)
+			}
+			if len(ls) == 0 {
+				// The deps.dev API will return an
+				// empty slice if the license is
+				// unknown.
+				ls = []models.License{models.License("UNKNOWN")}
+			}
+			licenses[i] = ls
+
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	return licenses, nil
+}
+
+func versionQuery(system depsdevpb.System, name string, version string) *depsdevpb.GetVersionRequest {
+	if system == depsdevpb.System_GO {
+		version = "v" + version
+	}
+
+	return &depsdevpb.GetVersionRequest{
+		VersionKey: &depsdevpb.VersionKey{
+			System:  system,
+			Name:    name,
+			Version: version,
+		},
+	}
+}

internal/clients/clientinterfaces/licensematcher.go

@@ -0,0 +1,11 @@
+package clientinterfaces
+
+import (
+	"context"
+
+	"github.com/google/osv-scanner/internal/imodels"
+)
+
+type LicenseMatcher interface {
+	MatchLicenses(ctx context.Context, psr []imodels.PackageScanResult) error
+}

internal/depsdev/depsdev.go

@@ -0,0 +1,21 @@
+package depsdev
+
+import (
+	"github.com/ossf/osv-schema/bindings/go/osvschema"
+
+	depsdevpb "deps.dev/api/v3"
+)
+
+// DepsdevAPI is the URL to the deps.dev API. It is documented at
+// docs.deps.dev/api.
+const DepsdevAPI = "api.deps.dev:443"
+
+// System maps from a lockfile system to the depsdev API system.
+var System = map[osvschema.Ecosystem]depsdevpb.System{
+	osvschema.EcosystemNPM:      depsdevpb.System_NPM,
+	osvschema.EcosystemNuGet:    depsdevpb.System_NUGET,
+	osvschema.EcosystemCratesIO: depsdevpb.System_CARGO,
+	osvschema.EcosystemGo:       depsdevpb.System_GO,
+	osvschema.EcosystemMaven:    depsdevpb.System_MAVEN,
+	osvschema.EcosystemPyPI:     depsdevpb.System_PYPI,
+}

internal/osvdev/config.go

@@ -18,7 +18,7 @@ func DefaultConfig() ClientConfig {
 		JitterMultiplier:           2,
 		BackoffDurationExponential: 2,
 		BackoffDurationMultiplier:  1,
-		UserAgent:                  "osv-scanner/" + version.OSVVersion,
+		UserAgent:                  "osv-scanner_scan/" + version.OSVVersion,
 		MaxConcurrentBatchRequests: 10,
 	}
 }

internal/osvdev/osvdev.go

@@ -26,6 +26,8 @@ const (
 
 	// MaxQueriesPerQueryBatchRequest is a limit set in osv.dev's API, so is not configurable
 	MaxQueriesPerQueryBatchRequest = 1000
+
+	DefaultBaseURL = "https://api.osv.dev"
 )
 
 type OSVClient struct {
@@ -39,7 +41,7 @@ func DefaultClient() *OSVClient {
 	return &OSVClient{
 		HTTPClient:  http.DefaultClient,
 		Config:      DefaultConfig(),
-		BaseHostURL: "https://api.osv.dev",
+		BaseHostURL: DefaultBaseURL,
 	}
 }
 

internal/resolution/client/depsdev_client.go

@@ -16,8 +16,8 @@ type DepsDevClient struct {
 	c *datasource.DepsDevAPIClient
 }
 
-func NewDepsDevClient(addr string) (*DepsDevClient, error) {
-	c, err := datasource.NewDepsDevAPIClient(addr)
+func NewDepsDevClient(addr string, userAgent string) (*DepsDevClient, error) {
+	c, err := datasource.NewDepsDevAPIClient(addr, userAgent)
 	if err != nil {
 		return nil, err
 	}

internal/resolution/datasource/depsdev_api.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	pb "deps.dev/api/v3"
-	"github.com/google/osv-scanner/pkg/osv"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 )
@@ -52,16 +51,16 @@ func makeVersionKey(k *pb.VersionKey) versionKey {
 	}
 }
 
-func NewDepsDevAPIClient(addr string) (*DepsDevAPIClient, error) {
+func NewDepsDevAPIClient(addr string, userAgent string) (*DepsDevAPIClient, error) {
 	certPool, err := x509.SystemCertPool()
 	if err != nil {
 		return nil, fmt.Errorf("getting system cert pool: %w", err)
 	}
 	creds := credentials.NewClientTLSFromCert(certPool, "")
 	dialOpts := []grpc.DialOption{grpc.WithTransportCredentials(creds)}
 
-	if osv.RequestUserAgent != "" {
-		dialOpts = append(dialOpts, grpc.WithUserAgent(osv.RequestUserAgent))
+	if userAgent != "" {
+		dialOpts = append(dialOpts, grpc.WithUserAgent(userAgent))
 	}
 
 	conn, err := grpc.NewClient(addr, dialOpts...)

internal/version/version.go

@@ -1,4 +1,4 @@
 package version
 
 // OSVVersion is the current release version, you should update this variable when doing a release
-var OSVVersion = "1.9.1"
+const OSVVersion = "1.9.1"