diff --git a/.gitignore b/.gitignore index 3b735ec..289405a 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ # Go workspace file go.work + +*~ \ No newline at end of file diff --git a/pkg/client/client.go b/pkg/client/client.go index 75dc3fc..ffdb1de 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -13,308 +13,35 @@ // limitations under the License. // Package client provides a rest client to talk to the Trusty API. +// +// Deprecated: moved to pkg/v1/client package client import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "os" - "strings" - "time" - - packageurl "github.com/package-url/packageurl-go" - khttp "sigs.k8s.io/release-utils/http" - - "github.com/stacklok/trusty-sdk-go/pkg/types" -) - -const ( - defaultEndpoint = "https://gh.trustypkg.dev" - endpointEnvVar = "TRUSTY_ENDPOINT" - reportPath = "v1/report" + v1 "github.com/stacklok/trusty-sdk-go/pkg/v1/client" ) // Options configures the Trusty API client -type Options struct { - HttpClient netClient - - // Workers is the number of parallel request the client makes to the API - Workers int - - // BaseURL of the Trusty API - BaseURL string - - // WaitForIngestion causes the http client to wait and retry if Trusty - // responds with a successful request but with a "pending" or "scoring" status - WaitForIngestion bool - - // ErrOnFailedIngestion makes the client return an error on a Report call - // when the ingestion failed internally withing trusty. If false, the - // report data willbe returned but the application needs to check the - // ingestion status and handle it. - ErrOnFailedIngestion bool - - // IngestionRetryWait is the number of seconds that the client will wait for - // package ingestion before retrying. - IngestionRetryWait int - - // IngestionMaxRetries is the maximum number of requests the client will - // send while waiting for ingestion to finish - IngestionMaxRetries int -} +// +// Deprecated: moved to pkg/v1/client +type Options = v1.Options // DefaultOptions is the default Trusty client options set -var DefaultOptions = Options{ - Workers: 2, - BaseURL: defaultEndpoint, - WaitForIngestion: true, - IngestionRetryWait: 5, -} - -type netClient interface { - GetRequestGroup([]string) ([]*http.Response, []error) - GetRequest(string) (*http.Response, error) -} +// +// Deprecated: moved to pkg/v1/client +var DefaultOptions = v1.DefaultOptions // New returns a new Trusty REST client -func New() *Trusty { - opts := DefaultOptions - opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) - if ep := os.Getenv(endpointEnvVar); ep != "" { - opts.BaseURL = ep - } - return NewWithOptions(opts) -} +// +// Deprecated: moved to pkg/v1/client +var New = v1.New // NewWithOptions returns a new client with the specified options set -func NewWithOptions(opts Options) *Trusty { - if opts.BaseURL == "" { - opts.BaseURL = DefaultOptions.BaseURL - } - - if opts.Workers == 0 { - opts.Workers = DefaultOptions.Workers - } - - if opts.HttpClient == nil { - opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) - } - - return &Trusty{ - Options: opts, - } -} - -func urlFromEndpointAndPaths( - baseUrl, endpoint string, params map[string]string, -) (*url.URL, error) { - u, err := url.Parse(baseUrl) - if err != nil { - return nil, fmt.Errorf("failed to parse endpoint: %w", err) - } - u = u.JoinPath(endpoint) - - // Add query parameters for package_name and package_type - q := u.Query() - for k, v := range params { - q.Set(k, v) - } - u.RawQuery = q.Encode() - - return u, nil -} +// +// Deprecated: moved to pkg/v1/client +var NewWithOptions = v1.NewWithOptions // Trusty is the main trusty client -type Trusty struct { - Options Options -} - -// GroupReport queries the Trusty API in parallel for a group of dependencies. -func (t *Trusty) GroupReport(_ context.Context, deps []*types.Dependency) ([]*types.Reply, error) { - urls := []string{} - for _, dep := range deps { - u, err := t.PackageEndpoint(dep) - if err != nil { - return nil, fmt.Errorf("unable to get endpoint for: %q: %w", dep.Name, err) - } - urls = append(urls, u) - } - - responses, errs := t.Options.HttpClient.GetRequestGroup(urls) - if err := errors.Join(errs...); err != nil { - return nil, fmt.Errorf("fetching data from Trusty: %w", err) - } - - // Parse the replies - resps := make([]*types.Reply, len(responses)) - for i := range responses { - defer responses[i].Body.Close() - dec := json.NewDecoder(responses[i].Body) - resps[i] = &types.Reply{} - if err := dec.Decode(resps[i]); err != nil { - return nil, fmt.Errorf("could not unmarshal response #%d: %w", i, err) - } - } - return resps, nil -} - -// PurlEndpoint returns the API endpoint url to query for data about a purl -func (t *Trusty) PurlEndpoint(purl string) (string, error) { - dep, err := t.PurlToDependency(purl) - if err != nil { - return "", fmt.Errorf("getting dependency from %q", purl) - } - ep, err := t.PackageEndpoint(dep) - if err != nil { - return "", fmt.Errorf("getting package endpoint: %w", err) - } - return ep, nil -} - -// PackageEndpoint takes a dependency and returns the Trusty endpoint to -// query data about it. -func (t *Trusty) PackageEndpoint(dep *types.Dependency) (string, error) { - // Check dependency data: - errs := []error{} - if dep.Name == "" { - errs = append(errs, fmt.Errorf("dependency has no name defined")) - } - if dep.Ecosystem.AsString() == "" { - errs = append(errs, fmt.Errorf("dependency has no ecosystem set")) - } - - if err := errors.Join(errs...); err != nil { - return "", err - } - - u, err := url.Parse(t.Options.BaseURL + "/" + reportPath) - if err != nil { - return "", fmt.Errorf("failed to parse endpoint: %w", err) - } - - params := map[string]string{ - "package_name": dep.Name, - "package_type": strings.ToLower(dep.Ecosystem.AsString()), - } - - // Add query parameters for package_name and package_type - q := u.Query() - for k, v := range params { - q.Set(k, v) - } - u.RawQuery = q.Encode() - - return u.String(), nil -} - -// PurlToEcosystem returns a trusty ecosystem constant from a Package URL's type -func (_ *Trusty) PurlToEcosystem(purl string) types.Ecosystem { - switch { - case strings.HasPrefix(purl, "pkg:golang"): - return types.ECOSYSTEM_GO - case strings.HasPrefix(purl, "pkg:npm"): - return types.ECOSYSTEM_NPM - case strings.HasPrefix(purl, "pkg:pypi"): - return types.ECOSYSTEM_PYPI - default: - return types.Ecosystem(0) - } -} - -// PurlToDependency takes a string with a package url -func (t *Trusty) PurlToDependency(purlString string) (*types.Dependency, error) { - e := t.PurlToEcosystem(purlString) - if e == 0 { - // Ecosystem nil or not supported - return nil, fmt.Errorf("ecosystem not supported") - } - - purl, err := packageurl.FromString(purlString) - if err != nil { - return nil, fmt.Errorf("unable to parse package url: %w", err) - } - name := purl.Name - if purl.Namespace != "" { - name = purl.Namespace + "/" + purl.Name - } - return &types.Dependency{ - Ecosystem: e, - Name: name, - Version: purl.Version, - }, nil -} - -// Report returns a dependency report with all the data that Trusty has -// available for a package. -func (t *Trusty) Report(_ context.Context, dep *types.Dependency) (*types.Reply, error) { - u, err := t.PackageEndpoint(dep) - if err != nil { - return nil, fmt.Errorf("computing package endpoint: %w", err) - } - - var r types.Reply - tries := 0 - for { - resp, err := t.Options.HttpClient.GetRequest(u) - if err != nil { - return nil, fmt.Errorf("could not send request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("received non-200 response: %d", resp.StatusCode) - } - - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&r); err != nil { - return nil, fmt.Errorf("could not unmarshal response: %w", err) - } - fmt.Printf("Attempt #%d to fetch package, status: %s", tries, r.PackageData.Status) - - shouldRetry, err := evalRetry(r.PackageData.Status, t.Options) - if err != nil { - return nil, err - } - - if !shouldRetry { - break - } - - tries++ - if tries > t.Options.IngestionMaxRetries { - return nil, fmt.Errorf("time out reached waiting for package ingestion") - } - time.Sleep(time.Duration(t.Options.IngestionRetryWait) * time.Second) - } - - return &r, err -} - -func evalRetry(status string, opts Options) (shouldRetry bool, err error) { - // First, error if the ingestion status is invalid - if status != types.IngestStatusFailed && status != types.IngestStatusComplete && - status != types.IngestStatusPending && status != types.IngestStatusScoring { - - return false, fmt.Errorf("unexpected ingestion status when querying package") - } - - if status == types.IngestStatusFailed && opts.ErrOnFailedIngestion { - return false, fmt.Errorf("upstream error ingesting package data") - } - - // Package ingestion is ready - if status == types.IngestStatusComplete { - return false, nil - } - - // Client configured to return raw response (even when package is not ready) - if !opts.WaitForIngestion || status == types.IngestStatusFailed { - return false, nil - } - - return true, nil -} +// +// Deprecated: moved to pkg/v1/client +type Trusty = v1.Trusty diff --git a/pkg/parser/cargo.go b/pkg/parser/cargo.go index 4772730..9aa6317 100644 --- a/pkg/parser/cargo.go +++ b/pkg/parser/cargo.go @@ -18,7 +18,7 @@ package parser import ( "github.com/BurntSushi/toml" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParseCargoToml parses the content of a Cargo.toml file and returns a slice of dependencies. diff --git a/pkg/parser/gomod.go b/pkg/parser/gomod.go index 38b8025..99f10cd 100644 --- a/pkg/parser/gomod.go +++ b/pkg/parser/gomod.go @@ -18,7 +18,7 @@ package parser import ( "strings" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParseGoMod parses the content of a go.mod file and returns a slice of dependencies. diff --git a/pkg/parser/packagejson.go b/pkg/parser/packagejson.go index 86e0da6..12f3a97 100644 --- a/pkg/parser/packagejson.go +++ b/pkg/parser/packagejson.go @@ -18,7 +18,7 @@ package parser import ( "encoding/json" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParsePackageJSON parses package.json content and extracts dependencies diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 8eadcb1..d16aa6e 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -21,7 +21,7 @@ import ( "log" "strings" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParsingFunction is a function type that takes a string as input and returns diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 67eb83e..a4e4288 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) func TestParse(t *testing.T) { diff --git a/pkg/parser/pomxml.go b/pkg/parser/pomxml.go index d870642..acd08f5 100644 --- a/pkg/parser/pomxml.go +++ b/pkg/parser/pomxml.go @@ -18,7 +18,7 @@ package parser import ( "encoding/xml" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParsePomXml parses the content of a POM XML file and returns a slice of dependencies diff --git a/pkg/parser/requirements.go b/pkg/parser/requirements.go index 9c18549..377006e 100644 --- a/pkg/parser/requirements.go +++ b/pkg/parser/requirements.go @@ -18,7 +18,7 @@ package parser import ( "strings" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) // ParseRequirementsTxt parses requirements.txt content and extracts dependencies diff --git a/pkg/types/response.go b/pkg/types/response.go index 5f86298..0f36cbf 100644 --- a/pkg/types/response.go +++ b/pkg/types/response.go @@ -14,112 +14,77 @@ package types -import "time" +import ( + v1 "github.com/stacklok/trusty-sdk-go/pkg/v1/types" +) // Reply is the response from the package report API -type Reply struct { - PackageName string `json:"package_name"` - PackageType string `json:"package_type"` - PackageVersion string `json:"package_version"` - Status string `json:"status"` - Summary ScoreSummary `json:"summary"` - Provenance *Provenance `json:"provenance"` - Activity *Activity `json:"activity"` - Typosquatting *Typosquatting `json:"typosquatting"` - Alternatives AlternativesList `json:"alternatives"` - PackageData PackageData `json:"package_data"` - SameOriginPackagesCount int `json:"same_origin_packages_count"` -} +// +// Deprecated: moved to pkg/v1/types +type Reply = v1.Reply // Activity captures a package's activity score -type Activity struct { - Score float64 `json:"score"` - Description ActivityDescription `json:"description"` -} +// +// Deprecated: moved to pkg/v1/types +type Activity = v1.Activity // ActivityDescription captures the fields of the activuty score -type ActivityDescription struct { - Repository float64 `json:"repo"` - User float64 `json:"user"` - // UpdatedAt *time.Time `json:"updated_at"` -} +// +// Deprecated: moved to pkg/v1/types +type ActivityDescription = v1.ActivityDescription // Typosquatting score for the package's name -type Typosquatting struct { - Score float64 `json:"score"` - Description TyposquattingDescription `json:"description"` - // UpdatedAt *time.Time `json:"updated_at"` -} +// +// Deprecated: moved to pkg/v1/types +type Typosquatting = v1.Typosquatting // TyposquattingDescription captures the dat details of the typosquatting score -type TyposquattingDescription struct { - TotalSimilarNames int `json:"total_similar_names"` -} +// +// Deprecated: moved to pkg/v1/types +type TyposquattingDescription = v1.TyposquattingDescription // Alternative is an alternative package returned from the package intelligence API -type Alternative struct { - PackageName string `json:"package_name"` - Score float64 `json:"score"` - PackageNameURL string -} +// +// Deprecated: moved to pkg/v1/types +type Alternative = v1.Alternative // AlternativesList is the alternatives block in the trusty API response -type AlternativesList struct { - Status string `json:"status"` - Packages []Alternative `json:"packages"` -} +// +// Deprecated: moved to pkg/v1/types +type AlternativesList = v1.AlternativesList // ScoreSummary is the summary score returned from the package intelligence API -type ScoreSummary struct { - Score *float64 `json:"score"` - Description map[string]any `json:"description"` -} +// +// Deprecated: moved to pkg/v1/types +type ScoreSummary = v1.ScoreSummary // PackageData contains the data about the queried package -type PackageData struct { - Archived bool `json:"archived"` - Deprecated bool `json:"is_deprecated"` - Malicious *MaliciousData `json:"malicious"` - Status string `json:"status"` - StatusCode string `json:"status_code"` -} +// +// Deprecated: moved to pkg/v1/types +type PackageData = v1.PackageData // MaliciousData contains the security details when a dependency is malicious -type MaliciousData struct { - Summary string `json:"summary"` - Details string `json:"details"` - Published *time.Time `json:"published"` - Modified *time.Time `json:"modified"` - Source string `json:"source"` -} +// +// Deprecated: moved to pkg/v1/types +type MaliciousData = v1.MaliciousData // Provenance has the package's provenance score and provenance type components -type Provenance struct { - Score float64 `json:"score"` - Description ProvenanceDescription `json:"description"` - // UpdatedAt *time.Time `json:"updated_at"` -} +// +// Deprecated: moved to pkg/v1/types +type Provenance = v1.Provenance // ProvenanceDescription contians the provenance types -type ProvenanceDescription struct { - Historical HistoricalProvenance `json:"hp"` - Sigstore SigstoreProvenance `json:"sigstore"` -} +// +// Deprecated: moved to pkg/v1/types +type ProvenanceDescription = v1.ProvenanceDescription // HistoricalProvenance has the historical provenance components from a package -type HistoricalProvenance struct { - Tags float64 `json:"tags"` - Common float64 `json:"common"` - Overlap float64 `json:"overlap"` - Versions float64 `json:"versions"` -} +// +// Deprecated: moved to pkg/v1/types +type HistoricalProvenance = v1.HistoricalProvenance // SigstoreProvenance has the sigstore certificate data when a package was signed // using a github actions workflow -type SigstoreProvenance struct { - Issuer string `json:"issuer"` - Workflow string `json:"workflow"` - SourceRepository string `json:"source_repo"` - TokenIssuer string `json:"token_issuer"` - Transparency string `json:"transparency"` -} +// +// Deprecated: moved to pkg/v1/types +type SigstoreProvenance = v1.SigstoreProvenance diff --git a/pkg/types/types.go b/pkg/types/types.go index 5200004..5349df7 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -14,78 +14,72 @@ // limitations under the License. // Package types is the collection of main data types used by the Trusty libraries +// +// Deprecated: moved to pkg/v1/types package types +import ( + v1 "github.com/stacklok/trusty-sdk-go/pkg/v1/types" +) + // Ecosystem is an identifier of a packaging system supported by Trusty -type Ecosystem int32 +// +// Deprecated: moved to pkg/v1/types +type Ecosystem = v1.Ecosystem // Dependency represents a generic dependency structure -type Dependency struct { - Name string - Version string - Ecosystem Ecosystem -} +// +// Deprecated: moved to pkg/v1/types +type Dependency = v1.Dependency const ( // ECOSYSTEM_NPM identifies the NPM ecosystem - ECOSYSTEM_NPM Ecosystem = 1 + // + // Deprecated: moved to pkg/v1/types + ECOSYSTEM_NPM Ecosystem = v1.ECOSYSTEM_NPM // ECOSYSTEM_GO identifies the Go language - ECOSYSTEM_GO Ecosystem = 2 + // + // Deprecated: moved to pkg/v1/types + ECOSYSTEM_GO Ecosystem = v1.ECOSYSTEM_GO // ECOSYSTEM_PYPI identifies the Python Package Index - ECOSYSTEM_PYPI Ecosystem = 3 + // + // Deprecated: moved to pkg/v1/types + ECOSYSTEM_PYPI Ecosystem = v1.ECOSYSTEM_PYPI // IngestStatusFailed ingestion failed permanently - IngestStatusFailed = "failed" + // + // Deprecated: moved to pkg/v1/types + IngestStatusFailed = v1.IngestStatusFailed // IngestStatusComplete means ingestion is done, data available - IngestStatusComplete = "complete" + // + // Deprecated: moved to pkg/v1/types + IngestStatusComplete = v1.IngestStatusComplete // IngestStatusPending means that the ingestion process is waiting to start - IngestStatusPending = "pending" + // + // Deprecated: moved to pkg/v1/types + IngestStatusPending = v1.IngestStatusPending // IngestStatusScoring means the scoring process is underway - IngestStatusScoring = "scoring" + // + // Deprecated: moved to pkg/v1/types + IngestStatusScoring = v1.IngestStatusScoring ) // Ecosystems enumerates the supported ecosystems -var Ecosystems = map[string]Ecosystem{ - "ECOSYSTEM_NPM": ECOSYSTEM_NPM, - "ECOSYSTEM_GO": ECOSYSTEM_GO, - "ECOSYSTEM_PYPI": ECOSYSTEM_PYPI, -} - -// AsString returns the string representation of the DepEcosystem -func (ecosystem Ecosystem) AsString() string { - switch ecosystem { - case ECOSYSTEM_NPM: - return "npm" - case ECOSYSTEM_GO: - return "Go" - case ECOSYSTEM_PYPI: - return "PyPI" - default: - return "" - } -} +// +// Deprecated: moved to pkg/v1/types +var Ecosystems = v1.Ecosystems // ConvertDepsToMap converts a slice of Dependency structs to a map for easier comparison -func ConvertDepsToMap(deps []Dependency) map[string]string { - depMap := make(map[string]string) - for _, dep := range deps { - depMap[dep.Name] = dep.Version - } - return depMap -} +// +// Deprecated: moved to pkg/v1/types +var ConvertDepsToMap = v1.ConvertDepsToMap // DiffDependencies compares two sets of dependencies (represented as maps) and finds what's added in newDeps. -func DiffDependencies(oldDeps, newDeps map[string]string) map[string]string { - addedDeps := make(map[string]string) - for dep, version := range newDeps { - if _, exists := oldDeps[dep]; !exists { - addedDeps[dep] = version - } - } - return addedDeps -} +// +// Deprecated: moved to pkg/v1/types +var DiffDependencies = v1.DiffDependencies diff --git a/pkg/v1/client/client.go b/pkg/v1/client/client.go new file mode 100644 index 0000000..a1336c1 --- /dev/null +++ b/pkg/v1/client/client.go @@ -0,0 +1,320 @@ +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package client provides a rest client to talk to the Trusty API. +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + + packageurl "github.com/package-url/packageurl-go" + khttp "sigs.k8s.io/release-utils/http" + + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" +) + +const ( + defaultEndpoint = "https://gh.trustypkg.dev" + endpointEnvVar = "TRUSTY_ENDPOINT" + reportPath = "v1/report" +) + +// Options configures the Trusty API client +type Options struct { + HttpClient netClient + + // Workers is the number of parallel request the client makes to the API + Workers int + + // BaseURL of the Trusty API + BaseURL string + + // WaitForIngestion causes the http client to wait and retry if Trusty + // responds with a successful request but with a "pending" or "scoring" status + WaitForIngestion bool + + // ErrOnFailedIngestion makes the client return an error on a Report call + // when the ingestion failed internally withing trusty. If false, the + // report data willbe returned but the application needs to check the + // ingestion status and handle it. + ErrOnFailedIngestion bool + + // IngestionRetryWait is the number of seconds that the client will wait for + // package ingestion before retrying. + IngestionRetryWait int + + // IngestionMaxRetries is the maximum number of requests the client will + // send while waiting for ingestion to finish + IngestionMaxRetries int +} + +// DefaultOptions is the default Trusty client options set +var DefaultOptions = Options{ + Workers: 2, + BaseURL: defaultEndpoint, + WaitForIngestion: true, + IngestionRetryWait: 5, +} + +type netClient interface { + GetRequestGroup([]string) ([]*http.Response, []error) + GetRequest(string) (*http.Response, error) +} + +// New returns a new Trusty REST client +func New() *Trusty { + opts := DefaultOptions + opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) + if ep := os.Getenv(endpointEnvVar); ep != "" { + opts.BaseURL = ep + } + return NewWithOptions(opts) +} + +// NewWithOptions returns a new client with the specified options set +func NewWithOptions(opts Options) *Trusty { + if opts.BaseURL == "" { + opts.BaseURL = DefaultOptions.BaseURL + } + + if opts.Workers == 0 { + opts.Workers = DefaultOptions.Workers + } + + if opts.HttpClient == nil { + opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) + } + + return &Trusty{ + Options: opts, + } +} + +func urlFromEndpointAndPaths( + baseUrl, endpoint string, params map[string]string, +) (*url.URL, error) { + u, err := url.Parse(baseUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint: %w", err) + } + u = u.JoinPath(endpoint) + + // Add query parameters for package_name and package_type + q := u.Query() + for k, v := range params { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + return u, nil +} + +// Trusty is the main trusty client +type Trusty struct { + Options Options +} + +// GroupReport queries the Trusty API in parallel for a group of dependencies. +func (t *Trusty) GroupReport(_ context.Context, deps []*types.Dependency) ([]*types.Reply, error) { + urls := []string{} + for _, dep := range deps { + u, err := t.PackageEndpoint(dep) + if err != nil { + return nil, fmt.Errorf("unable to get endpoint for: %q: %w", dep.Name, err) + } + urls = append(urls, u) + } + + responses, errs := t.Options.HttpClient.GetRequestGroup(urls) + if err := errors.Join(errs...); err != nil { + return nil, fmt.Errorf("fetching data from Trusty: %w", err) + } + + // Parse the replies + resps := make([]*types.Reply, len(responses)) + for i := range responses { + defer responses[i].Body.Close() + dec := json.NewDecoder(responses[i].Body) + resps[i] = &types.Reply{} + if err := dec.Decode(resps[i]); err != nil { + return nil, fmt.Errorf("could not unmarshal response #%d: %w", i, err) + } + } + return resps, nil +} + +// PurlEndpoint returns the API endpoint url to query for data about a purl +func (t *Trusty) PurlEndpoint(purl string) (string, error) { + dep, err := t.PurlToDependency(purl) + if err != nil { + return "", fmt.Errorf("getting dependency from %q", purl) + } + ep, err := t.PackageEndpoint(dep) + if err != nil { + return "", fmt.Errorf("getting package endpoint: %w", err) + } + return ep, nil +} + +// PackageEndpoint takes a dependency and returns the Trusty endpoint to +// query data about it. +func (t *Trusty) PackageEndpoint(dep *types.Dependency) (string, error) { + // Check dependency data: + errs := []error{} + if dep.Name == "" { + errs = append(errs, fmt.Errorf("dependency has no name defined")) + } + if dep.Ecosystem.AsString() == "" { + errs = append(errs, fmt.Errorf("dependency has no ecosystem set")) + } + + if err := errors.Join(errs...); err != nil { + return "", err + } + + u, err := url.Parse(t.Options.BaseURL + "/" + reportPath) + if err != nil { + return "", fmt.Errorf("failed to parse endpoint: %w", err) + } + + params := map[string]string{ + "package_name": dep.Name, + "package_type": strings.ToLower(dep.Ecosystem.AsString()), + } + + // Add query parameters for package_name and package_type + q := u.Query() + for k, v := range params { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + return u.String(), nil +} + +// PurlToEcosystem returns a trusty ecosystem constant from a Package URL's type +func (_ *Trusty) PurlToEcosystem(purl string) types.Ecosystem { + switch { + case strings.HasPrefix(purl, "pkg:golang"): + return types.ECOSYSTEM_GO + case strings.HasPrefix(purl, "pkg:npm"): + return types.ECOSYSTEM_NPM + case strings.HasPrefix(purl, "pkg:pypi"): + return types.ECOSYSTEM_PYPI + default: + return types.Ecosystem(0) + } +} + +// PurlToDependency takes a string with a package url +func (t *Trusty) PurlToDependency(purlString string) (*types.Dependency, error) { + e := t.PurlToEcosystem(purlString) + if e == 0 { + // Ecosystem nil or not supported + return nil, fmt.Errorf("ecosystem not supported") + } + + purl, err := packageurl.FromString(purlString) + if err != nil { + return nil, fmt.Errorf("unable to parse package url: %w", err) + } + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + purl.Name + } + return &types.Dependency{ + Ecosystem: e, + Name: name, + Version: purl.Version, + }, nil +} + +// Report returns a dependency report with all the data that Trusty has +// available for a package. +func (t *Trusty) Report(_ context.Context, dep *types.Dependency) (*types.Reply, error) { + u, err := t.PackageEndpoint(dep) + if err != nil { + return nil, fmt.Errorf("computing package endpoint: %w", err) + } + + var r types.Reply + tries := 0 + for { + resp, err := t.Options.HttpClient.GetRequest(u) + if err != nil { + return nil, fmt.Errorf("could not send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received non-200 response: %d", resp.StatusCode) + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&r); err != nil { + return nil, fmt.Errorf("could not unmarshal response: %w", err) + } + fmt.Printf("Attempt #%d to fetch package, status: %s", tries, r.PackageData.Status) + + shouldRetry, err := evalRetry(r.PackageData.Status, t.Options) + if err != nil { + return nil, err + } + + if !shouldRetry { + break + } + + tries++ + if tries > t.Options.IngestionMaxRetries { + return nil, fmt.Errorf("time out reached waiting for package ingestion") + } + time.Sleep(time.Duration(t.Options.IngestionRetryWait) * time.Second) + } + + return &r, err +} + +func evalRetry(status string, opts Options) (shouldRetry bool, err error) { + // First, error if the ingestion status is invalid + if status != types.IngestStatusFailed && status != types.IngestStatusComplete && + status != types.IngestStatusPending && status != types.IngestStatusScoring { + + return false, fmt.Errorf("unexpected ingestion status when querying package") + } + + if status == types.IngestStatusFailed && opts.ErrOnFailedIngestion { + return false, fmt.Errorf("upstream error ingesting package data") + } + + // Package ingestion is ready + if status == types.IngestStatusComplete { + return false, nil + } + + // Client configured to return raw response (even when package is not ready) + if !opts.WaitForIngestion || status == types.IngestStatusFailed { + return false, nil + } + + return true, nil +} diff --git a/pkg/client/client_test.go b/pkg/v1/client/client_test.go similarity index 99% rename from pkg/client/client_test.go rename to pkg/v1/client/client_test.go index 13e3bb4..199235c 100644 --- a/pkg/client/client_test.go +++ b/pkg/v1/client/client_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/stacklok/trusty-sdk-go/pkg/types" + "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) func newFakeClient() *fakeClient { diff --git a/pkg/v1/types/response.go b/pkg/v1/types/response.go new file mode 100644 index 0000000..5f86298 --- /dev/null +++ b/pkg/v1/types/response.go @@ -0,0 +1,125 @@ +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "time" + +// Reply is the response from the package report API +type Reply struct { + PackageName string `json:"package_name"` + PackageType string `json:"package_type"` + PackageVersion string `json:"package_version"` + Status string `json:"status"` + Summary ScoreSummary `json:"summary"` + Provenance *Provenance `json:"provenance"` + Activity *Activity `json:"activity"` + Typosquatting *Typosquatting `json:"typosquatting"` + Alternatives AlternativesList `json:"alternatives"` + PackageData PackageData `json:"package_data"` + SameOriginPackagesCount int `json:"same_origin_packages_count"` +} + +// Activity captures a package's activity score +type Activity struct { + Score float64 `json:"score"` + Description ActivityDescription `json:"description"` +} + +// ActivityDescription captures the fields of the activuty score +type ActivityDescription struct { + Repository float64 `json:"repo"` + User float64 `json:"user"` + // UpdatedAt *time.Time `json:"updated_at"` +} + +// Typosquatting score for the package's name +type Typosquatting struct { + Score float64 `json:"score"` + Description TyposquattingDescription `json:"description"` + // UpdatedAt *time.Time `json:"updated_at"` +} + +// TyposquattingDescription captures the dat details of the typosquatting score +type TyposquattingDescription struct { + TotalSimilarNames int `json:"total_similar_names"` +} + +// Alternative is an alternative package returned from the package intelligence API +type Alternative struct { + PackageName string `json:"package_name"` + Score float64 `json:"score"` + PackageNameURL string +} + +// AlternativesList is the alternatives block in the trusty API response +type AlternativesList struct { + Status string `json:"status"` + Packages []Alternative `json:"packages"` +} + +// ScoreSummary is the summary score returned from the package intelligence API +type ScoreSummary struct { + Score *float64 `json:"score"` + Description map[string]any `json:"description"` +} + +// PackageData contains the data about the queried package +type PackageData struct { + Archived bool `json:"archived"` + Deprecated bool `json:"is_deprecated"` + Malicious *MaliciousData `json:"malicious"` + Status string `json:"status"` + StatusCode string `json:"status_code"` +} + +// MaliciousData contains the security details when a dependency is malicious +type MaliciousData struct { + Summary string `json:"summary"` + Details string `json:"details"` + Published *time.Time `json:"published"` + Modified *time.Time `json:"modified"` + Source string `json:"source"` +} + +// Provenance has the package's provenance score and provenance type components +type Provenance struct { + Score float64 `json:"score"` + Description ProvenanceDescription `json:"description"` + // UpdatedAt *time.Time `json:"updated_at"` +} + +// ProvenanceDescription contians the provenance types +type ProvenanceDescription struct { + Historical HistoricalProvenance `json:"hp"` + Sigstore SigstoreProvenance `json:"sigstore"` +} + +// HistoricalProvenance has the historical provenance components from a package +type HistoricalProvenance struct { + Tags float64 `json:"tags"` + Common float64 `json:"common"` + Overlap float64 `json:"overlap"` + Versions float64 `json:"versions"` +} + +// SigstoreProvenance has the sigstore certificate data when a package was signed +// using a github actions workflow +type SigstoreProvenance struct { + Issuer string `json:"issuer"` + Workflow string `json:"workflow"` + SourceRepository string `json:"source_repo"` + TokenIssuer string `json:"token_issuer"` + Transparency string `json:"transparency"` +} diff --git a/pkg/v1/types/types.go b/pkg/v1/types/types.go new file mode 100644 index 0000000..5200004 --- /dev/null +++ b/pkg/v1/types/types.go @@ -0,0 +1,91 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types is the collection of main data types used by the Trusty libraries +package types + +// Ecosystem is an identifier of a packaging system supported by Trusty +type Ecosystem int32 + +// Dependency represents a generic dependency structure +type Dependency struct { + Name string + Version string + Ecosystem Ecosystem +} + +const ( + // ECOSYSTEM_NPM identifies the NPM ecosystem + ECOSYSTEM_NPM Ecosystem = 1 + + // ECOSYSTEM_GO identifies the Go language + ECOSYSTEM_GO Ecosystem = 2 + + // ECOSYSTEM_PYPI identifies the Python Package Index + ECOSYSTEM_PYPI Ecosystem = 3 + + // IngestStatusFailed ingestion failed permanently + IngestStatusFailed = "failed" + + // IngestStatusComplete means ingestion is done, data available + IngestStatusComplete = "complete" + + // IngestStatusPending means that the ingestion process is waiting to start + IngestStatusPending = "pending" + + // IngestStatusScoring means the scoring process is underway + IngestStatusScoring = "scoring" +) + +// Ecosystems enumerates the supported ecosystems +var Ecosystems = map[string]Ecosystem{ + "ECOSYSTEM_NPM": ECOSYSTEM_NPM, + "ECOSYSTEM_GO": ECOSYSTEM_GO, + "ECOSYSTEM_PYPI": ECOSYSTEM_PYPI, +} + +// AsString returns the string representation of the DepEcosystem +func (ecosystem Ecosystem) AsString() string { + switch ecosystem { + case ECOSYSTEM_NPM: + return "npm" + case ECOSYSTEM_GO: + return "Go" + case ECOSYSTEM_PYPI: + return "PyPI" + default: + return "" + } +} + +// ConvertDepsToMap converts a slice of Dependency structs to a map for easier comparison +func ConvertDepsToMap(deps []Dependency) map[string]string { + depMap := make(map[string]string) + for _, dep := range deps { + depMap[dep.Name] = dep.Version + } + return depMap +} + +// DiffDependencies compares two sets of dependencies (represented as maps) and finds what's added in newDeps. +func DiffDependencies(oldDeps, newDeps map[string]string) map[string]string { + addedDeps := make(map[string]string) + for dep, version := range newDeps { + if _, exists := oldDeps[dep]; !exists { + addedDeps[dep] = version + } + } + return addedDeps +} diff --git a/pkg/types/types_test.go b/pkg/v1/types/types_test.go similarity index 100% rename from pkg/types/types_test.go rename to pkg/v1/types/types_test.go