From 4204413d736f6b74cb0807ce7ab79b75e4713814 Mon Sep 17 00:00:00 2001 From: ParthaI <47887552+ParthaI@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:22:32 +0530 Subject: [PATCH] Add table gcp_artifact_registry_repository Closes #473 (#496) Co-authored-by: Madhushree Ray --- .../gcp_artifact_registry_repository.md | 103 ++++++ gcp/artifact_registry_location_list.go | 55 ++++ gcp/plugin.go | 1 + gcp/service.go | 24 +- gcp/table_gcp_artifact_registry_repository.go | 305 ++++++++++++++++++ 5 files changed, 487 insertions(+), 1 deletion(-) create mode 100644 docs/tables/gcp_artifact_registry_repository.md create mode 100644 gcp/artifact_registry_location_list.go create mode 100644 gcp/table_gcp_artifact_registry_repository.go diff --git a/docs/tables/gcp_artifact_registry_repository.md b/docs/tables/gcp_artifact_registry_repository.md new file mode 100644 index 00000000..b83c5206 --- /dev/null +++ b/docs/tables/gcp_artifact_registry_repository.md @@ -0,0 +1,103 @@ +# Table: gcp_artifact_registry_repository + +Google Cloud Artifact Registry is a managed artifact repository service provided by Google Cloud Platform (GCP). It is designed to store and manage software packages, container images, and other development artifacts. Artifact Registry helps organizations manage their dependencies, share software artifacts, and ensure the reliability and security of their software supply chain. + +### Basic info + +```sql +select + name, + cleanup_policy_dry_run, + create_time, + format, + kms_key_name, + mode +from + gcp_artifact_registry_repository; +``` + +### List unencrypted repositories + +```sql +select + name, + cleanup_policy_dry_run, + create_time, + kms_key_name +from + gcp_artifact_registry_repository +where + kms_key_name = ''; +``` + +### List docker format package repositories + +```sql +select + name, + create_time, + description, + size_bytes, + format +from + gcp_artifact_registry_repository +where + format = 'DOCKER'; +``` + +### List standard repositories + +```sql +select + name, + format, + mode, + create_time +from + gcp_artifact_registry_repository +where + mode = 'STANDARD_REPOSITORY'; +``` + +### List repositories that satisfies physical zone separation + +```sql +select + name, + mode, + format, + satisfies_pzs, + description, + create_time +from + gcp_artifact_registry_repository +where + satisfies_pzs; +``` + +### Get docker configuration of repositories + +```sql +select + name, + docker_config -> 'ImmutableTags' as immutable_tags, + docker_config ->> 'ForceSendFields' as force_send_fields, + docker_config ->> 'NullFields' as null_fields +from + gcp_artifact_registry_repository; +``` + +### Get remote repository config details of repositories + +```sql +select + name, + remote_repository_config ->> 'AptRepository' as apt_repository, + remote_repository_config ->> 'DockerRepository' as docker_repository, + remote_repository_config ->> 'MavenRepository' as maven_repository, + remote_repository_config ->> 'NpmRepository' as npm_repository, + remote_repository_config ->> 'PythonRepository' as python_repository, + remote_repository_config ->> 'YumRepository' as yum_repository +from + gcp_artifact_registry_repository; +``` \ No newline at end of file diff --git a/gcp/artifact_registry_location_list.go b/gcp/artifact_registry_location_list.go new file mode 100644 index 00000000..96af882b --- /dev/null +++ b/gcp/artifact_registry_location_list.go @@ -0,0 +1,55 @@ +package gcp + +import ( + "context" + + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "google.golang.org/api/artifactregistry/v1" +) + + +// BuildregionList :: return a list of matrix items, one per region specified +func BuildArtifactRegistryLocationList(ctx context.Context, d *plugin.QueryData) []map[string]interface{} { + + // have we already created and cached the locations? + locationCacheKey := "ArtifactRegistry" + if cachedData, ok := d.ConnectionManager.Cache.Get(locationCacheKey); ok { + plugin.Logger(ctx).Trace("listlocationDetails:", cachedData.([]map[string]interface{})) + return cachedData.([]map[string]interface{}) + } + + // Create Service Connection + service, err := ArtifactRegistryService(ctx, d) + if err != nil { + return nil + } + + // Get project details + projectData, err := activeProject(ctx, d) + if err != nil { + return nil + } + project := projectData.Project + + resp := service.Projects.Locations.List("projects/" + project) + if err != nil { + return nil + } + + var locations []*artifactregistry.Location + + if err := resp.Pages(ctx, func(page *artifactregistry.ListLocationsResponse) error { + locations = append(locations, page.Locations...) + return nil + }); err != nil { + return nil + } + + // validate location list + matrix := make([]map[string]interface{}, len(locations)) + for i, location := range locations { + matrix[i] = map[string]interface{}{matrixKeyLocation: location.LocationId} + } + d.ConnectionManager.Cache.Set(locationCacheKey, matrix) + return matrix +} diff --git a/gcp/plugin.go b/gcp/plugin.go index 3dc685b1..e39c9222 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -33,6 +33,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { }, TableMap: map[string]*plugin.Table{ "gcp_apikeys_key": tableGcpApiKeysKey(ctx), + "gcp_artifact_registry_repository": tableGcpArtifactRegistryRepository(ctx), "gcp_audit_policy": tableGcpAuditPolicy(ctx), "gcp_bigquery_dataset": tableGcpBigQueryDataset(ctx), "gcp_bigquery_job": tableGcpBigQueryJob(ctx), diff --git a/gcp/service.go b/gcp/service.go index 9b3627fc..b7ab55ea 100644 --- a/gcp/service.go +++ b/gcp/service.go @@ -3,10 +3,11 @@ package gcp import ( "context" - redis "cloud.google.com/go/redis/apiv1" + "cloud.google.com/go/redis/apiv1" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "google.golang.org/api/accessapproval/v1" "google.golang.org/api/apikeys/v2" + "google.golang.org/api/artifactregistry/v1" "google.golang.org/api/bigquery/v2" "google.golang.org/api/bigtableadmin/v2" "google.golang.org/api/billingbudgets/v1" @@ -136,6 +137,27 @@ func BigQueryService(ctx context.Context, d *plugin.QueryData) (*bigquery.Servic return svc, nil } +// ArtifactRegistryService returns the service connection for GCP ArtifactRegistry service +func ArtifactRegistryService(ctx context.Context, d *plugin.QueryData) (*artifactregistry.Service, error) { + // have we already created and cached the service? + serviceCacheKey := "ArtifactRegistryService" + if cachedData, ok := d.ConnectionManager.Cache.Get(serviceCacheKey); ok { + return cachedData.(*artifactregistry.Service), nil + } + + // To get config arguments from plugin config file + opts := setSessionConfig(ctx, d.Connection) + + // so it was not in cache - create service + svc, err := artifactregistry.NewService(ctx, opts...) + if err != nil { + return nil, err + } + + d.ConnectionManager.Cache.Set(serviceCacheKey, svc) + return svc, nil +} + // BigtableAdminService returns the service connection for GCP Bigtable Admin service func BigtableAdminService(ctx context.Context, d *plugin.QueryData) (*bigtableadmin.Service, error) { // have we already created and cached the service? diff --git a/gcp/table_gcp_artifact_registry_repository.go b/gcp/table_gcp_artifact_registry_repository.go new file mode 100644 index 00000000..a4ec5380 --- /dev/null +++ b/gcp/table_gcp_artifact_registry_repository.go @@ -0,0 +1,305 @@ +package gcp + +import ( + "context" + "strings" + + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + + "google.golang.org/api/artifactregistry/v1" +) + +//// TABLE DEFINITION + +func tableGcpArtifactRegistryRepository(ctx context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_artifact_registry_repository", + Description: "GCP Artifact Registry Repository", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "location"}), + Hydrate: getArtifactRegistryRepository, + }, + List: &plugin.ListConfig{ + Hydrate: listArtifactRegistryRepositories, + KeyColumns: plugin.KeyColumnSlice{ + { + Name: "location", + Require: plugin.Optional, + }, + }, + }, + GetMatrixItemFunc: BuildArtifactRegistryLocationList, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The name of the repository.", + Type: proto.ColumnType_STRING, + Transform: transform.FromP(artifactRegistryRepositoryData, "Title"), + }, + { + Name: "cleanup_policy_dry_run", + Description: "If true, the cleanup pipeline is prevented from deleting versions in this repository.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "create_time", + Description: "The time when the repository was created.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "description", + Description: "The user-provided description of the repository.", + Type: proto.ColumnType_STRING, + }, + { + Name: "format", + Description: "The format of packages that are stored in the repository.", + Type: proto.ColumnType_STRING, + }, + { + Name: "kms_key_name", + Description: "The Cloud KMS resource name of the customer managed encryption key that's used to encrypt the contents of the Repository.", + Type: proto.ColumnType_STRING, + }, + { + Name: "mode", + Description: "The mode of the repository.", + Type: proto.ColumnType_STRING, + }, + { + Name: "satisfies_pzs", + Description: "If set, the repository satisfies physical zone separation.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "size_bytes", + Description: "The size, in bytes, of all artifact storage in this repository.", + Type: proto.ColumnType_INT, + }, + { + Name: "update_time", + Description: "The time when the repository was last updated.", + Type: proto.ColumnType_TIMESTAMP, + }, + + // JSON field + { + Name: "cleanup_policies", + Description: "Cleanup policies for this repository.", + Type: proto.ColumnType_JSON, + }, + { + Name: "docker_config", + Description: "Docker repository config contains repository level configuration for the repositories of docker type.", + Type: proto.ColumnType_JSON, + }, + { + Name: "maven_config", + Description: "Maven repository config contains repository level configuration for the repositories of maven type.", + Type: proto.ColumnType_JSON, + }, + { + Name: "remote_repository_config", + Description: "Configuration specific for a Remote Repository.", + Type: proto.ColumnType_JSON, + }, + { + Name: "sbom_config", + Description: "Config and state for sbom generation for resources within this Repository.", + Type: proto.ColumnType_JSON, + }, + { + Name: "virtual_repository_config", + Description: "Configuration specific for a Virtual Repository.", + Type: proto.ColumnType_JSON, + }, + { + Name: "self_link", + Description: "An URL that can be used to access the resource again.", + Type: proto.ColumnType_STRING, + Hydrate: artifactRegistryRepositorySelfLink, + Transform: transform.FromValue(), + }, + { + Name: "labels", + Description: "A set of labels associated with this repository.", + Type: proto.ColumnType_JSON, + }, + + // standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(artifactRegistryRepositoryData, "Title"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Labels"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromP(artifactRegistryRepositoryData, "Akas"), + }, + + // Standard GCP columns + { + Name: "location", + Description: ColumnDescriptionLocation, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(artifactRegistryRepositoryData, "Location"), + }, + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(artifactRegistryRepositoryData, "Project"), + }, + }, + } +} + +//// LIST FUNCTION + +func listArtifactRegistryRepositories(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + region := d.EqualsQualString("location") + + var location string + matrixLocation := d.EqualsQualString(matrixKeyLocation) + // Since, when the service API is disabled, matrixLocation value will be nil + if matrixLocation != "" { + location = matrixLocation + } + + // Minimize API call for given location + if region != "" && region != location { + return nil, nil + } + + // Create Service Connection + service, err := ArtifactRegistryService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_artifact_registry_repository.listArtifactRegistryRepositories", "service_error", err) + return nil, err + } + + // Max limit isn't mentioned in the documentation + // Default limit is set as 1000 + pageSize := types.Int64(1000) + limit := d.QueryContext.Limit + if d.QueryContext.Limit != nil { + if *limit < *pageSize { + pageSize = limit + } + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + data := "projects/" + project + "/locations/" + location + + resp := service.Projects.Locations.Repositories.List(data).PageSize(*pageSize) + if err := resp.Pages(ctx, func(page *artifactregistry.ListRepositoriesResponse) error { + for _, repo := range page.Repositories { + d.StreamListItem(ctx, repo) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + page.NextPageToken = "" + return nil + } + } + return nil + }); err != nil { + plugin.Logger(ctx).Error("gcp_artifact_registry_repository.listArtifactRegistryRepositories", "api_error", err) + return nil, err + } + + return nil, nil +} + +//// HYDRATE FUNCTIONS + +func getArtifactRegistryRepository(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := ArtifactRegistryService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_artifact_registry_repository.getArtifactRegistryRepository", "service_error", err) + return nil, err + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + name := d.EqualsQuals["name"].GetStringValue() + location := d.EqualsQuals["location"].GetStringValue() + + // check if name or location is empty + if name == "" || location == "" { + return nil, nil + } + + resp, err := service.Projects.Locations.Repositories.Get("projects/" + project + "/locations/" + location + "/repositories/" + name).Do() + if err != nil { + plugin.Logger(ctx).Error("gcp_artifact_registry_repository.getArtifactRegistryRepository", "api_error", err) + return nil, err + } + + return resp, nil +} + +func artifactRegistryRepositorySelfLink(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + data := h.Item.(*artifactregistry.Repository) + + var location string + matrixLocation := d.EqualsQualString(matrixKeyLocation) + // Since, when the service API is disabled, matrixLocation value will be nil + if matrixLocation != "" { + location = matrixLocation + } + + projectID := strings.Split(data.Name, "/")[1] + name := strings.Split(data.Name, "/")[5] + + selfLink := "https://artifactregistry.googleapis.com/v1/projects/" + projectID + "/regions/" + location + "/repositories/" + name + + return selfLink, nil +} + +//// TRANSFORM FUNCTIONS + +func artifactRegistryRepositoryData(ctx context.Context, h *transform.TransformData) (interface{}, error) { + data := h.HydrateItem.(*artifactregistry.Repository) + param := h.Param.(string) + + projectID := strings.Split(data.Name, "/")[1] + name := strings.Split(data.Name, "/")[5] + location := strings.Split(data.Name, "/")[3] + + turbotData := map[string]interface{}{ + "Project": projectID, + "Title": name, + "Location": location, + "Akas": []string{"gcp://artifactregistry.googleapis.com/projects/" + projectID + "/repositories/" + name}, + } + + return turbotData[param], nil +}