From 8029e576d58b696ea57032dc7a8c942007b96f6a Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Thu, 22 Jun 2023 20:09:55 +0530 Subject: [PATCH 1/5] Add table `gcp_storage_object`. Closes #459 --- docs/tables/gcp_storage_object.md | 75 +++++++ gcp/plugin.go | 1 + gcp/table_gcp_storage_object.go | 342 ++++++++++++++++++++++++++++++ 3 files changed, 418 insertions(+) create mode 100644 docs/tables/gcp_storage_object.md create mode 100644 gcp/table_gcp_storage_object.go diff --git a/docs/tables/gcp_storage_object.md b/docs/tables/gcp_storage_object.md new file mode 100644 index 00000000..f348b26b --- /dev/null +++ b/docs/tables/gcp_storage_object.md @@ -0,0 +1,75 @@ +# Table: gcp_storage_object + +Storage buckets are the basic containers that hold data. Everything that you store in cloud Storage must be contained in a bucket. + +## Examples + +### Basic info + +```sql +select + id, + name, + bucket, + content_type, + generation, + size, + storage_class, + time_created, + owner +from + gcp_storage_object; +``` + +### List storage objects encrypted with customer managed keys + +```sql +select + id, + name, + bucket +from + gcp_storage_object +where + not kms_key_name is not null; +``` + +### Get total objects and size of each bucket + +```sql +select + bucket, + count(*) as total_objects, + sum(size) as total_size_bytes +from + gcp_storage_object +group by + bucket; +``` + +### List of members and their associated iam roles for each objects + +```sql +select + bucket, + name, + p -> 'members' as member, + p ->> 'role' as role, + p ->> 'version' as version +from + gcp_storage_object, + jsonb_array_elements(iam_policy -> 'bindings') as p; +``` + +### List of storage objects whose retention period is less than 7 days + +```sql +select + bucket, + name, + extract(epoch from (retention_expiration_time - current_timestamp)) as retention_period_secs +from + gcp_storage_object +where + extract(epoch from (retention_expiration_time - current_timestamp)) < 604800; +``` \ No newline at end of file diff --git a/gcp/plugin.go b/gcp/plugin.go index 38f8bb58..1bfa2e09 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -119,6 +119,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_sql_database_instance_metric_cpu_utilization_daily": tableGcpSQLDatabaseInstanceMetricCpuUtilizationDaily(ctx), "gcp_sql_database_instance_metric_cpu_utilization_hourly": tableGcpSQLDatabaseInstanceMetricCpuUtilizationHourly(ctx), "gcp_storage_bucket": tableGcpStorageBucket(ctx), + "gcp_storage_object": tableGcpStorageObject(ctx), /* https://github.com/turbot/steampipe/issues/108 "gcp_compute_route": tableGcpComputeRoute(ctx), diff --git a/gcp/table_gcp_storage_object.go b/gcp/table_gcp_storage_object.go new file mode 100644 index 00000000..44b25b95 --- /dev/null +++ b/gcp/table_gcp_storage_object.go @@ -0,0 +1,342 @@ +package gcp + +import ( + "context" + + "github.com/turbot/go-kit/helpers" + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + "google.golang.org/api/storage/v1" +) + +func tableGcpStorageObject(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_storage_object", + Description: "GCP Storage Object", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"bucket", "name"}), + Hydrate: getStorageObject, + }, + List: &plugin.ListConfig{ + KeyColumns: plugin.OptionalColumns([]string{"bucket"}), + ParentHydrate: listGcpStorageBuckets, + Hydrate: listStorageObjects, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The name of the object.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The ID of the object, including the bucket name, object name, and generation number.", + Type: proto.ColumnType_STRING, + }, + { + Name: "bucket", + Description: "The name of the bucket containing this object.", + Type: proto.ColumnType_STRING, + }, + { + Name: "cache_control", + Description: "Cache-Control directive for the object data.", + Type: proto.ColumnType_STRING, + }, + { + Name: "component_count", + Description: "Number of underlying components that make up this object.", + Type: proto.ColumnType_INT, + }, + { + Name: "content_disposition", + Description: "Content-Disposition of the object data.", + Type: proto.ColumnType_STRING, + }, + { + Name: "content_encoding", + Description: "Content-Encoding of the object data.", + Type: proto.ColumnType_STRING, + }, + { + Name: "content_language", + Description: "Content-Language of the object data.", + Type: proto.ColumnType_STRING, + }, + { + Name: "content_type", + Description: "Content-Type of the object data.", + Type: proto.ColumnType_STRING, + }, + { + Name: "crc32c", + Description: "CRC32c checksum, as described in RFC 4960, Appendix B; encoded using base64 in big-endian byte order.", + Transform: transform.FromField("Crc32c"), + Type: proto.ColumnType_STRING, + }, + { + Name: "custom_time", + Description: "A timestamp in RFC 3339 format specified by the user for an object", + Transform: transform.FromGo().NullIfZero(), + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "etag", + Description: "Entity tag for the object", + Type: proto.ColumnType_STRING, + }, + { + Name: "event_based_hold", + Description: "Whether or not the object is under event-based hold.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "generation", + Description: "The content generation of this object.", + Type: proto.ColumnType_INT, + }, + { + Name: "kind", + Description: "The kind of item this is.", + Type: proto.ColumnType_STRING, + }, + { + Name: "kms_key_name", + Description: "Cloud KMS Key used to encrypt this object, if the object is encrypted by such a key.", + Type: proto.ColumnType_STRING, + }, + { + Name: "md5_hash", + Description: "MD5 hash of the data; encoded using base64", + Transform: transform.FromField("Md5Hash"), + Type: proto.ColumnType_STRING, + }, + { + Name: "media_link", + Description: "Media download link", + Type: proto.ColumnType_STRING, + }, + { + Name: "metadata", + Description: "User-provided metadata, in key/value pairs.", + Type: proto.ColumnType_STRING, + }, + { + Name: "metageneration", + Description: "The version of the metadata for this object at this generation.", + Type: proto.ColumnType_INT, + }, + { + Name: "retention_expiration_time", + Description: "A server-determined value that specifies the earliest time that the object's retention period expires.", + Transform: transform.FromGo().NullIfZero(), + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "self_link", + Description: "The link to this object.", + Type: proto.ColumnType_STRING, + }, + { + Name: "size", + Description: "Content-Length of the data in bytes.", + Type: proto.ColumnType_INT, + }, + { + Name: "storage_class", + Description: "Storage class of the object.", + Type: proto.ColumnType_STRING, + }, + { + Name: "temporary_hold", + Description: "Whether or not the object is under temporary hold.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "time_created", + Description: "The creation time of the object.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "time_deleted", + Description: "The deletion time of the object.", + Transform: transform.FromGo().NullIfZero(), + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "time_storage_class_updated", + Description: "The time at which the object's storage class was last changed.", + Transform: transform.FromGo().NullIfZero(), + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "updated", + Description: "The modification time of the object metadata.", + Transform: transform.FromGo().NullIfZero(), + Type: proto.ColumnType_TIMESTAMP, + }, + + // JSON fields + { + Name: "acl", + Description: "Access controls on the object.", + Type: proto.ColumnType_JSON, + }, + { + Name: "customer_encryption", + Description: "Metadata of customer-supplied encryption key, if the object is encrypted by such a key", + Type: proto.ColumnType_JSON, + }, + { + Name: "owner", + Description: "The owner of the object. This will always be the uploader of the object.", + Type: proto.ColumnType_JSON, + }, + { + Name: "iam_policy", + Description: "An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources. A `Policy` is a collection of `bindings`. A `binding` binds one or more `members` to a single `role`. Members can be user accounts, service accounts, Google groups, and domains (such as G Suite). A `role` is a named list of permissions; each `role` can be an IAM predefined role or a user-created custom role. For some types of Google Cloud resources, a `binding` can also specify a `condition`, which is a logical expression that allows access to a resource only if the expression evaluates to `true`.", + Hydrate: getStorageObjectIAMPolicy, + Transform: transform.FromValue(), + Type: proto.ColumnType_JSON, + }, + + // standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Hydrate: getObjectAka, + Transform: transform.FromValue(), + }, + + // standard GCP columns + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Hydrate: plugin.HydrateFunc(getProject).WithCache(), + Transform: transform.FromValue(), + }, + }, + } +} + +//// LIST FUNCTION + +func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + bucket := h.Item.(*storage.Bucket).Name + + if d.EqualsQuals["bucket"] != nil && d.EqualsQualString("bucket") != bucket { + return nil, nil + } + + service, err := StorageService(ctx, d) + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "connection_error", err) + return nil, err + } + + projection := "noAcl" + if helpers.StringSliceContains(d.QueryContext.Columns, "acl") { + projection = "full" + } + + // Max limit isn't mentioned in the documentation + // Default limit is set as 1000 + maxResults := types.Int64(1000) + limit := d.QueryContext.Limit + if d.QueryContext.Limit != nil { + if *limit < *maxResults { + maxResults = limit + } + } + + resp := service.Objects.List(bucket).Projection(projection).MaxResults(*maxResults) + if err := resp.Pages(ctx, func(page *storage.Objects) error { + for _, object := range page.Items { + d.StreamListItem(ctx, object) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + page.NextPageToken = "" + break + } + } + return nil + }); err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "api_error", err) + return nil, err + } + + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + bucket := d.EqualsQuals["bucket"].GetStringValue() + name := d.EqualsQuals["name"].GetStringValue() + + // Return nil, if input parameters are empty + if bucket == "" || name == "" { + return nil, nil + } + + service, err := StorageService(ctx, d) + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "connection_error", err) + return nil, err + } + + req, err := service.Objects.Get(bucket, name).Do() + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "api_error", err) + return nil, err + } + + return req, nil +} + +func getStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + object := h.Item.(*storage.Object) + + // Create Session + service, err := StorageService(ctx, d) + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "connection_error", err) + return nil, err + } + + resp, err := service.Objects.GetIamPolicy(object.Bucket, object.Name).Do() + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "api_error", err) + return nil, err + } + + return resp, nil +} + +func getObjectAka(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + object := h.Item.(*storage.Object) + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + plugin.Logger(ctx).Trace("gcp_storage_object.getObjectAka", "cache_error", err) + return nil, err + } + project := projectId.(string) + + akas := []string{"gcp://storage.googleapis.com/projects/" + project + "/buckets/" + object.Bucket + "/objects/" + object.Name} + return akas, nil +} From 18d7fed6594b482d114eb3f77cb89c1c4c2f5a33 Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Thu, 22 Jun 2023 20:26:37 +0530 Subject: [PATCH 2/5] Ignore iam policy error --- gcp/table_gcp_storage_object.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gcp/table_gcp_storage_object.go b/gcp/table_gcp_storage_object.go index 44b25b95..5e40f6a6 100644 --- a/gcp/table_gcp_storage_object.go +++ b/gcp/table_gcp_storage_object.go @@ -2,12 +2,14 @@ package gcp import ( "context" + "strings" "github.com/turbot/go-kit/helpers" "github.com/turbot/go-kit/types" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + "google.golang.org/api/googleapi" "google.golang.org/api/storage/v1" ) @@ -318,6 +320,11 @@ func getStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plug resp, err := service.Objects.GetIamPolicy(object.Bucket, object.Name).Do() if err != nil { + + // Return nil, if uniform bucket-level access is enabled + if strings.Contains(err.(*googleapi.Error).Message, "Object policies are disabled for bucket") { + return nil, nil + } plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "api_error", err) return nil, err } From dcbabfd89bc145fecd153957c361ee33847d6499 Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Thu, 22 Jun 2023 21:27:23 +0530 Subject: [PATCH 3/5] Fix table name --- ...object.md => gcp_storage_bucket_object.md} | 24 +++++++-------- gcp/plugin.go | 2 +- ....go => table_gcp_storage_bucket_object.go} | 30 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) rename docs/tables/{gcp_storage_object.md => gcp_storage_bucket_object.md} (71%) rename gcp/{table_gcp_storage_object.go => table_gcp_storage_bucket_object.go} (88%) diff --git a/docs/tables/gcp_storage_object.md b/docs/tables/gcp_storage_bucket_object.md similarity index 71% rename from docs/tables/gcp_storage_object.md rename to docs/tables/gcp_storage_bucket_object.md index f348b26b..1d73914f 100644 --- a/docs/tables/gcp_storage_object.md +++ b/docs/tables/gcp_storage_bucket_object.md @@ -1,6 +1,6 @@ -# Table: gcp_storage_object +# Table: gcp_storage_bucket_object -Storage buckets are the basic containers that hold data. Everything that you store in cloud Storage must be contained in a bucket. +The Objects resource represents an object within Cloud Storage. Objects are pieces of data that you have uploaded to Cloud Storage. ## Examples @@ -11,14 +11,11 @@ select id, name, bucket, - content_type, - generation, size, storage_class, - time_created, - owner + time_created from - gcp_storage_object; + gcp_storage_bucket_object; ``` ### List storage objects encrypted with customer managed keys @@ -27,11 +24,12 @@ from select id, name, - bucket + bucket, + kms_key_name from - gcp_storage_object + gcp_storage_bucket_object where - not kms_key_name is not null; + kms_key_name != ''; ``` ### Get total objects and size of each bucket @@ -42,7 +40,7 @@ select count(*) as total_objects, sum(size) as total_size_bytes from - gcp_storage_object + gcp_storage_bucket_object group by bucket; ``` @@ -57,7 +55,7 @@ select p ->> 'role' as role, p ->> 'version' as version from - gcp_storage_object, + gcp_storage_bucket_object, jsonb_array_elements(iam_policy -> 'bindings') as p; ``` @@ -69,7 +67,7 @@ select name, extract(epoch from (retention_expiration_time - current_timestamp)) as retention_period_secs from - gcp_storage_object + gcp_storage_bucket_object where extract(epoch from (retention_expiration_time - current_timestamp)) < 604800; ``` \ No newline at end of file diff --git a/gcp/plugin.go b/gcp/plugin.go index 1bfa2e09..d82dd6c3 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -119,7 +119,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_sql_database_instance_metric_cpu_utilization_daily": tableGcpSQLDatabaseInstanceMetricCpuUtilizationDaily(ctx), "gcp_sql_database_instance_metric_cpu_utilization_hourly": tableGcpSQLDatabaseInstanceMetricCpuUtilizationHourly(ctx), "gcp_storage_bucket": tableGcpStorageBucket(ctx), - "gcp_storage_object": tableGcpStorageObject(ctx), + "gcp_storage_bucket_object": tableGcpBucketStorageObject(ctx), /* https://github.com/turbot/steampipe/issues/108 "gcp_compute_route": tableGcpComputeRoute(ctx), diff --git a/gcp/table_gcp_storage_object.go b/gcp/table_gcp_storage_bucket_object.go similarity index 88% rename from gcp/table_gcp_storage_object.go rename to gcp/table_gcp_storage_bucket_object.go index 5e40f6a6..579eb1dc 100644 --- a/gcp/table_gcp_storage_object.go +++ b/gcp/table_gcp_storage_bucket_object.go @@ -13,18 +13,18 @@ import ( "google.golang.org/api/storage/v1" ) -func tableGcpStorageObject(_ context.Context) *plugin.Table { +func tableGcpBucketStorageObject(_ context.Context) *plugin.Table { return &plugin.Table{ - Name: "gcp_storage_object", + Name: "gcp_storage_bucket_object", Description: "GCP Storage Object", Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"bucket", "name"}), - Hydrate: getStorageObject, + Hydrate: getBucketStorageObject, }, List: &plugin.ListConfig{ KeyColumns: plugin.OptionalColumns([]string{"bucket"}), ParentHydrate: listGcpStorageBuckets, - Hydrate: listStorageObjects, + Hydrate: listBucketStorageObjects, }, Columns: []*plugin.Column{ { @@ -199,7 +199,7 @@ func tableGcpStorageObject(_ context.Context) *plugin.Table { { Name: "iam_policy", Description: "An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources. A `Policy` is a collection of `bindings`. A `binding` binds one or more `members` to a single `role`. Members can be user accounts, service accounts, Google groups, and domains (such as G Suite). A `role` is a named list of permissions; each `role` can be an IAM predefined role or a user-created custom role. For some types of Google Cloud resources, a `binding` can also specify a `condition`, which is a logical expression that allows access to a resource only if the expression evaluates to `true`.", - Hydrate: getStorageObjectIAMPolicy, + Hydrate: getBucketStorageObjectIAMPolicy, Transform: transform.FromValue(), Type: proto.ColumnType_JSON, }, @@ -233,7 +233,7 @@ func tableGcpStorageObject(_ context.Context) *plugin.Table { //// LIST FUNCTION -func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func listBucketStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { bucket := h.Item.(*storage.Bucket).Name if d.EqualsQuals["bucket"] != nil && d.EqualsQualString("bucket") != bucket { @@ -242,7 +242,7 @@ func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.Hydr service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.listBucketStorageObjects", "connection_error", err) return nil, err } @@ -275,7 +275,7 @@ func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.Hydr } return nil }); err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.listBucketStorageObjects", "api_error", err) return nil, err } @@ -284,7 +284,7 @@ func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.Hydr //// HYDRATE FUNCTIONS -func getStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func getBucketStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { bucket := d.EqualsQuals["bucket"].GetStringValue() name := d.EqualsQuals["name"].GetStringValue() @@ -295,26 +295,26 @@ func getStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObject", "connection_error", err) return nil, err } req, err := service.Objects.Get(bucket, name).Do() if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObject", "api_error", err) return nil, err } return req, nil } -func getStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func getBucketStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { object := h.Item.(*storage.Object) // Create Session service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObjectIAMPolicy", "connection_error", err) return nil, err } @@ -325,7 +325,7 @@ func getStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plug if strings.Contains(err.(*googleapi.Error).Message, "Object policies are disabled for bucket") { return nil, nil } - plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObjectIAMPolicy", "api_error", err) return nil, err } @@ -339,7 +339,7 @@ func getObjectAka(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateDat getProjectCached := plugin.HydrateFunc(getProject).WithCache() projectId, err := getProjectCached(ctx, d, h) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_object.getObjectAka", "cache_error", err) + plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getObjectAka", "cache_error", err) return nil, err } project := projectId.(string) From 32e5da5044677a4fa0ff3d9aa411154213f1e363 Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Fri, 23 Jun 2023 17:12:51 +0530 Subject: [PATCH 4/5] Update table name and docs --- docs/tables/gcp_storage_bucket_object.md | 73 ----------- docs/tables/gcp_storage_object.md | 117 ++++++++++++++++++ gcp/plugin.go | 2 +- ..._object.go => table_gcp_storage_object.go} | 38 +++--- 4 files changed, 137 insertions(+), 93 deletions(-) delete mode 100644 docs/tables/gcp_storage_bucket_object.md create mode 100644 docs/tables/gcp_storage_object.md rename gcp/{table_gcp_storage_bucket_object.go => table_gcp_storage_object.go} (86%) diff --git a/docs/tables/gcp_storage_bucket_object.md b/docs/tables/gcp_storage_bucket_object.md deleted file mode 100644 index 1d73914f..00000000 --- a/docs/tables/gcp_storage_bucket_object.md +++ /dev/null @@ -1,73 +0,0 @@ -# Table: gcp_storage_bucket_object - -The Objects resource represents an object within Cloud Storage. Objects are pieces of data that you have uploaded to Cloud Storage. - -## Examples - -### Basic info - -```sql -select - id, - name, - bucket, - size, - storage_class, - time_created -from - gcp_storage_bucket_object; -``` - -### List storage objects encrypted with customer managed keys - -```sql -select - id, - name, - bucket, - kms_key_name -from - gcp_storage_bucket_object -where - kms_key_name != ''; -``` - -### Get total objects and size of each bucket - -```sql -select - bucket, - count(*) as total_objects, - sum(size) as total_size_bytes -from - gcp_storage_bucket_object -group by - bucket; -``` - -### List of members and their associated iam roles for each objects - -```sql -select - bucket, - name, - p -> 'members' as member, - p ->> 'role' as role, - p ->> 'version' as version -from - gcp_storage_bucket_object, - jsonb_array_elements(iam_policy -> 'bindings') as p; -``` - -### List of storage objects whose retention period is less than 7 days - -```sql -select - bucket, - name, - extract(epoch from (retention_expiration_time - current_timestamp)) as retention_period_secs -from - gcp_storage_bucket_object -where - extract(epoch from (retention_expiration_time - current_timestamp)) < 604800; -``` \ No newline at end of file diff --git a/docs/tables/gcp_storage_object.md b/docs/tables/gcp_storage_object.md new file mode 100644 index 00000000..f7f57afa --- /dev/null +++ b/docs/tables/gcp_storage_object.md @@ -0,0 +1,117 @@ +# Table: gcp_storage_object + +The Objects resource represents an object within Cloud Storage. Objects are pieces of data that you have uploaded to Cloud Storage. + +## Examples + +### Basic info + +```sql +select + id, + name, + bucket, + size, + storage_class, + time_created +from + gcp_storage_object +where + bucket = 'steampipe-test'; +``` + +### Get a specific object in a bucket + +```sql +select + id, + name, + bucket, + size, + storage_class, + time_created +from + gcp_storage_object +where + bucket = 'steampipe-test' + and name = 'test/logs/2021/03/01/12/abc.txt'; +``` + +### List storage objects encrypted with customer managed keys + +```sql +select + id, + name, + bucket, + kms_key_name +from + gcp_storage_object +where + bucket = 'steampipe-test' + and kms_key_name != ''; +``` + +### Get total objects and size of each bucket + +```sql +select + bucket, + count(*) as total_objects, + sum(size) as total_size_bytes +from + gcp_storage_object o, + gcp_storage_bucket b +where + o.bucket = b.name +group by + bucket; +``` + +### List of members and their associated IAM roles for each objects + +```sql +select + bucket, + name, + p -> 'members' as member, + p ->> 'role' as role, + p ->> 'version' as version +from + gcp_storage_object, + jsonb_array_elements(iam_policy -> 'bindings') as p +where + bucket = 'steampipe-test'; +``` + +### List of storage objects whose retention period is less than 7 days + +```sql +select + bucket, + name, + extract(epoch from (retention_expiration_time - current_timestamp)) as retention_period_secs +from + gcp_storage_object +where + extract(epoch from (retention_expiration_time - current_timestamp)) < 604800 + and bucket = 'steampipe-test'; +``` + +### Get accsess controls on each object in a bucket + +```sql +select + bucket, + name as object_name, + a ->> 'entity' as entity, + a ->> 'role' as role, + a ->> 'email' as email, + a ->> 'domain' as domain, + a ->> 'projectTeam' as project_team +from + gcp_storage_object, + jsonb_array_elements(acl) as a +where + bucket = 'steampipe-test'; +``` diff --git a/gcp/plugin.go b/gcp/plugin.go index d82dd6c3..1bfa2e09 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -119,7 +119,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_sql_database_instance_metric_cpu_utilization_daily": tableGcpSQLDatabaseInstanceMetricCpuUtilizationDaily(ctx), "gcp_sql_database_instance_metric_cpu_utilization_hourly": tableGcpSQLDatabaseInstanceMetricCpuUtilizationHourly(ctx), "gcp_storage_bucket": tableGcpStorageBucket(ctx), - "gcp_storage_bucket_object": tableGcpBucketStorageObject(ctx), + "gcp_storage_object": tableGcpStorageObject(ctx), /* https://github.com/turbot/steampipe/issues/108 "gcp_compute_route": tableGcpComputeRoute(ctx), diff --git a/gcp/table_gcp_storage_bucket_object.go b/gcp/table_gcp_storage_object.go similarity index 86% rename from gcp/table_gcp_storage_bucket_object.go rename to gcp/table_gcp_storage_object.go index 579eb1dc..d08f4eee 100644 --- a/gcp/table_gcp_storage_bucket_object.go +++ b/gcp/table_gcp_storage_object.go @@ -13,18 +13,17 @@ import ( "google.golang.org/api/storage/v1" ) -func tableGcpBucketStorageObject(_ context.Context) *plugin.Table { +func tableGcpStorageObject(_ context.Context) *plugin.Table { return &plugin.Table{ - Name: "gcp_storage_bucket_object", + Name: "gcp_storage_object", Description: "GCP Storage Object", Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"bucket", "name"}), - Hydrate: getBucketStorageObject, + Hydrate: getStorageObject, }, List: &plugin.ListConfig{ - KeyColumns: plugin.OptionalColumns([]string{"bucket"}), - ParentHydrate: listGcpStorageBuckets, - Hydrate: listBucketStorageObjects, + KeyColumns: plugin.SingleColumn("bucket"), + Hydrate: listStorageObjects, }, Columns: []*plugin.Column{ { @@ -199,7 +198,7 @@ func tableGcpBucketStorageObject(_ context.Context) *plugin.Table { { Name: "iam_policy", Description: "An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources. A `Policy` is a collection of `bindings`. A `binding` binds one or more `members` to a single `role`. Members can be user accounts, service accounts, Google groups, and domains (such as G Suite). A `role` is a named list of permissions; each `role` can be an IAM predefined role or a user-created custom role. For some types of Google Cloud resources, a `binding` can also specify a `condition`, which is a logical expression that allows access to a resource only if the expression evaluates to `true`.", - Hydrate: getBucketStorageObjectIAMPolicy, + Hydrate: getStorageObjectIAMPolicy, Transform: transform.FromValue(), Type: proto.ColumnType_JSON, }, @@ -233,16 +232,17 @@ func tableGcpBucketStorageObject(_ context.Context) *plugin.Table { //// LIST FUNCTION -func listBucketStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - bucket := h.Item.(*storage.Bucket).Name +func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + bucket := d.EqualsQualString("bucket") - if d.EqualsQuals["bucket"] != nil && d.EqualsQualString("bucket") != bucket { + // The bucket name should not be empty + if bucket == "" { return nil, nil } service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.listBucketStorageObjects", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "connection_error", err) return nil, err } @@ -275,7 +275,7 @@ func listBucketStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugi } return nil }); err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.listBucketStorageObjects", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.listStorageObjects", "api_error", err) return nil, err } @@ -284,7 +284,7 @@ func listBucketStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugi //// HYDRATE FUNCTIONS -func getBucketStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func getStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { bucket := d.EqualsQuals["bucket"].GetStringValue() name := d.EqualsQuals["name"].GetStringValue() @@ -295,26 +295,26 @@ func getBucketStorageObject(ctx context.Context, d *plugin.QueryData, h *plugin. service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObject", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "connection_error", err) return nil, err } req, err := service.Objects.Get(bucket, name).Do() if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObject", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObject", "api_error", err) return nil, err } return req, nil } -func getBucketStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func getStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { object := h.Item.(*storage.Object) // Create Session service, err := StorageService(ctx, d) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObjectIAMPolicy", "connection_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "connection_error", err) return nil, err } @@ -325,7 +325,7 @@ func getBucketStorageObjectIAMPolicy(ctx context.Context, d *plugin.QueryData, h if strings.Contains(err.(*googleapi.Error).Message, "Object policies are disabled for bucket") { return nil, nil } - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getBucketStorageObjectIAMPolicy", "api_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.getStorageObjectIAMPolicy", "api_error", err) return nil, err } @@ -339,7 +339,7 @@ func getObjectAka(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateDat getProjectCached := plugin.HydrateFunc(getProject).WithCache() projectId, err := getProjectCached(ctx, d, h) if err != nil { - plugin.Logger(ctx).Trace("gcp_storage_bucket_object.getObjectAka", "cache_error", err) + plugin.Logger(ctx).Trace("gcp_storage_object.getObjectAka", "cache_error", err) return nil, err } project := projectId.(string) From ce86d7b159168909b08b83785ed22c36beed91d2 Mon Sep 17 00:00:00 2001 From: Karan Popat Date: Tue, 27 Jun 2023 23:58:24 +0530 Subject: [PATCH 5/5] Set projection as `full` for gcp storage resources --- gcp/table_gcp_storage_bucket.go | 8 +------- gcp/table_gcp_storage_object.go | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/gcp/table_gcp_storage_bucket.go b/gcp/table_gcp_storage_bucket.go index f44a54a0..febe91a1 100644 --- a/gcp/table_gcp_storage_bucket.go +++ b/gcp/table_gcp_storage_bucket.go @@ -3,7 +3,6 @@ package gcp import ( "context" - "github.com/turbot/go-kit/helpers" "github.com/turbot/go-kit/types" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" @@ -254,11 +253,6 @@ func listGcpStorageBuckets(ctx context.Context, d *plugin.QueryData, h *plugin.H return nil, err } - projection := "noAcl" - if helpers.StringSliceContains(d.QueryContext.Columns, "acl") || helpers.StringSliceContains(d.QueryContext.Columns, "default_object_acl") { - projection = "full" - } - // Max limit isn't mentioned in the documentation // Default limit is set as 1000 maxResults := types.Int64(1000) @@ -269,7 +263,7 @@ func listGcpStorageBuckets(ctx context.Context, d *plugin.QueryData, h *plugin.H } } - resp := service.Buckets.List(project).Projection(projection).MaxResults(*maxResults) + resp := service.Buckets.List(project).Projection("full").MaxResults(*maxResults) if err := resp.Pages(ctx, func(page *storage.Buckets) error { for _, bucket := range page.Items { d.StreamListItem(ctx, bucket) diff --git a/gcp/table_gcp_storage_object.go b/gcp/table_gcp_storage_object.go index d08f4eee..fdd0e858 100644 --- a/gcp/table_gcp_storage_object.go +++ b/gcp/table_gcp_storage_object.go @@ -4,7 +4,6 @@ import ( "context" "strings" - "github.com/turbot/go-kit/helpers" "github.com/turbot/go-kit/types" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" @@ -246,11 +245,6 @@ func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.Hydr return nil, err } - projection := "noAcl" - if helpers.StringSliceContains(d.QueryContext.Columns, "acl") { - projection = "full" - } - // Max limit isn't mentioned in the documentation // Default limit is set as 1000 maxResults := types.Int64(1000) @@ -261,7 +255,7 @@ func listStorageObjects(ctx context.Context, d *plugin.QueryData, h *plugin.Hydr } } - resp := service.Objects.List(bucket).Projection(projection).MaxResults(*maxResults) + resp := service.Objects.List(bucket).Projection("full").MaxResults(*maxResults) if err := resp.Pages(ctx, func(page *storage.Objects) error { for _, object := range page.Items { d.StreamListItem(ctx, object)