From 8c30580256f8a1024e24961ed9d58930a26d752d Mon Sep 17 00:00:00 2001 From: ParthaI Date: Thu, 17 Jun 2021 15:50:14 +0530 Subject: [PATCH 1/8] Add table gcp_logging_bucket closes #226 --- docs/tables/gcp_logging_bucket.md | 28 +++ .../tests/gcp_logging_bucket/dependencies.txt | 0 .../gcp_logging_bucket/test-get-expected.json | 8 + .../gcp_logging_bucket/test-get-query.sql | 3 + .../test-list-expected.json | 6 + .../gcp_logging_bucket/test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 10 + .../gcp_logging_bucket/test-turbot-query.sql | 3 + .../tests/gcp_logging_bucket/variables.json | 1 + .../tests/gcp_logging_bucket/variables.tf | 60 ++++++ gcp/plugin.go | 1 + gcp/table_gcp_logging_bucket.go | 180 ++++++++++++++++++ 14 files changed, 307 insertions(+) create mode 100644 docs/tables/gcp_logging_bucket.md create mode 100644 gcp-test/tests/gcp_logging_bucket/dependencies.txt create mode 100644 gcp-test/tests/gcp_logging_bucket/test-get-expected.json create mode 100644 gcp-test/tests/gcp_logging_bucket/test-get-query.sql create mode 100644 gcp-test/tests/gcp_logging_bucket/test-list-expected.json create mode 100644 gcp-test/tests/gcp_logging_bucket/test-list-query.sql create mode 100644 gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json create mode 100644 gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql create mode 100644 gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json create mode 100644 gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql create mode 100644 gcp-test/tests/gcp_logging_bucket/variables.json create mode 100644 gcp-test/tests/gcp_logging_bucket/variables.tf create mode 100644 gcp/table_gcp_logging_bucket.go diff --git a/docs/tables/gcp_logging_bucket.md b/docs/tables/gcp_logging_bucket.md new file mode 100644 index 00000000..49b25312 --- /dev/null +++ b/docs/tables/gcp_logging_bucket.md @@ -0,0 +1,28 @@ +# Table: gcp_logging_bucket + +Logs buckets store the logs that are routed from other projects, folders, or organizations. + +### Basic info + +```sql +select + name, + lifecycle_state, + description, + retention_days +from + gcp_logging_bucket; +``` + + +### List of locked bucket + +```sql +select + name, + locked +from + gcp_logging_bucket +where + locked; +``` diff --git a/gcp-test/tests/gcp_logging_bucket/dependencies.txt b/gcp-test/tests/gcp_logging_bucket/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/gcp-test/tests/gcp_logging_bucket/test-get-expected.json b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json new file mode 100644 index 00000000..807945b6 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json @@ -0,0 +1,8 @@ +[ + { + "location": "{{ output.region_id.value }}", + "name": "{{ output.resource_id.value }}", + "project": "{{ output.project_id.value }}", + "retention_days": "30" + } +] \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-get-query.sql b/gcp-test/tests/gcp_logging_bucket/test-get-query.sql new file mode 100644 index 00000000..2e6a4236 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-get-query.sql @@ -0,0 +1,3 @@ +select name, retention_days, location, project +from gcp.gcp_logging_bucket +where name = '{{ output.resource_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-expected.json b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json new file mode 100644 index 00000000..367f0d44 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json @@ -0,0 +1,6 @@ +[ + { + "name": "{{ output.resource_id.value }}", + "title": "{{ output.resource_id.value }}" + } +] \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-query.sql b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql new file mode 100644 index 00000000..1fa0586c --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql @@ -0,0 +1,3 @@ +select name, title +from gcp.gcp_logging_bucket +where name = '{{ output.resource_id.value }}' \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json b/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json new file mode 100644 index 00000000..ec747fa4 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql b/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql new file mode 100644 index 00000000..f934c98a --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, disabled, description, filter +from gcp.gcp_logging_exclusion +where name = 'projects/parker-aaa/locations/global/dummy-{{resourceName}}' \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json new file mode 100644 index 00000000..1ee10fb6 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json @@ -0,0 +1,10 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}" + ], + "location": "{{ output.region_id.value }}", + "project": "{{ output.project_id.value }}", + "title": "{{ output.resource_id.value }}" + } +] \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql new file mode 100644 index 00000000..cd9742e4 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql @@ -0,0 +1,3 @@ +select title, akas, location, project +from gcp.gcp_logging_bucket +where name = '{{ output.resource_id.value }}' \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/variables.json b/gcp-test/tests/gcp_logging_bucket/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/variables.json @@ -0,0 +1 @@ +{} diff --git a/gcp-test/tests/gcp_logging_bucket/variables.tf b/gcp-test/tests/gcp_logging_bucket/variables.tf new file mode 100644 index 00000000..15be66a5 --- /dev/null +++ b/gcp-test/tests/gcp_logging_bucket/variables.tf @@ -0,0 +1,60 @@ + +variable "resource_name" { + type = string + default = "turbot-test-20200125-create-update" + description = "Name of the resource used throughout the test." +} + +variable "gcp_project" { + type = string + default = "niteowl-aaa" + description = "GCP project used for the test." +} + +variable "gcp_region" { + type = string + default = "us-east1" + description = "GCP region used for the test." +} + +variable "gcp_zone" { + type = string + default = "us-east1-b" +} + +provider "google" { + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +data "google_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "gcp://cloudresourcemanager.googleapis.com/projects/${data.google_client_config.current.project}" + } +} + +resource "google_logging_project_bucket_config" "named_test_resource" { + project = var.gcp_project + location = var.gcp_region + retention_days = 30 + bucket_id = var.resource_name +} + +output "resource_aka" { + value = "gcp://logging.googleapis.com/${google_logging_project_bucket_config.named_test_resource.id}" +} + +output "resource_id" { + value = google_logging_project_bucket_config.named_test_resource.id +} + +output "project_id" { + value = var.gcp_project +} + +output "region_id" { + value = var.gcp_region +} \ No newline at end of file diff --git a/gcp/plugin.go b/gcp/plugin.go index 9ae99f98..e08563da 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -66,6 +66,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_iam_role": tableGcpIamRole(ctx), "gcp_kms_key": tableGcpKmsKey(ctx), "gcp_kms_key_ring": tableGcpKmsKeyRing(ctx), + "gcp_logging_bucket": tableGcpLoggingBucket(ctx), "gcp_logging_exclusion": tableGcpLoggingExclusion(ctx), "gcp_logging_metric": tableGcpLoggingMetric(ctx), "gcp_logging_sink": tableGcpLoggingSink(ctx), diff --git a/gcp/table_gcp_logging_bucket.go b/gcp/table_gcp_logging_bucket.go new file mode 100644 index 00000000..c7b70512 --- /dev/null +++ b/gcp/table_gcp_logging_bucket.go @@ -0,0 +1,180 @@ +package gcp + +import ( + "context" + "errors" + "regexp" + "strings" + + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "google.golang.org/api/logging/v2" +) + +//// TABLE DEFINITION +func tableGcpLoggingBucket(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_logging_bucket", + Description: "GCP Logging Bucket", + Get: &plugin.GetConfig{ + KeyColumns: plugin.SingleColumn("name"), + Hydrate: getLoggingBucket, + }, + List: &plugin.ListConfig{ + Hydrate: listLoggingBuckets, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name of the bucket.", + Type: proto.ColumnType_STRING, + }, + { + Name: "create_time", + Description: "The creation timestamp of the bucket. This is not set for any of the default buckets.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + { + Name: "description", + Description: "Describes this bucket.", + Type: proto.ColumnType_STRING, + }, + { + Name: "lifecycle_state", + Description: "The bucket lifecycle state.", + Type: proto.ColumnType_STRING, + }, + { + Name: "locked", + Description: "Locked: Whether the bucket has been locked. The retention period on a locked bucket may not be changed. Locked buckets may only be deleted if they are empty.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "retention_days", + Description: "Logs will be retained by default for this amount of time, after which they will automatically be deleted.", + Type: proto.ColumnType_STRING, + }, + { + Name: "update_time", + Description: "he last update timestamp of the bucket.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + + // standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.From(bucketAka), + }, + + // standard gcp columns + { + Name: "location", + Description: ColumnDescriptionLocation, + Type: proto.ColumnType_STRING, + Transform: transform.From(bucketLocation), + }, + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Hydrate: getProject, + Transform: transform.FromValue(), + }, + }, + } +} + +//// LIST FUNCTION + +func listLoggingBuckets(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("listLoggingBuckets") + + // Create service connection + service, err := LoggingService(ctx, d) + if err != nil { + return nil, err + } + + // Get project details + projectData, err := activeProject(ctx, d) + if err != nil { + return nil, err + } + + project := projectData.Project + + resp := service.Projects.Locations.Buckets.List("projects/" + project + "/locations/-") + if err := resp.Pages( + ctx, + func(page *logging.ListBucketsResponse) error { + for _, bucket := range page.Buckets { + d.StreamListItem(ctx, bucket) + } + return nil + }, + ); err != nil { + return nil, err + } + + return nil, err +} + +//// HYDRATED FUNCTIONS + +func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getLoggingBucket") + + // Create Service Connection + service, err := LoggingService(ctx, d) + if err != nil { + return nil, err + } + + bucketName := d.KeyColumnQuals["name"].GetStringValue() + + // Match the bucket name pattern if we are doing query with where clause by passing name value + matched, err := regexp.MatchString(`projects/.+/locations/.+/buckets/.+`, bucketName) + + if !matched { + return nil, errors.New("Bucket name should match 'projects/[PROJECT_ID]/locations/[LOCATION_ID]/buckets/[BUCKET_ID]'") + } + + op, err := service.Projects.Locations.Buckets.Get(bucketName).Do() + if err != nil { + return nil, err + } + + return op, nil + +} + +//// TRANSFORM FUNCTIONS + +func bucketLocation(_ context.Context, d *transform.TransformData) (interface{}, error) { + bucket := d.HydrateItem.(*logging.LogBucket) + location := strings.Split(bucket.Name, "/")[3] + if location != "" { + return location, nil + } + return "", nil +} + +func bucketAka(_ context.Context, d *transform.TransformData) (interface{}, error) { + bucketName := d.HydrateItem.(*logging.LogBucket).Name + if bucketName != "" { + return []string{"gcp://logging.googleapis.com/" + bucketName}, nil + } + return nil, nil +} From c19628aff4021f9aacaddaa3f57c034c5b967e1f Mon Sep 17 00:00:00 2001 From: ParthaI Date: Thu, 24 Jun 2021 10:27:18 +0530 Subject: [PATCH 2/8] Made changes as per review comments --- gcp/table_gcp_logging_bucket.go | 50 +++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/gcp/table_gcp_logging_bucket.go b/gcp/table_gcp_logging_bucket.go index c7b70512..2dfefac7 100644 --- a/gcp/table_gcp_logging_bucket.go +++ b/gcp/table_gcp_logging_bucket.go @@ -2,8 +2,6 @@ package gcp import ( "context" - "errors" - "regexp" "strings" "github.com/turbot/steampipe-plugin-sdk/grpc/proto" @@ -14,12 +12,13 @@ import ( ) //// TABLE DEFINITION + func tableGcpLoggingBucket(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "gcp_logging_bucket", Description: "GCP Logging Bucket", Get: &plugin.GetConfig{ - KeyColumns: plugin.SingleColumn("name"), + KeyColumns: plugin.AllColumns([]string{"name", "location"}), Hydrate: getLoggingBucket, }, List: &plugin.ListConfig{ @@ -30,6 +29,13 @@ func tableGcpLoggingBucket(_ context.Context) *plugin.Table { Name: "name", Description: "The resource name of the bucket.", Type: proto.ColumnType_STRING, + Transform: transform.From(bucketName), + }, + { + Name: "self_link", + Description: "The server-defined URL for the resource.", + Type: proto.ColumnType_STRING, + Transform: transform.From(bucketSelfLink), }, { Name: "create_time", @@ -49,22 +55,22 @@ func tableGcpLoggingBucket(_ context.Context) *plugin.Table { }, { Name: "locked", - Description: "Locked: Whether the bucket has been locked. The retention period on a locked bucket may not be changed. Locked buckets may only be deleted if they are empty.", + Description: "Specifies whether the bucket has been locked, or not. The retention period on a locked bucket may not be changed. Locked buckets may only be deleted if they are empty.", Type: proto.ColumnType_BOOL, }, { Name: "retention_days", Description: "Logs will be retained by default for this amount of time, after which they will automatically be deleted.", - Type: proto.ColumnType_STRING, + Type: proto.ColumnType_INT, }, { Name: "update_time", - Description: "he last update timestamp of the bucket.", + Description: "The last update timestamp of the bucket.", Type: proto.ColumnType_TIMESTAMP, Transform: transform.FromGo().NullIfZero(), }, - // standard steampipe columns + // GCP standard columns { Name: "title", Description: ColumnDescriptionTitle, @@ -115,6 +121,7 @@ func listLoggingBuckets(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr project := projectData.Project + // If we want to list out all the bucket through out all the region then we have to pass '-' character in param after '/locations/' otherwise we have to specify a particular region for listing out the buckets resp := service.Projects.Locations.Buckets.List("projects/" + project + "/locations/-") if err := resp.Pages( ctx, @@ -143,15 +150,17 @@ func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat } bucketName := d.KeyColumnQuals["name"].GetStringValue() + locationId := d.KeyColumnQuals["location"].GetStringValue() - // Match the bucket name pattern if we are doing query with where clause by passing name value - matched, err := regexp.MatchString(`projects/.+/locations/.+/buckets/.+`, bucketName) + projectInfo, err := activeProject(ctx, d) - if !matched { - return nil, errors.New("Bucket name should match 'projects/[PROJECT_ID]/locations/[LOCATION_ID]/buckets/[BUCKET_ID]'") + if err != nil { + return nil, err } - op, err := service.Projects.Locations.Buckets.Get(bucketName).Do() + bucketNameWithLocation := "projects/" + projectInfo.Project + "/locations/" + locationId + "/buckets/" + bucketName + + op, err := service.Projects.Locations.Buckets.Get(bucketNameWithLocation).Do() if err != nil { return nil, err } @@ -162,6 +171,23 @@ func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat //// TRANSFORM FUNCTIONS +func bucketName(_ context.Context, d *transform.TransformData) (interface{}, error) { + bucket := d.HydrateItem.(*logging.LogBucket) + name := strings.Split(bucket.Name, "/")[len(strings.Split(bucket.Name, "/"))-1] + if name != "" { + return name, nil + } + return "", nil +} + +func bucketSelfLink(_ context.Context, d *transform.TransformData) (interface{}, error) { + bucket := d.HydrateItem.(*logging.LogBucket) + if bucket.Name == "" { + return nil, nil + } + return "https://logging.googleapis.com/v2/" + bucket.Name, nil +} + func bucketLocation(_ context.Context, d *transform.TransformData) (interface{}, error) { bucket := d.HydrateItem.(*logging.LogBucket) location := strings.Split(bucket.Name, "/")[3] From c1680c0eb1638dc4804e4256d82337f489d79103 Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 24 Jun 2021 10:58:26 +0530 Subject: [PATCH 3/8] Updated transform function --- gcp/table_gcp_logging_bucket.go | 64 ++++++++++++--------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/gcp/table_gcp_logging_bucket.go b/gcp/table_gcp_logging_bucket.go index 2dfefac7..fbd516e1 100644 --- a/gcp/table_gcp_logging_bucket.go +++ b/gcp/table_gcp_logging_bucket.go @@ -29,13 +29,13 @@ func tableGcpLoggingBucket(_ context.Context) *plugin.Table { Name: "name", Description: "The resource name of the bucket.", Type: proto.ColumnType_STRING, - Transform: transform.From(bucketName), + Transform: transform.FromField("Name").Transform(lastPathElement), }, { - Name: "self_link", + Name: "self_link", Description: "The server-defined URL for the resource.", - Type: proto.ColumnType_STRING, - Transform: transform.From(bucketSelfLink), + Type: proto.ColumnType_STRING, + Transform: transform.FromP(loggingBucketTurbotData, "SelfLink"), }, { Name: "create_time", @@ -75,28 +75,27 @@ func tableGcpLoggingBucket(_ context.Context) *plugin.Table { Name: "title", Description: ColumnDescriptionTitle, Type: proto.ColumnType_STRING, - Transform: transform.FromField("Name"), + Transform: transform.FromField("Name").Transform(lastPathElement), }, { Name: "akas", Description: ColumnDescriptionAkas, Type: proto.ColumnType_JSON, - Transform: transform.From(bucketAka), + Transform: transform.FromP(loggingBucketTurbotData, "Akas"), }, - // standard gcp columns + // GCP standard columns { Name: "location", Description: ColumnDescriptionLocation, Type: proto.ColumnType_STRING, - Transform: transform.From(bucketLocation), + Transform: transform.FromP(loggingBucketTurbotData, "Location"), }, { Name: "project", Description: ColumnDescriptionProject, Type: proto.ColumnType_STRING, - Hydrate: getProject, - Transform: transform.FromValue(), + Transform: transform.FromP(loggingBucketTurbotData, "Project"), }, }, } @@ -121,7 +120,7 @@ func listLoggingBuckets(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr project := projectData.Project - // If we want to list out all the bucket through out all the region then we have to pass '-' character in param after '/locations/' otherwise we have to specify a particular region for listing out the buckets + // '-' for all locations... resp := service.Projects.Locations.Buckets.List("projects/" + project + "/locations/-") if err := resp.Pages( ctx, @@ -138,7 +137,7 @@ func listLoggingBuckets(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr return nil, err } -//// HYDRATED FUNCTIONS +//// HYDRATE FUNCTIONS func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { plugin.Logger(ctx).Trace("getLoggingBucket") @@ -153,11 +152,9 @@ func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat locationId := d.KeyColumnQuals["location"].GetStringValue() projectInfo, err := activeProject(ctx, d) - if err != nil { return nil, err } - bucketNameWithLocation := "projects/" + projectInfo.Project + "/locations/" + locationId + "/buckets/" + bucketName op, err := service.Projects.Locations.Buckets.Get(bucketNameWithLocation).Do() @@ -171,36 +168,19 @@ func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat //// TRANSFORM FUNCTIONS -func bucketName(_ context.Context, d *transform.TransformData) (interface{}, error) { - bucket := d.HydrateItem.(*logging.LogBucket) - name := strings.Split(bucket.Name, "/")[len(strings.Split(bucket.Name, "/"))-1] - if name != "" { - return name, nil - } - return "", nil -} +func loggingBucketTurbotData(_ context.Context, d *transform.TransformData) (interface{}, error) { + data := d.HydrateItem.(*logging.LogBucket) + param := d.Param.(string) -func bucketSelfLink(_ context.Context, d *transform.TransformData) (interface{}, error) { - bucket := d.HydrateItem.(*logging.LogBucket) - if bucket.Name == "" { - return nil, nil - } - return "https://logging.googleapis.com/v2/" + bucket.Name, nil -} + // Fetch data from name + splittedTitle := strings.Split(data.Name, "/") -func bucketLocation(_ context.Context, d *transform.TransformData) (interface{}, error) { - bucket := d.HydrateItem.(*logging.LogBucket) - location := strings.Split(bucket.Name, "/")[3] - if location != "" { - return location, nil + turbotData := map[string]interface{}{ + "Project": splittedTitle[1], + "Location": splittedTitle[3], + "SelfLink": "https://logging.googleapis.com/v2/" + data.Name, + "Akas": []string{"gcp://logging.googleapis.com/" + data.Name}, } - return "", nil -} -func bucketAka(_ context.Context, d *transform.TransformData) (interface{}, error) { - bucketName := d.HydrateItem.(*logging.LogBucket).Name - if bucketName != "" { - return []string{"gcp://logging.googleapis.com/" + bucketName}, nil - } - return nil, nil + return turbotData[param], nil } From c6a6d0090bcd9048d25305467bce3b617958b99d Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 24 Jun 2021 11:22:10 +0530 Subject: [PATCH 4/8] Fixed notfound test and updated indentation --- .../tests/gcp_logging_bucket/test-get-expected.json | 2 +- .../tests/gcp_logging_bucket/test-list-expected.json | 2 +- gcp-test/tests/gcp_logging_bucket/test-list-query.sql | 2 +- .../gcp_logging_bucket/test-not-found-expected.json | 2 +- .../tests/gcp_logging_bucket/test-not-found-query.sql | 6 +++--- .../tests/gcp_logging_bucket/test-turbot-expected.json | 6 ++---- .../tests/gcp_logging_bucket/test-turbot-query.sql | 2 +- gcp-test/tests/gcp_logging_bucket/variables.tf | 10 +++++----- 8 files changed, 15 insertions(+), 17 deletions(-) diff --git a/gcp-test/tests/gcp_logging_bucket/test-get-expected.json b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json index 807945b6..f7483b0c 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-get-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json @@ -5,4 +5,4 @@ "project": "{{ output.project_id.value }}", "retention_days": "30" } -] \ No newline at end of file +] diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-expected.json b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json index 367f0d44..e7d61639 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-list-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json @@ -3,4 +3,4 @@ "name": "{{ output.resource_id.value }}", "title": "{{ output.resource_id.value }}" } -] \ No newline at end of file +] diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-query.sql b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql index 1fa0586c..0312a07b 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-list-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql @@ -1,3 +1,3 @@ select name, title from gcp.gcp_logging_bucket -where name = '{{ output.resource_id.value }}' \ No newline at end of file +where name = '{{ output.resource_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json b/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json index ec747fa4..19765bd5 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-not-found-expected.json @@ -1 +1 @@ -null \ No newline at end of file +null diff --git a/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql b/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql index f934c98a..d0835116 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-not-found-query.sql @@ -1,3 +1,3 @@ -select name, disabled, description, filter -from gcp.gcp_logging_exclusion -where name = 'projects/parker-aaa/locations/global/dummy-{{resourceName}}' \ No newline at end of file +select name, self_link, description +from gcp.gcp_logging_bucket +where name = 'dummy-{{resourceName}}' and location = '{{ output.region_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json index 1ee10fb6..cca58a31 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json @@ -1,10 +1,8 @@ [ { - "akas": [ - "{{ output.resource_aka.value }}" - ], + "akas": ["{{ output.resource_aka.value }}"], "location": "{{ output.region_id.value }}", "project": "{{ output.project_id.value }}", "title": "{{ output.resource_id.value }}" } -] \ No newline at end of file +] diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql index cd9742e4..7f174090 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql @@ -1,3 +1,3 @@ select title, akas, location, project from gcp.gcp_logging_bucket -where name = '{{ output.resource_id.value }}' \ No newline at end of file +where name = '{{ output.resource_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/variables.tf b/gcp-test/tests/gcp_logging_bucket/variables.tf index 15be66a5..e78ec679 100644 --- a/gcp-test/tests/gcp_logging_bucket/variables.tf +++ b/gcp-test/tests/gcp_logging_bucket/variables.tf @@ -37,10 +37,10 @@ data "null_data_source" "resource" { } resource "google_logging_project_bucket_config" "named_test_resource" { - project = var.gcp_project - location = var.gcp_region - retention_days = 30 - bucket_id = var.resource_name + project = var.gcp_project + location = var.gcp_region + retention_days = 30 + bucket_id = var.resource_name } output "resource_aka" { @@ -57,4 +57,4 @@ output "project_id" { output "region_id" { value = var.gcp_region -} \ No newline at end of file +} From 67fea406937fb83a08f2df05deb0e57abbcbb8f6 Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 24 Jun 2021 11:39:49 +0530 Subject: [PATCH 5/8] Added check to return nil, if no input is provided --- gcp/table_gcp_logging_bucket.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gcp/table_gcp_logging_bucket.go b/gcp/table_gcp_logging_bucket.go index fbd516e1..57bd51a0 100644 --- a/gcp/table_gcp_logging_bucket.go +++ b/gcp/table_gcp_logging_bucket.go @@ -151,6 +151,11 @@ func getLoggingBucket(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat bucketName := d.KeyColumnQuals["name"].GetStringValue() locationId := d.KeyColumnQuals["location"].GetStringValue() + // Return nil, if no input provided + if bucketName == "" || locationId == "" { + return nil, nil + } + projectInfo, err := activeProject(ctx, d) if err != nil { return nil, err From 102bf97731b9dccddb449fbc3ac9b8e1527b06a1 Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 24 Jun 2021 12:10:30 +0530 Subject: [PATCH 6/8] Fixed integration test --- gcp-test/tests/gcp_logging_bucket/test-get-expected.json | 4 ++-- gcp-test/tests/gcp_logging_bucket/test-get-query.sql | 2 +- gcp-test/tests/gcp_logging_bucket/test-list-expected.json | 4 ++-- gcp-test/tests/gcp_logging_bucket/test-list-query.sql | 2 +- gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json | 2 +- gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql | 2 +- gcp-test/tests/gcp_logging_bucket/variables.tf | 6 +++++- 7 files changed, 13 insertions(+), 9 deletions(-) diff --git a/gcp-test/tests/gcp_logging_bucket/test-get-expected.json b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json index f7483b0c..6b68e06b 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-get-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-get-expected.json @@ -1,8 +1,8 @@ [ { "location": "{{ output.region_id.value }}", - "name": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", "project": "{{ output.project_id.value }}", - "retention_days": "30" + "retention_days": 1 } ] diff --git a/gcp-test/tests/gcp_logging_bucket/test-get-query.sql b/gcp-test/tests/gcp_logging_bucket/test-get-query.sql index 2e6a4236..fa483a35 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-get-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-get-query.sql @@ -1,3 +1,3 @@ select name, retention_days, location, project from gcp.gcp_logging_bucket -where name = '{{ output.resource_id.value }}'; \ No newline at end of file +where name = '{{ resourceName }}' and location = '{{ output.region_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-expected.json b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json index e7d61639..6a9606db 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-list-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-list-expected.json @@ -1,6 +1,6 @@ [ { - "name": "{{ output.resource_id.value }}", - "title": "{{ output.resource_id.value }}" + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" } ] diff --git a/gcp-test/tests/gcp_logging_bucket/test-list-query.sql b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql index 0312a07b..83d5812b 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-list-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-list-query.sql @@ -1,3 +1,3 @@ select name, title from gcp.gcp_logging_bucket -where name = '{{ output.resource_id.value }}'; \ No newline at end of file +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json index cca58a31..ed9832a9 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-expected.json @@ -3,6 +3,6 @@ "akas": ["{{ output.resource_aka.value }}"], "location": "{{ output.region_id.value }}", "project": "{{ output.project_id.value }}", - "title": "{{ output.resource_id.value }}" + "title": "{{ resourceName }}" } ] diff --git a/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql index 7f174090..653ac487 100644 --- a/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql +++ b/gcp-test/tests/gcp_logging_bucket/test-turbot-query.sql @@ -1,3 +1,3 @@ select title, akas, location, project from gcp.gcp_logging_bucket -where name = '{{ output.resource_id.value }}'; \ No newline at end of file +where name = '{{ resourceName }}' and location = '{{ output.region_id.value }}'; \ No newline at end of file diff --git a/gcp-test/tests/gcp_logging_bucket/variables.tf b/gcp-test/tests/gcp_logging_bucket/variables.tf index e78ec679..af198602 100644 --- a/gcp-test/tests/gcp_logging_bucket/variables.tf +++ b/gcp-test/tests/gcp_logging_bucket/variables.tf @@ -39,7 +39,7 @@ data "null_data_source" "resource" { resource "google_logging_project_bucket_config" "named_test_resource" { project = var.gcp_project location = var.gcp_region - retention_days = 30 + retention_days = 1 bucket_id = var.resource_name } @@ -51,6 +51,10 @@ output "resource_id" { value = google_logging_project_bucket_config.named_test_resource.id } +output "resource_name" { + value = var.resource_name +} + output "project_id" { value = var.gcp_project } From 64531aa663d3d884141c99ae3e964d4d27f6beed Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Thu, 24 Jun 2021 12:27:42 +0530 Subject: [PATCH 7/8] Updated docs --- docs/tables/gcp_logging_bucket.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tables/gcp_logging_bucket.md b/docs/tables/gcp_logging_bucket.md index 49b25312..9e138664 100644 --- a/docs/tables/gcp_logging_bucket.md +++ b/docs/tables/gcp_logging_bucket.md @@ -2,6 +2,8 @@ Logs buckets store the logs that are routed from other projects, folders, or organizations. +## Examples + ### Basic info ```sql From 9e7bf765389a9a9ca0273c041b8a7a7ea7ff2c95 Mon Sep 17 00:00:00 2001 From: cbruno10 Date: Thu, 24 Jun 2021 10:04:32 -0400 Subject: [PATCH 8/8] Update gcp_logging_bucket.md --- docs/tables/gcp_logging_bucket.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/tables/gcp_logging_bucket.md b/docs/tables/gcp_logging_bucket.md index 9e138664..75695932 100644 --- a/docs/tables/gcp_logging_bucket.md +++ b/docs/tables/gcp_logging_bucket.md @@ -1,6 +1,6 @@ # Table: gcp_logging_bucket -Logs buckets store the logs that are routed from other projects, folders, or organizations. +Logging buckets store the logs that are routed from other projects, folders, or organizations. ## Examples @@ -16,8 +16,7 @@ from gcp_logging_bucket; ``` - -### List of locked bucket +### List locked buckets ```sql select