From 6ee8481aecd7afb6a9fddf4e87722327cc4355ae Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 14:02:25 +0530 Subject: [PATCH 01/13] Add table gcp_logging_log_entry Closes #462 --- gcp/plugin.go | 1 + gcp/table_gcp_logging_log_entry.go | 213 +++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 gcp/table_gcp_logging_log_entry.go diff --git a/gcp/plugin.go b/gcp/plugin.go index 1bfa2e09..6d9f8dd1 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -95,6 +95,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_kubernetes_node_pool": tableGcpKubernetesNodePool(ctx), "gcp_logging_bucket": tableGcpLoggingBucket(ctx), "gcp_logging_exclusion": tableGcpLoggingExclusion(ctx), + "gcp_logging_log_entry": tableGcpLoggingLogEntry(ctx), "gcp_logging_metric": tableGcpLoggingMetric(ctx), "gcp_logging_sink": tableGcpLoggingSink(ctx), "gcp_monitoring_alert_policy": tableGcpMonitoringAlert(ctx), diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go new file mode 100644 index 00000000..b2272f2f --- /dev/null +++ b/gcp/table_gcp_logging_log_entry.go @@ -0,0 +1,213 @@ +package gcp + +import ( + "context" + + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + + "google.golang.org/api/logging/v2" +) + +//// TABLE DEFINITION + +func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_logging_log_entry", + Description: "GCP Logging Log Entry", + Get: &plugin.GetConfig{ + KeyColumns: plugin.SingleColumn("name"), + Hydrate: getGcpLoggingExclusion, + }, + List: &plugin.ListConfig{ + Hydrate: listGcpLoggingLogEntries, + }, + Columns: []*plugin.Column{ + { + Name: "log_name", + Description: "The resource name of the log to which this log entry belongs to.", + Type: proto.ColumnType_STRING, + }, + { + Name: "insert_id", + Description: "A unique identifier for the log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "log_entry_operation_first", + Description: "Set this to True if this is the first log entry in the operation.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("Operation.First"), + }, + { + Name: "log_entry_operation_last", + Description: "Set this to True if this is the last log entry in the operation.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("Operation.Last"), + }, + { + Name: "log_entry_operation_id", + Description: "An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Operation.Id"), + }, + { + Name: "log_entry_operation_producer", + Description: "An arbitrary producer identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Operation.Producer"), + }, + { + Name: "receive_timestamp", + Description: "The time the log entry was received by Logging.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "resource_type", + Description: "The monitored resource type.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Resource.Type"), + }, + { + Name: "severity", + Description: "The severity of the log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "span_id", + Description: "The ID of the Cloud Trace (https://cloud.google.com/trace) span associated with the current operation in which the log is being written.", + Type: proto.ColumnType_STRING, + }, + { + Name: "text_payload", + Description: "The log entry payload, represented as a Unicode string (UTF-8).", + Type: proto.ColumnType_STRING, + }, + { + Name: "timestamp", + Description: "The time the event described by the log entry occurred.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "trace", + Description: "The REST resource name of the trace being written to Cloud Trace (https://cloud.google.com/trace) in association with this log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "trace_sampled", + Description: "The sampling decision of the trace associated with the log entry.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "split_index", + Description: "The index of this LogEntry in the sequence of split log entries.", + Type: proto.ColumnType_INT, + Transform: transform.FromField("Split.Index"), + }, + { + Name: "total_splits", + Description: "The total number of log entries that the original LogEntry was split into.", + Type: proto.ColumnType_INT, + Transform: transform.FromField("Split.TotalSplits"), + }, + { + Name: "split_uid", + Description: "A globally unique identifier for all log entries in a sequence of split log entries.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Split.Uid"), + }, + { + Name: "resource_lebels", + Description: "Values for all of the labels listed in the associated monitored resource descriptor.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Resource.Labels"), + }, + { + Name: "source_location", + Description: "Source code location information associated with the log entry, if any.", + Type: proto.ColumnType_JSON, + }, + + // // standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("LogName"), + }, + + // standard gcp columns + { + Name: "location", + Description: ColumnDescriptionLocation, + Type: proto.ColumnType_STRING, + Transform: transform.FromConstant("global"), + }, + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Hydrate: plugin.HydrateFunc(getProject).WithCache(), + Transform: transform.FromValue(), + }, + }, + } +} + +//// FETCH FUNCTIONS + +func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := LoggingService(ctx, d) + if err != nil { + return nil, err + } + + // Max limit isn't mentioned in the documentation + // Default limit is set as 1000 + pageSize := types.Int64(1000) + limit := d.QueryContext.Limit + if d.QueryContext.Limit != nil { + if *limit < *pageSize { + pageSize = limit + } + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + param := &logging.ListLogEntriesRequest{ + PageSize: *pageSize, + ProjectIds: []string{project}, + } + + op := service.Entries.List(param) + + if err := op.Pages( + ctx, + func(page *logging.ListLogEntriesResponse) error { + for _, entry := range page.Entries { + d.StreamListItem(ctx, entry) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + page.NextPageToken = "" + return nil + } + } + return nil + }, + ); err != nil { + return nil, err + } + + return nil, err +} From 465158463712d81acf2cd6af6971aab652e24776 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 14:03:24 +0530 Subject: [PATCH 02/13] Removed the get config --- gcp/table_gcp_logging_log_entry.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index b2272f2f..d8c158b0 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -17,10 +17,6 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "gcp_logging_log_entry", Description: "GCP Logging Log Entry", - Get: &plugin.GetConfig{ - KeyColumns: plugin.SingleColumn("name"), - Hydrate: getGcpLoggingExclusion, - }, List: &plugin.ListConfig{ Hydrate: listGcpLoggingLogEntries, }, From 23b021fe39094b76410500ad01f80e1376181508 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 15:44:45 +0530 Subject: [PATCH 03/13] Added optional quals --- gcp/table_gcp_logging_log_entry.go | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index d8c158b0..d156e9c6 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -19,6 +19,11 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Description: "GCP Logging Log Entry", List: &plugin.ListConfig{ Hydrate: listGcpLoggingLogEntries, + KeyColumns: plugin.KeyColumnSlice{ + {Name: "resource_type", Require: plugin.Optional}, + {Name: "severity", Require: plugin.Optional}, + {Name: "log_name", Require: plugin.Optional}, + }, }, Columns: []*plugin.Column{ { @@ -184,6 +189,35 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi ProjectIds: []string{project}, } + resourceType := d.EqualsQualString("resource_type") + severity := d.EqualsQualString("severity") + logName := d.EqualsQualString("log_name") + filter := "" + + if resourceType != "" { + filter = "resource.type" + " = \"" + resourceType + "\"" + } + + if severity != "" { + if filter != "" { + filter = filter + " AND severity" + " = \"" + severity + "\"" + } else { + filter = "severity" + " = \"" + severity + "\"" + } + } + + if logName != "" { + if filter != "" { + filter = filter + " AND logName" + " = \"" + logName + "\"" + } else { + filter = "logName" + " = \"" + logName + "\"" + } + } + + if filter != "" { + param.Filter = filter + } + op := service.Entries.List(param) if err := op.Pages( From 8498e269577b9fe06adc7596a21afce4a1b41c8c Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 15:46:29 +0530 Subject: [PATCH 04/13] Added insert_id as optional quals --- gcp/table_gcp_logging_log_entry.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index d156e9c6..6f9b2c9d 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -23,6 +23,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { {Name: "resource_type", Require: plugin.Optional}, {Name: "severity", Require: plugin.Optional}, {Name: "log_name", Require: plugin.Optional}, + {Name: "insert_id", Require: plugin.Optional}, }, }, Columns: []*plugin.Column{ @@ -192,6 +193,7 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi resourceType := d.EqualsQualString("resource_type") severity := d.EqualsQualString("severity") logName := d.EqualsQualString("log_name") + insertId := d.EqualsQualString("insert_id") filter := "" if resourceType != "" { @@ -214,6 +216,14 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi } } + if insertId != "" { + if filter != "" { + filter = filter + " AND insertId" + " = \"" + insertId + "\"" + } else { + filter = "insertId" + " = \"" + insertId + "\"" + } + } + if filter != "" { param.Filter = filter } From a2d9d449691fc7a54485b2605234f05cfb2e9b9c Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 16:48:29 +0530 Subject: [PATCH 05/13] Added get config --- gcp/table_gcp_logging_log_entry.go | 56 ++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 6f9b2c9d..76452d80 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -17,13 +17,16 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "gcp_logging_log_entry", Description: "GCP Logging Log Entry", + Get: &plugin.GetConfig{ + KeyColumns: plugin.SingleColumn("insert_id"), + Hydrate: getGcpLoggingLogEntry, + }, List: &plugin.ListConfig{ Hydrate: listGcpLoggingLogEntries, KeyColumns: plugin.KeyColumnSlice{ {Name: "resource_type", Require: plugin.Optional}, {Name: "severity", Require: plugin.Optional}, {Name: "log_name", Require: plugin.Optional}, - {Name: "insert_id", Require: plugin.Optional}, }, }, Columns: []*plugin.Column{ @@ -193,7 +196,6 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi resourceType := d.EqualsQualString("resource_type") severity := d.EqualsQualString("severity") logName := d.EqualsQualString("log_name") - insertId := d.EqualsQualString("insert_id") filter := "" if resourceType != "" { @@ -216,14 +218,6 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi } } - if insertId != "" { - if filter != "" { - filter = filter + " AND insertId" + " = \"" + insertId + "\"" - } else { - filter = "insertId" + " = \"" + insertId + "\"" - } - } - if filter != "" { param.Filter = filter } @@ -251,3 +245,45 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi return nil, err } + +//// HYDRATE FUNCTION + +func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := LoggingService(ctx, d) + if err != nil { + return nil, err + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + param := &logging.ListLogEntriesRequest{ + ProjectIds: []string{project}, + } + + insertId := d.EqualsQualString("insert_id") + filter := "" + + if insertId != "" { + filter = "insertId" + " = \"" + insertId + "\"" + } + param.Filter = filter + + op, err := service.Entries.List(param).Do() + + if err != nil { + return nil, err + } + + if len(op.Entries) > 0 { + return op.Entries[0], nil + } + + return nil, nil +} From d193f474f9fe230af0f6a32ed1cf8c9e0379ec57 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 30 Jun 2023 16:50:39 +0530 Subject: [PATCH 06/13] Updated the title filter --- gcp/table_gcp_logging_log_entry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 76452d80..6607bba4 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -135,12 +135,12 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Type: proto.ColumnType_JSON, }, - // // standard steampipe columns + // standard steampipe columns { Name: "title", Description: ColumnDescriptionTitle, Type: proto.ColumnType_STRING, - Transform: transform.FromField("LogName"), + Transform: transform.FromField("InsertId"), }, // standard gcp columns From f13e9db52d5339e5af267453f11d97eaf536f9fe Mon Sep 17 00:00:00 2001 From: ParthaI Date: Mon, 3 Jul 2023 09:15:50 +0530 Subject: [PATCH 07/13] Added doc for the table --- docs/tables/gcp_logging_log_entry.md | 85 ++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docs/tables/gcp_logging_log_entry.md diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md new file mode 100644 index 00000000..0c318362 --- /dev/null +++ b/docs/tables/gcp_logging_log_entry.md @@ -0,0 +1,85 @@ + +# Table: gcp_logging_log_entry + +AWS Logging Log Entry refers to a single log event or entry in a log stream within an AWS service's logging system. It contains information about a specific occurrence or activity that is being logged. Log entries typically include details such as timestamp, log message, log level, request ID, and other relevant metadata. + +## Examples + +### Basic info + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_id, + receive_timestamp +from + gcp_logging_log_entry; +``` + +### Get log entries by resource type + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + resource_type, + span_id, + text_payload +from + gcp_logging_log_entry +where + resource_type = 'audited_resource'; +``` + +### List log entries with NOTICE severity + +```sql +select + log_name, + insert_id, + resource_type, + severity, + span_id, + timestamp +from + gcp_logging_log_entry +where + severity = 'NOTICE'; +``` + +### List log entries in last 30 days + +```sql +select + log_name, + insert_id, + receive_timestamp, + trace_sampled, + span_id, + timestamp +from + gcp_logging_log_entry +where + timestamp >= now() - interval '30' day; +``` + +### Filter log entries by log name + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity +from + gcp_logging_log_entry +where + log_name = 'projects/parker-aaa/logs/cloudaudit.googleapis.com%2Factivity'; +``` \ No newline at end of file From 9ea35c74ff4bf5374f2f1ff28235e08863808045 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Mon, 3 Jul 2023 09:22:15 +0530 Subject: [PATCH 08/13] Updated the comment and added the log statement --- gcp/table_gcp_logging_log_entry.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 6607bba4..5233ac6d 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -135,7 +135,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Type: proto.ColumnType_JSON, }, - // standard steampipe columns + // Standard steampipe columns { Name: "title", Description: ColumnDescriptionTitle, @@ -143,7 +143,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Transform: transform.FromField("InsertId"), }, - // standard gcp columns + // Standard gcp columns { Name: "location", Description: ColumnDescriptionLocation, @@ -167,6 +167,7 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi // Create Service Connection service, err := LoggingService(ctx, d) if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.listGcpLoggingLogEntries", "service_error", err) return nil, err } @@ -240,6 +241,7 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi return nil }, ); err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.listGcpLoggingLogEntries", "api_error", err) return nil, err } @@ -252,6 +254,7 @@ func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.H // Create Service Connection service, err := LoggingService(ctx, d) if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.getGcpLoggingLogEntry", "service_error", err) return nil, err } @@ -278,6 +281,7 @@ func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.H op, err := service.Entries.List(param).Do() if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.getGcpLoggingLogEntry", "api_error", err) return nil, err } From 9f22821f20161c21b3d05d428438630335ae07e4 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Wed, 5 Jul 2023 11:13:00 +0530 Subject: [PATCH 09/13] Made changes as per review comments --- docs/tables/gcp_logging_log_entry.md | 3 +-- gcp/table_gcp_logging_log_entry.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md index 0c318362..4eb5c8df 100644 --- a/docs/tables/gcp_logging_log_entry.md +++ b/docs/tables/gcp_logging_log_entry.md @@ -1,7 +1,6 @@ - # Table: gcp_logging_log_entry -AWS Logging Log Entry refers to a single log event or entry in a log stream within an AWS service's logging system. It contains information about a specific occurrence or activity that is being logged. Log entries typically include details such as timestamp, log message, log level, request ID, and other relevant metadata. +In Google Cloud Platform (GCP), a logging log entry represents a single log event captured by GCP's logging service. It contains information about a specific occurrence or action that took place within a GCP resource or service. Each log entry contains various metadata and data fields that provide details about the event, such as the log severity, timestamp, log message, log name, resource information, and any additional structured data associated with the event. ## Examples diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 5233ac6d..310bb341 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -124,7 +124,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Transform: transform.FromField("Split.Uid"), }, { - Name: "resource_lebels", + Name: "resource_labels", Description: "Values for all of the labels listed in the associated monitored resource descriptor.", Type: proto.ColumnType_JSON, Transform: transform.FromField("Resource.Labels"), From c6c43b870186cfbc318b719daca31467a8501a4f Mon Sep 17 00:00:00 2001 From: ParthaI Date: Thu, 6 Jul 2023 15:56:34 +0530 Subject: [PATCH 10/13] Added more optional quals to query this table --- gcp/table_gcp_logging_log_entry.go | 85 +++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 310bb341..ec527f00 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -2,6 +2,7 @@ package gcp import ( "context" + "time" "github.com/turbot/go-kit/types" "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" @@ -27,6 +28,11 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { {Name: "resource_type", Require: plugin.Optional}, {Name: "severity", Require: plugin.Optional}, {Name: "log_name", Require: plugin.Optional}, + {Name: "span_id", Require: plugin.Optional}, + {Name: "text_payload", Require: plugin.Optional}, + {Name: "receive_timestamp", Require: plugin.Optional}, + {Name: "timestamp", Require: plugin.Optional}, + {Name: "trace", Require: plugin.Optional}, }, }, Columns: []*plugin.Column{ @@ -194,30 +200,7 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi ProjectIds: []string{project}, } - resourceType := d.EqualsQualString("resource_type") - severity := d.EqualsQualString("severity") - logName := d.EqualsQualString("log_name") - filter := "" - - if resourceType != "" { - filter = "resource.type" + " = \"" + resourceType + "\"" - } - - if severity != "" { - if filter != "" { - filter = filter + " AND severity" + " = \"" + severity + "\"" - } else { - filter = "severity" + " = \"" + severity + "\"" - } - } - - if logName != "" { - if filter != "" { - filter = filter + " AND logName" + " = \"" + logName + "\"" - } else { - filter = "logName" + " = \"" + logName + "\"" - } - } + filter := buildLoggingLogEntryFilterParam(d.Quals) if filter != "" { param.Filter = filter @@ -276,7 +259,7 @@ func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.H if insertId != "" { filter = "insertId" + " = \"" + insertId + "\"" } - param.Filter = filter + param.Filter = filter op, err := service.Entries.List(param).Do() @@ -291,3 +274,55 @@ func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.H return nil, nil } + +//// UTILITY FUNCTION + +func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string { + filter := "" + + filterQuals := []filterQualMap{ + {"resource_type", "resource.type", "string"}, + {"severity", "severity", "string"}, + {"log_name", "logName", "string"}, + {"span_id", "spanId", "string"}, + {"text_payload", "textPayload", "string"}, + {"trace", "trace", "string"}, + {"receive_timestamp", "receiveTimestamp", "timestamp"}, + {"timestamp", "timestamp", "timestamp"}, + } + + for _, filterQualItem := range filterQuals { + filterQual := equalQuals[filterQualItem.ColumnName] + if filterQual == nil { + continue + } + + // Check only if filter qual map matches with optional column name + if filterQual.Name == filterQualItem.ColumnName { + if filterQual.Quals == nil { + continue + } + } + + for _, qual := range filterQual.Quals { + if qual.Value != nil { + value := qual.Value + switch filterQualItem.Type { + case "string": + if filter == "" { + filter = filterQualItem.PropertyPath + " = \"" + value.GetStringValue() + "\"" + } else { + filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetStringValue() + "\"" + } + case "timestamp": + if filter == "" { + filter = filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } else { + filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } + } + } + } + } + return filter +} From 2ed32d144f0e7c80fc6b7e6572dae8e1efdf1d39 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Thu, 6 Jul 2023 16:02:55 +0530 Subject: [PATCH 11/13] Added log_entry_operation_id as optional quals --- gcp/table_gcp_logging_log_entry.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index ec527f00..8a3e271e 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -33,6 +33,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { {Name: "receive_timestamp", Require: plugin.Optional}, {Name: "timestamp", Require: plugin.Optional}, {Name: "trace", Require: plugin.Optional}, + {Name: "log_entry_operation_id", Require: plugin.Optional}, }, }, Columns: []*plugin.Column{ @@ -287,6 +288,7 @@ func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string {"span_id", "spanId", "string"}, {"text_payload", "textPayload", "string"}, {"trace", "trace", "string"}, + {"log_entry_operation_id", "operation.id", "string"}, {"receive_timestamp", "receiveTimestamp", "timestamp"}, {"timestamp", "timestamp", "timestamp"}, } From b87a6fe5abae2c2d34b7d475d1e5751c0b2d88f8 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Wed, 12 Jul 2023 16:09:53 +0530 Subject: [PATCH 12/13] Added filter as optional quals and updated the doc with more example query using filter optional quals --- docs/tables/gcp_logging_log_entry.md | 93 +++++++++++++++++++++++++++- gcp/table_gcp_logging_log_entry.go | 17 ++++- 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md index 4eb5c8df..faecb31e 100644 --- a/docs/tables/gcp_logging_log_entry.md +++ b/docs/tables/gcp_logging_log_entry.md @@ -2,6 +2,21 @@ In Google Cloud Platform (GCP), a logging log entry represents a single log event captured by GCP's logging service. It contains information about a specific occurrence or action that took place within a GCP resource or service. Each log entry contains various metadata and data fields that provide details about the event, such as the log severity, timestamp, log message, log name, resource information, and any additional structured data associated with the event. +**Important notes:** + +- For improved performance, it is advised that you use the optional qual `timestamp` to limit the result set to a specific time period. +- This table supports optional quals. Queries with optional quals are optimised to use Logging filters. Optional quals are supported for the following columns: + - `resource_type` + - `severity` + - `log_name` + - `span_id` + - `text_payload` + - `receive_timestamp` + - `timestamp` + - `trace` + - `log_entry_operation_id` + - `filter` + ## Examples ### Basic info @@ -66,6 +81,41 @@ where timestamp >= now() - interval '30' day; ``` +### List log entries that occurred between five to ten minutes ago + +```sql +select + log_name, + insert_id, + receive_timestamp, + trace_sampled, + severity, + resource_type +from + gcp_logging_log_entry +where + log_name = 'projects/parker-abbb/logs/cloudaudit.googleapis.com%2Factivity' +and + timestamp between (now() - interval '10 minutes') and (now() - interval '5 minutes') +order by + receive_timestamp asc; +``` + +### Get the last log entries + +```sql +select + log_name, + insert_id, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity, + text_payload +from + gcp_logging_log_entry +where log_entry_operation_last; +``` ### Filter log entries by log name ```sql @@ -80,5 +130,46 @@ select from gcp_logging_log_entry where - log_name = 'projects/parker-aaa/logs/cloudaudit.googleapis.com%2Factivity'; + log_name = 'projects/parker-abbb/logs/cloudaudit.googleapis.com%2Factivity'; +``` + +## Filter examples + +For more information on Logging log entry filters, please refer to [Filter Pattern Syntax](https://cloud.google.com/logging/docs/view/logging-query-language). + +### List log entries of Compute Engine VMs with serverity error + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity +from + gcp_logging_log_entry +where + filter = 'resource.type = "gce_instance" AND (severity = ERROR OR "error")'; +``` + +### List events originating from a specific IP address range that occurred over the last hour + +```sql +select + log_name, + insert_id, + receive_timestamp, + resource_type, + severity, + timestamp, + resource_labels +from + gcp_logging_log_entry +where + filter = 'logName = "projects/my_project/logs/my_log" AND ip_in_net(jsonPayload.realClientIP, "10.1.2.0/24")' + and timestamp >= now() - interval '1 hour' +order by + receive_timestamp asc; ``` \ No newline at end of file diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 8a3e271e..5ba2b644 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -34,6 +34,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { {Name: "timestamp", Require: plugin.Optional}, {Name: "trace", Require: plugin.Optional}, {Name: "log_entry_operation_id", Require: plugin.Optional}, + {Name: "filter", Require: plugin.Optional, CacheMatch: "exact"}, }, }, Columns: []*plugin.Column{ @@ -59,6 +60,12 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Type: proto.ColumnType_BOOL, Transform: transform.FromField("Operation.Last"), }, + { + Name: "filter", + Type: proto.ColumnType_STRING, + Description: "The filter pattern for the search.", + Transform: transform.FromQual("filter"), + }, { Name: "log_entry_operation_id", Description: "An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation.", @@ -150,7 +157,7 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Transform: transform.FromField("InsertId"), }, - // Standard gcp columns + // Standard GCP columns { Name: "location", Description: ColumnDescriptionLocation, @@ -201,7 +208,13 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi ProjectIds: []string{project}, } - filter := buildLoggingLogEntryFilterParam(d.Quals) + filter := "" + + if d.EqualsQualString("filter") != "" { + filter = d.EqualsQualString("filter") + } else { + filter = buildLoggingLogEntryFilterParam(d.Quals) + } if filter != "" { param.Filter = filter From f3e663ff0917d3c03ce01a66364111df8937d821 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Wed, 12 Jul 2023 16:15:06 +0530 Subject: [PATCH 13/13] Formated the doc --- docs/tables/gcp_logging_log_entry.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md index faecb31e..65cbb253 100644 --- a/docs/tables/gcp_logging_log_entry.md +++ b/docs/tables/gcp_logging_log_entry.md @@ -116,6 +116,7 @@ from gcp_logging_log_entry where log_entry_operation_last; ``` + ### Filter log entries by log name ```sql