diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md new file mode 100644 index 00000000..65cbb253 --- /dev/null +++ b/docs/tables/gcp_logging_log_entry.md @@ -0,0 +1,176 @@ +# Table: gcp_logging_log_entry + +In Google Cloud Platform (GCP), a logging log entry represents a single log event captured by GCP's logging service. It contains information about a specific occurrence or action that took place within a GCP resource or service. Each log entry contains various metadata and data fields that provide details about the event, such as the log severity, timestamp, log message, log name, resource information, and any additional structured data associated with the event. + +**Important notes:** + +- For improved performance, it is advised that you use the optional qual `timestamp` to limit the result set to a specific time period. +- This table supports optional quals. Queries with optional quals are optimised to use Logging filters. Optional quals are supported for the following columns: + - `resource_type` + - `severity` + - `log_name` + - `span_id` + - `text_payload` + - `receive_timestamp` + - `timestamp` + - `trace` + - `log_entry_operation_id` + - `filter` + +## Examples + +### Basic info + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_id, + receive_timestamp +from + gcp_logging_log_entry; +``` + +### Get log entries by resource type + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + resource_type, + span_id, + text_payload +from + gcp_logging_log_entry +where + resource_type = 'audited_resource'; +``` + +### List log entries with NOTICE severity + +```sql +select + log_name, + insert_id, + resource_type, + severity, + span_id, + timestamp +from + gcp_logging_log_entry +where + severity = 'NOTICE'; +``` + +### List log entries in last 30 days + +```sql +select + log_name, + insert_id, + receive_timestamp, + trace_sampled, + span_id, + timestamp +from + gcp_logging_log_entry +where + timestamp >= now() - interval '30' day; +``` + +### List log entries that occurred between five to ten minutes ago + +```sql +select + log_name, + insert_id, + receive_timestamp, + trace_sampled, + severity, + resource_type +from + gcp_logging_log_entry +where + log_name = 'projects/parker-abbb/logs/cloudaudit.googleapis.com%2Factivity' +and + timestamp between (now() - interval '10 minutes') and (now() - interval '5 minutes') +order by + receive_timestamp asc; +``` + +### Get the last log entries + +```sql +select + log_name, + insert_id, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity, + text_payload +from + gcp_logging_log_entry +where log_entry_operation_last; +``` + +### Filter log entries by log name + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity +from + gcp_logging_log_entry +where + log_name = 'projects/parker-abbb/logs/cloudaudit.googleapis.com%2Factivity'; +``` + +## Filter examples + +For more information on Logging log entry filters, please refer to [Filter Pattern Syntax](https://cloud.google.com/logging/docs/view/logging-query-language). + +### List log entries of Compute Engine VMs with serverity error + +```sql +select + log_name, + insert_id, + log_entry_operation_first, + log_entry_operation_last, + receive_timestamp, + resource_type, + severity +from + gcp_logging_log_entry +where + filter = 'resource.type = "gce_instance" AND (severity = ERROR OR "error")'; +``` + +### List events originating from a specific IP address range that occurred over the last hour + +```sql +select + log_name, + insert_id, + receive_timestamp, + resource_type, + severity, + timestamp, + resource_labels +from + gcp_logging_log_entry +where + filter = 'logName = "projects/my_project/logs/my_log" AND ip_in_net(jsonPayload.realClientIP, "10.1.2.0/24")' + and timestamp >= now() - interval '1 hour' +order by + receive_timestamp asc; +``` \ No newline at end of file diff --git a/gcp/plugin.go b/gcp/plugin.go index 1bfa2e09..6d9f8dd1 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -95,6 +95,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_kubernetes_node_pool": tableGcpKubernetesNodePool(ctx), "gcp_logging_bucket": tableGcpLoggingBucket(ctx), "gcp_logging_exclusion": tableGcpLoggingExclusion(ctx), + "gcp_logging_log_entry": tableGcpLoggingLogEntry(ctx), "gcp_logging_metric": tableGcpLoggingMetric(ctx), "gcp_logging_sink": tableGcpLoggingSink(ctx), "gcp_monitoring_alert_policy": tableGcpMonitoringAlert(ctx), diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go new file mode 100644 index 00000000..5ba2b644 --- /dev/null +++ b/gcp/table_gcp_logging_log_entry.go @@ -0,0 +1,343 @@ +package gcp + +import ( + "context" + "time" + + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + + "google.golang.org/api/logging/v2" +) + +//// TABLE DEFINITION + +func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_logging_log_entry", + Description: "GCP Logging Log Entry", + Get: &plugin.GetConfig{ + KeyColumns: plugin.SingleColumn("insert_id"), + Hydrate: getGcpLoggingLogEntry, + }, + List: &plugin.ListConfig{ + Hydrate: listGcpLoggingLogEntries, + KeyColumns: plugin.KeyColumnSlice{ + {Name: "resource_type", Require: plugin.Optional}, + {Name: "severity", Require: plugin.Optional}, + {Name: "log_name", Require: plugin.Optional}, + {Name: "span_id", Require: plugin.Optional}, + {Name: "text_payload", Require: plugin.Optional}, + {Name: "receive_timestamp", Require: plugin.Optional}, + {Name: "timestamp", Require: plugin.Optional}, + {Name: "trace", Require: plugin.Optional}, + {Name: "log_entry_operation_id", Require: plugin.Optional}, + {Name: "filter", Require: plugin.Optional, CacheMatch: "exact"}, + }, + }, + Columns: []*plugin.Column{ + { + Name: "log_name", + Description: "The resource name of the log to which this log entry belongs to.", + Type: proto.ColumnType_STRING, + }, + { + Name: "insert_id", + Description: "A unique identifier for the log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "log_entry_operation_first", + Description: "Set this to True if this is the first log entry in the operation.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("Operation.First"), + }, + { + Name: "log_entry_operation_last", + Description: "Set this to True if this is the last log entry in the operation.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("Operation.Last"), + }, + { + Name: "filter", + Type: proto.ColumnType_STRING, + Description: "The filter pattern for the search.", + Transform: transform.FromQual("filter"), + }, + { + Name: "log_entry_operation_id", + Description: "An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Operation.Id"), + }, + { + Name: "log_entry_operation_producer", + Description: "An arbitrary producer identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Operation.Producer"), + }, + { + Name: "receive_timestamp", + Description: "The time the log entry was received by Logging.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "resource_type", + Description: "The monitored resource type.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Resource.Type"), + }, + { + Name: "severity", + Description: "The severity of the log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "span_id", + Description: "The ID of the Cloud Trace (https://cloud.google.com/trace) span associated with the current operation in which the log is being written.", + Type: proto.ColumnType_STRING, + }, + { + Name: "text_payload", + Description: "The log entry payload, represented as a Unicode string (UTF-8).", + Type: proto.ColumnType_STRING, + }, + { + Name: "timestamp", + Description: "The time the event described by the log entry occurred.", + Type: proto.ColumnType_TIMESTAMP, + }, + { + Name: "trace", + Description: "The REST resource name of the trace being written to Cloud Trace (https://cloud.google.com/trace) in association with this log entry.", + Type: proto.ColumnType_STRING, + }, + { + Name: "trace_sampled", + Description: "The sampling decision of the trace associated with the log entry.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "split_index", + Description: "The index of this LogEntry in the sequence of split log entries.", + Type: proto.ColumnType_INT, + Transform: transform.FromField("Split.Index"), + }, + { + Name: "total_splits", + Description: "The total number of log entries that the original LogEntry was split into.", + Type: proto.ColumnType_INT, + Transform: transform.FromField("Split.TotalSplits"), + }, + { + Name: "split_uid", + Description: "A globally unique identifier for all log entries in a sequence of split log entries.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Split.Uid"), + }, + { + Name: "resource_labels", + Description: "Values for all of the labels listed in the associated monitored resource descriptor.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Resource.Labels"), + }, + { + Name: "source_location", + Description: "Source code location information associated with the log entry, if any.", + Type: proto.ColumnType_JSON, + }, + + // Standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("InsertId"), + }, + + // Standard GCP columns + { + Name: "location", + Description: ColumnDescriptionLocation, + Type: proto.ColumnType_STRING, + Transform: transform.FromConstant("global"), + }, + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Hydrate: plugin.HydrateFunc(getProject).WithCache(), + Transform: transform.FromValue(), + }, + }, + } +} + +//// FETCH FUNCTIONS + +func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := LoggingService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.listGcpLoggingLogEntries", "service_error", err) + return nil, err + } + + // Max limit isn't mentioned in the documentation + // Default limit is set as 1000 + pageSize := types.Int64(1000) + limit := d.QueryContext.Limit + if d.QueryContext.Limit != nil { + if *limit < *pageSize { + pageSize = limit + } + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + param := &logging.ListLogEntriesRequest{ + PageSize: *pageSize, + ProjectIds: []string{project}, + } + + filter := "" + + if d.EqualsQualString("filter") != "" { + filter = d.EqualsQualString("filter") + } else { + filter = buildLoggingLogEntryFilterParam(d.Quals) + } + + if filter != "" { + param.Filter = filter + } + + op := service.Entries.List(param) + + if err := op.Pages( + ctx, + func(page *logging.ListLogEntriesResponse) error { + for _, entry := range page.Entries { + d.StreamListItem(ctx, entry) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + page.NextPageToken = "" + return nil + } + } + return nil + }, + ); err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.listGcpLoggingLogEntries", "api_error", err) + return nil, err + } + + return nil, err +} + +//// HYDRATE FUNCTION + +func getGcpLoggingLogEntry(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := LoggingService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.getGcpLoggingLogEntry", "service_error", err) + return nil, err + } + + // Get project details + getProjectCached := plugin.HydrateFunc(getProject).WithCache() + projectId, err := getProjectCached(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + param := &logging.ListLogEntriesRequest{ + ProjectIds: []string{project}, + } + + insertId := d.EqualsQualString("insert_id") + filter := "" + + if insertId != "" { + filter = "insertId" + " = \"" + insertId + "\"" + } + param.Filter = filter + + op, err := service.Entries.List(param).Do() + + if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.getGcpLoggingLogEntry", "api_error", err) + return nil, err + } + + if len(op.Entries) > 0 { + return op.Entries[0], nil + } + + return nil, nil +} + +//// UTILITY FUNCTION + +func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string { + filter := "" + + filterQuals := []filterQualMap{ + {"resource_type", "resource.type", "string"}, + {"severity", "severity", "string"}, + {"log_name", "logName", "string"}, + {"span_id", "spanId", "string"}, + {"text_payload", "textPayload", "string"}, + {"trace", "trace", "string"}, + {"log_entry_operation_id", "operation.id", "string"}, + {"receive_timestamp", "receiveTimestamp", "timestamp"}, + {"timestamp", "timestamp", "timestamp"}, + } + + for _, filterQualItem := range filterQuals { + filterQual := equalQuals[filterQualItem.ColumnName] + if filterQual == nil { + continue + } + + // Check only if filter qual map matches with optional column name + if filterQual.Name == filterQualItem.ColumnName { + if filterQual.Quals == nil { + continue + } + } + + for _, qual := range filterQual.Quals { + if qual.Value != nil { + value := qual.Value + switch filterQualItem.Type { + case "string": + if filter == "" { + filter = filterQualItem.PropertyPath + " = \"" + value.GetStringValue() + "\"" + } else { + filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetStringValue() + "\"" + } + case "timestamp": + if filter == "" { + filter = filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } else { + filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } + } + } + } + } + return filter +}