feat(promtail): Adding S3 log parser support for AWS GuardDuty (grafana#13148)

Co-authored-by: James Callahan <https://github.com/james-callahan>ase enter the commit message for your changes. Lines starting

Author: samuelebstein

tools/lambda-promtail/lambda-promtail/s3.go

@@ -50,6 +50,7 @@ const (
 	LB_NLB_TYPE                string = "net"
 	LB_ALB_TYPE                string = "app"
 	WAF_LOG_TYPE               string = "WAFLogs"
+	GUARDDUTY_LOG_TYPE         string = "GuardDuty"
 )
 
 var (
@@ -75,13 +76,18 @@ var (
 	// source: https://docs.aws.amazon.com/waf/latest/developerguide/logging-s3.html
 	// format: aws-waf-logs-suffix[/prefix]/AWSLogs/aws-account-id/WAFLogs/region/webacl-name/year/month/day/hour/minute/aws-account-id_waflogs_region_webacl-name_timestamp_hash.log.gz
 	// example: aws-waf-logs-test/AWSLogs/11111111111/WAFLogs/us-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz
+	// AWS GuardDuty
+	// source: https://docs.aws.amazon.com/guardduty/latest/ug/guardduty_exportfindings.html
+	// format: my-bucket/AWSLogs/aws-account-id/GuardDuty/region/year/month/day/random-string.jsonl.gz
+	// example: my-bucket/AWSLogs/123456789012/GuardDuty/us-east-1/2024/05/30/07a3f2ce-1485-3031-b842-e1f324c4a48d.jsonl.gz
 	defaultFilenameRegex     = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?P<lb_type>app|net)\.*?)?(?P<src>[a-zA-Z0-9\-]+)`)
 	defaultTimestampRegex    = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`)
 	cloudtrailFilenameRegex  = regexp.MustCompile(`AWSLogs\/(?P<organization_id>o-[a-z0-9]{10,32})?\/?(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
 	cloudfrontFilenameRegex  = regexp.MustCompile(`(?P<prefix>.*)\/(?P<src>[A-Z0-9]+)\.(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)-(.+)`)
 	cloudfrontTimestampRegex = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+\s\d+:\d+:\d+)`)
 	wafFilenameRegex         = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>WAFLogs)\/(?P<region>[\w-]+)\/(?P<src>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/(?P<hour>\d+)\/(?P<minute>\d+)\/\d+\_waflogs\_[\w-]+_[\w-]+_\d+T\d+Z_\w+`)
 	wafTimestampRegex        = regexp.MustCompile(`"timestamp":\s*(?P<timestamp>\d+),`)
+	guarddutyFilenameRegex   = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>GuardDuty)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/.+`)
 	parsers                  = map[string]parserConfig{
 		FLOW_LOG_TYPE: {
 			logTypeLabel:    "s3_vpc_flow",
@@ -122,6 +128,14 @@ var (
 			timestampRegex: wafTimestampRegex,
 			timestampType:  "unix",
 		},
+		GUARDDUTY_LOG_TYPE: {
+			logTypeLabel:    "s3_guardduty",
+			filenameRegex:   guarddutyFilenameRegex,
+			ownerLabelKey:   "account_id",
+			timestampFormat: time.RFC3339,
+			timestampRegex:  defaultTimestampRegex,
+			timestampType:   "string",
+		},
 	}
 )
 
@@ -165,7 +179,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
 	ls = applyLabels(ls)
 
 	// extract the timestamp of the nested event and sends the rest as raw json
-	if labels["type"] == CLOUDTRAIL_LOG_TYPE {
+	if labels["type"] == CLOUDTRAIL_LOG_TYPE || labels["type"] == GUARDDUTY_LOG_TYPE {
 		records := make(chan Record)
 		jsonStream := NewJSONStream(records)
 		go jsonStream.Start(gzreader, parser.skipHeaderCount)

tools/lambda-promtail/lambda-promtail/s3_test.go

@@ -93,6 +93,38 @@ func Test_getLabels(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "s3_guardduty",
+			args: args{
+				record: events.S3EventRecord{
+					AWSRegion: "us-east-1",
+					S3: events.S3Entity{
+						Bucket: events.S3Bucket{
+							Name: "s3_guardduty_test",
+							OwnerIdentity: events.S3UserIdentity{
+								PrincipalID: "test",
+							},
+						},
+						Object: events.S3Object{
+							Key: "AWSLogs/123456789012/GuardDuty/us-east-1/2024/05/30/07a3f2ce-1485-3031-b842-e1f324c4a48d.jsonl.gz",
+						},
+					},
+				},
+			},
+			want: map[string]string{
+				"account_id":    "123456789012",
+				"bucket":        "s3_guardduty_test",
+				"bucket_owner":  "test",
+				"bucket_region": "us-east-1",
+				"day":           "30",
+				"key":           "AWSLogs/123456789012/GuardDuty/us-east-1/2024/05/30/07a3f2ce-1485-3031-b842-e1f324c4a48d.jsonl.gz",
+				"month":         "05",
+				"region":        "us-east-1",
+				"type":          GUARDDUTY_LOG_TYPE,
+				"year":          "2024",
+			},
+			wantErr: false,
+		},
 		{
 			name: "s3_flow_logs",
 			args: args{

tools/lambda-promtail/main.tf

@@ -251,7 +251,7 @@ resource "aws_s3_bucket_notification" "this" {
     lambda_function_arn = aws_lambda_function.this.arn
     events              = ["s3:ObjectCreated:*"]
     filter_prefix       = "AWSLogs/"
-    filter_suffix       = ".log.gz"
+    filter_suffix       = var.filter_suffix
   }
 
   depends_on = [

tools/lambda-promtail/variables.tf

@@ -16,6 +16,12 @@ variable "bucket_names" {
   default     = []
 }
 
+variable "filter_suffix" {
+  type        = string
+  description = "Suffix for S3 bucket notification filter"
+  default     = ".gz"
+}
+
 variable "log_group_names" {
   type        = set(string)
   description = "List of CloudWatch Log Group names to create Subscription Filters for."