add cirrus post[C]-batch lambda

Element84 · Aug 19, 2024 · 04da35a · 04da35a
1 parent d70e231
commit 04da35a
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 0 deletions.
diff --git a/default.tfvars b/default.tfvars
@@ -180,6 +180,10 @@ cirrus_inputs = {
       timeout = 15
       memory  = 128
   }
+  post_batch_lambda = {
+      timeout = 15
+      memory  = 128
+  }
 }
 
 cirrus_dashboard_inputs = {

diff --git a/inputs.tf b/inputs.tf
@@ -375,6 +375,10 @@ variable "cirrus_inputs" {
       timeout = number
       memory  = number
     })
+    post_batch_lambda = object({
+      timeout = number
+      memory  = number
+    })
   })
   default = {
     data_bucket    = "cirrus-data-bucket-name"
@@ -405,6 +409,10 @@ variable "cirrus_inputs" {
       timeout = 15
       memory  = 128
     }
+    post_batch_lambda = {
+      timeout = 15
+      memory  = 128
+    }
   }
 }
 

diff --git a/modules/cirrus/functions.tf b/modules/cirrus/functions.tf
@@ -16,6 +16,8 @@ module "functions" {
   cirrus_update_state_lambda_memory                = var.cirrus_update_state_lambda_memory
   cirrus_pre_batch_lambda_timeout                  = var.cirrus_pre_batch_lambda_timeout
   cirrus_pre_batch_lambda_memory                   = var.cirrus_pre_batch_lambda_memory
+  cirrus_post_batch_lambda_timeout                 = var.cirrus_pre_batch_lambda_timeout
+  cirrus_post_batch_lambda_memory                  = var.cirrus_pre_batch_lambda_memory
   cirrus_state_dynamodb_table_name                 = module.base-builtins.cirrus_state_dynamodb_table_name
   cirrus_state_dynamodb_table_arn                  = module.base-builtins.cirrus_state_dynamodb_table_arn
   cirrus_state_event_timestreamwrite_database_name = module.base-builtins.cirrus_state_event_timestreamwrite_database_name

diff --git a/modules/cirrus/functions/inputs.tf b/modules/cirrus/functions/inputs.tf
@@ -73,6 +73,18 @@ variable "cirrus_pre_batch_lambda_memory" {
   default     = 128
 }
 
+variable "cirrus_post_batch_lambda_timeout" {
+  description = "Cirrus post-batch lambda timeout (sec)"
+  type        = number
+  default     = 15
+}
+
+variable "cirrus_post_batch_lambda_memory" {
+  description = "Cirrus post-batch lambda memory (MB)"
+  type        = number
+  default     = 128
+}
+
 variable "cirrus_state_dynamodb_table_name" {
   description = "Cirrus state dynamodb table name"
   type        = string

diff --git a/modules/cirrus/functions/post-batch.tf b/modules/cirrus/functions/post-batch.tf
@@ -0,0 +1,99 @@
+resource "aws_iam_role" "cirrus_post_batch_lambda_role" {
+  name_prefix = "${var.cirrus_prefix}-process-role-"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Effect": "Allow"
+    }
+  ]
+}
+EOF
+
+}
+
+resource "aws_iam_policy" "cirrus_post_batch_lambda_policy" {
+  name_prefix = "${var.cirrus_prefix}-process-policy-"
+
+  # TODO: the secret thing is probably not gonna work without some fixes in boto3utils...
+  # We should probably reconsider if this is the right solution.
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:ListBucket",
+        "s3:GetObject",
+        "s3:GetBucketLocation"
+      ],
+      "Resource": "*",
+      "Effect": "Allow"
+    },
+    {
+      "Action": "secretsmanager:GetSecretValue",
+      "Resource": [
+        "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${var.cirrus_prefix}*"
+      ],
+      "Effect": "Allow"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject"
+      ],
+      "Resource": "arn:aws:s3:::${var.cirrus_payload_bucket}*"
+    }
+    {
+      "Effect": "Allow",
+      "Action": [
+        "logs:GetLogEvents"
+      ],
+      "Resource": "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/batch/*"
+  ]
+}
+EOF
+
+}
+
+resource "aws_iam_role_policy_attachment" "cirrus_post_batch_lambda_role_policy_attachment1" {
+  role       = aws_iam_role.cirrus_post_batch_lambda_role.name
+  policy_arn = aws_iam_policy.cirrus_post_batch_lambda_policy.arn
+}
+
+resource "aws_iam_role_policy_attachment" "cirrus_post_batch_lambda_role_policy_attachment2" {
+  role       = aws_iam_role.cirrus_post_batch_lambda_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
+}
+
+resource "aws_lambda_function" "cirrus_post_batch" {
+  filename                       = "${path.module}/cirrus-lambda-dist.zip"
+  function_name                  = "${var.cirrus_prefix}-post-batch"
+  description                    = "Cirrus Post-batch Lambda"
+  role                           = aws_iam_role.cirrus_post_batch_lambda_role.arn
+  handler                        = "post_batch.lambda_handler"
+  source_code_hash               = filebase64sha256("${path.module}/cirrus-lambda-dist.zip")
+  runtime                        = "python3.12"
+  timeout                        = var.cirrus_post_batch_lambda_timeout
+  memory_size                    = var.cirrus_post_batch_lambda_memory
+  publish                        = true
+  architectures                  = ["arm64"]
+
+  environment {
+    variables = {
+      CIRRUS_LOG_LEVEL           = var.cirrus_log_level
+      CIRRUS_PAYLOAD_BUCKET      = var.cirrus_payload_bucket
+    }
+  }
+
+  vpc_config {
+    security_group_ids           = var.vpc_security_group_ids
+    subnet_ids                   = var.vpc_subnet_ids
+  }
+}
diff --git a/modules/cirrus/inputs.tf b/modules/cirrus/inputs.tf
@@ -110,6 +110,18 @@ variable "cirrus_pre_batch_lambda_memory" {
   default     = 128
 }
 
+variable "cirrus_post_batch_lambda_timeout" {
+  description = "Cirrus post-batch lambda timeout (sec)"
+  type        = number
+  default     = 15
+}
+
+variable "cirrus_post_batch_lambda_memory" {
+  description = "Cirrus post-batch lambda memory (MB)"
+  type        = number
+  default     = 128
+}
+
 variable "vpc_subnet_ids" {
   description = "List of subnet ids in the FilmDrop vpc"
   type        = list(string)

diff --git a/profiles/cirrus/inputs.tf b/profiles/cirrus/inputs.tf
@@ -57,6 +57,10 @@ variable "cirrus_inputs" {
       timeout = number
       memory  = number
     })
+    post_batch_lambda = object({
+      timeout = number
+      memory  = number
+    })
   })
   default = {
     data_bucket    = "cirrus-data-bucket-name"
@@ -87,5 +91,9 @@ variable "cirrus_inputs" {
       timeout = 15
       memory  = 128
     }
+    post_batch_lambda = {
+      timeout = 15
+      memory  = 128
+    }
   }
 }
diff --git a/profiles/cirrus/main.tf b/profiles/cirrus/main.tf
@@ -21,4 +21,6 @@ module "cirrus" {
   cirrus_update_state_lambda_memory                         = var.cirrus_inputs.update_state_lambda.memory
   cirrus_pre_batch_lambda_timeout                           = var.cirrus_inputs.pre_batch_lambda.timeout
   cirrus_pre_batch_lambda_memory                            = var.cirrus_inputs.pre_batch_lambda.memory
+  cirrus_post_batch_lambda_timeout                          = var.cirrus_inputs.post_batch_lambda.timeout
+  cirrus_post_batch_lambda_memory                           = var.cirrus_inputs.post_batch_lambda.memory
 }
diff --git a/profiles/core/inputs.tf b/profiles/core/inputs.tf
@@ -375,6 +375,10 @@ variable "cirrus_inputs" {
       timeout = number
       memory  = number
     })
+    post_batch_lambda = object({
+      timeout = number
+      memory  = number
+    })
   })
   default = {
     data_bucket    = "cirrus-data-bucket-name"
@@ -405,6 +409,10 @@ variable "cirrus_inputs" {
       timeout = 15
       memory  = 128
     }
+    post_batch_lambda = {
+      timeout = 15
+      memory  = 128
+    }
   }
 }