Cirrus Tasks and Workflows (#84)

* Renamed Cirrus function directory and module
* Renamed Cirrus base-builtins module
* Added Cirrus task-batch-compute module
* Added Cirrus task module
* Added Cirrus workflow module
* Switched pre-batch and post-batch configuration files to conditional Cirrus task module inputs

---------

Co-authored-by: Ryan Bickford <[email protected]>

.gitignore

@@ -32,3 +32,4 @@ stac-server.tgz
 terraform-visual-report
 terraform.tgz
 modules/stac-server/historical-ingest/lambda/package/*
+!modules/cirrus/cirrus-lambda-dist.zip

.snyk

@@ -19,7 +19,7 @@ ignore:
     - 'modules/stac-server/api.tf > *':
         reason: Open API - no auth required
         created: 2024-03-21T14:35:23.783Z
-    - './profiles/core/../cirrus/../../modules/cirrus/functions/api.tf > *':
+    - './profiles/core/../cirrus/../../modules/cirrus/builtin-functions/api.tf > *':
         reason: Open API - no auth required
         created: 2024-09-12T14:35:23.783Z
   SNYK-CC-00185:

ci.tfvars

@@ -184,6 +184,10 @@ cirrus_inputs = {
     timeout = 15
     memory  = 128
   }
+  # TODO - CVG - add proper tests
+  task_batch_compute = []
+  tasks              = []
+  workflows          = []
 }
 
 cirrus_dashboard_inputs = {

default.tfvars

@@ -188,6 +188,9 @@ cirrus_inputs = {
     timeout = 15
     memory  = 128
   }
+  task_batch_compute = []
+  tasks              = []
+  workflows          = []
 }
 
 cirrus_dashboard_inputs = {

inputs.tf

@@ -373,6 +373,195 @@ variable "cirrus_inputs" {
       timeout = number
       memory  = number
     })
+    task_batch_compute = optional(list(object({
+      name = string
+      batch_compute_environment_existing = optional(object({
+        name       = string
+        is_fargate = bool
+      }))
+      batch_compute_environment = optional(object({
+        compute_resources = object({
+          max_vcpus           = number
+          type                = string
+          allocation_strategy = optional(string)
+          bid_percentage      = optional(number)
+          desired_vcpus       = optional(number)
+          ec2_configuration = optional(object({
+            image_id_override = optional(string)
+            image_type        = optional(string)
+          }))
+          ec2_key_pair       = optional(string)
+          instance_type      = optional(list(string))
+          min_vcpus          = optional(number)
+          placement_group    = optional(string)
+          security_group_ids = optional(list(string))
+          subnets            = optional(list(string))
+        })
+        state = optional(string)
+        type  = optional(string)
+        update_policy = optional(object({
+          job_execution_timeout_minutes = number
+          terminate_jobs_on_update      = bool
+        }))
+      }))
+      batch_job_queue_existing = optional(object({
+        name = string
+      }))
+      batch_job_queue = optional(object({
+        fair_share_policy = optional(object({
+          compute_reservation = optional(number)
+          share_decay_seconds = optional(number)
+          share_distributions = list(object({
+            share_identifier = string
+            weight_factor    = number
+          }))
+        }))
+        state = optional(string)
+      }))
+      ec2_launch_template_existing = optional(object({
+        name = string
+      }))
+      ec2_launch_template = optional(object({
+        user_data     = optional(string)
+        ebs_optimized = optional(bool)
+        block_device_mappings = optional(list(object({
+          device_name  = string
+          no_device    = optional(bool)
+          virtual_name = optional(string)
+          ebs = optional(object({
+            delete_on_termination = optional(bool)
+            encrypted             = optional(bool)
+            iops                  = optional(string)
+            kms_key_id            = optional(string)
+            snapshot_id           = optional(string)
+            throughput            = optional(number)
+            volume_size           = optional(number)
+            volume_type           = optional(string)
+          }))
+        })))
+      }))
+    })))
+    tasks = optional(list(object({
+      name = string
+      common_role_statements = optional(list(object({
+        sid           = string
+        effect        = string
+        actions       = list(string)
+        resources     = list(string)
+        not_actions   = optional(list(string))
+        not_resources = optional(list(string))
+        condition = optional(object({
+          test     = string
+          variable = string
+          values   = list(string)
+        }))
+        principals = optional(object({
+          type        = string
+          identifiers = list(string)
+        }))
+        not_principals = optional(object({
+          type        = string
+          identifiers = list(string)
+        }))
+      })))
+      lambda = optional(object({
+        description   = optional(string)
+        ecr_image_uri = optional(string)
+        filename      = optional(string)
+        image_config = optional(object({
+          command           = optional(list(string))
+          entry_point       = optional(list(string))
+          working_directory = optional(string)
+        }))
+        s3_bucket       = optional(string)
+        s3_key          = optional(string)
+        handler         = optional(string)
+        runtime         = optional(string)
+        timeout_seconds = optional(number)
+        memory_mb       = optional(number)
+        publish         = optional(bool)
+        architectures   = optional(list(string))
+        env_vars        = optional(map(string))
+        vpc_enabled     = optional(bool)
+        role_statements = optional(list(object({
+          sid           = string
+          effect        = string
+          actions       = list(string)
+          resources     = list(string)
+          not_actions   = optional(list(string))
+          not_resources = optional(list(string))
+          condition = optional(object({
+            test     = string
+            variable = string
+            values   = list(string)
+          }))
+          principals = optional(object({
+            type        = string
+            identifiers = list(string)
+          }))
+          not_principals = optional(object({
+            type        = string
+            identifiers = list(string)
+          }))
+        })))
+        alarms = optional(list(object({
+          critical            = bool
+          statistic           = string
+          metric_name         = string
+          comparison_operator = string
+          threshold           = number
+          period              = optional(number, 60)
+          evaluation_periods  = optional(number, 5)
+        })))
+      }))
+      batch = optional(object({
+        task_batch_compute_name = string
+        container_properties    = string
+        retry_strategy = optional(object({
+          attempts = number
+          evaluate_on_exit = optional(list(object({
+            action           = string
+            on_exit_code     = optional(string)
+            on_reason        = optional(string)
+            on_status_reason = optional(string)
+          })))
+        }))
+        parameters = optional(map(string))
+        role_statements = optional(list(object({
+          sid           = string
+          effect        = string
+          actions       = list(string)
+          resources     = list(string)
+          not_actions   = optional(list(string))
+          not_resources = optional(list(string))
+          condition = optional(object({
+            test     = string
+            variable = string
+            values   = list(string)
+          }))
+          principals = optional(object({
+            type        = string
+            identifiers = list(string)
+          }))
+          not_principals = optional(object({
+            type        = string
+            identifiers = list(string)
+          }))
+        })))
+        scheduling_priority = optional(number)
+        timeout_seconds     = optional(number)
+      }))
+    })))
+    workflows = optional(list(object({
+      name                   = string
+      non_cirrus_lambda_arns = optional(list(string))
+      template_filepath      = string
+      template_variables = optional(map(object({
+        task_name = string
+        task_type = string
+        task_attr = string
+      })))
+    })))
   })
   default = {
     data_bucket    = "cirrus-data-bucket-name"
@@ -412,6 +601,9 @@ variable "cirrus_inputs" {
       timeout = 15
       memory  = 128
     }
+    task_batch_compute = []
+    tasks              = []
+    workflows          = []
   }
 }
 

modules/cirrus/builtins.tf → modules/cirrus/base.tf

@@ -1,4 +1,9 @@
-module "base-builtins" {
+moved {
+  from = module.base-builtins
+  to   = module.base
+}
+
+module "base" {
   source = "./base"
 
   cirrus_prefix                                             = local.cirrus_prefix

modules/cirrus/functions/api.tf → modules/cirrus/builtin-functions/api.tf

@@ -89,12 +89,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_api_lambda_role_policy_attachm
 }
 
 resource "aws_lambda_function" "cirrus_api" {
-  filename         = "${path.module}/cirrus-lambda-dist.zip"
+  filename         = "${path.module}/../cirrus-lambda-dist.zip"
   function_name    = "${var.cirrus_prefix}-api"
   description      = "Cirrus API Lambda"
   role             = aws_iam_role.cirrus_api_lambda_role.arn
   handler          = "api.lambda_handler"
-  source_code_hash = filebase64sha256("${path.module}/cirrus-lambda-dist.zip")
+  source_code_hash = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
   runtime          = "python3.12"
   timeout          = var.cirrus_api_lambda_timeout
   memory_size      = var.cirrus_api_lambda_memory
@@ -239,7 +239,7 @@ resource "aws_api_gateway_account" "cirrus_api_gateway_cw_role" {
 }
 
 resource "aws_iam_role" "cirrus_api_gw_role" {
-  name_prefix = "${var.cirrus_prefix}-cirrus-${data.aws_region.current.name}"
+  name_prefix = "${var.cirrus_prefix}-${data.aws_region.current.name}"
 
   assume_role_policy = <<EOF
 {
@@ -258,7 +258,7 @@ EOF
 }
 
 resource "aws_iam_policy" "cirrus_api_gw_policy" {
-  name_prefix = "${var.cirrus_prefix}-cirrus-${data.aws_region.current.name}-apigw"
+  name_prefix = "${var.cirrus_prefix}-${data.aws_region.current.name}-apigw"
 
   policy = <<EOF
 {

modules/cirrus/functions/inputs.tf → modules/cirrus/builtin-functions/inputs.tf

@@ -61,30 +61,6 @@ variable "cirrus_update_state_lambda_memory" {
   default     = 128
 }
 
-variable "cirrus_pre_batch_lambda_timeout" {
-  description = "Cirrus pre-batch lambda timeout (sec)"
-  type        = number
-  default     = 15
-}
-
-variable "cirrus_pre_batch_lambda_memory" {
-  description = "Cirrus pre-batch lambda memory (MB)"
-  type        = number
-  default     = 128
-}
-
-variable "cirrus_post_batch_lambda_timeout" {
-  description = "Cirrus post-batch lambda timeout (sec)"
-  type        = number
-  default     = 15
-}
-
-variable "cirrus_post_batch_lambda_memory" {
-  description = "Cirrus post-batch lambda memory (MB)"
-  type        = number
-  default     = 128
-}
-
 variable "cirrus_state_dynamodb_table_name" {
   description = "Cirrus state dynamodb table name"
   type        = string
@@ -172,4 +148,4 @@ variable "deploy_alarms" {
   type        = bool
   default     = true
   description = "Deploy Cirrus Alarms stack"
-}
+}

modules/cirrus/functions/process.tf → modules/cirrus/builtin-functions/process.tf

@@ -121,12 +121,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_process_lambda_role_policy_att
 }
 
 resource "aws_lambda_function" "cirrus_process" {
-  filename                       = "${path.module}/cirrus-lambda-dist.zip"
+  filename                       = "${path.module}/../cirrus-lambda-dist.zip"
   function_name                  = "${var.cirrus_prefix}-process"
   description                    = "Cirrus Process Lambda"
   role                           = aws_iam_role.cirrus_process_lambda_role.arn
   handler                        = "process.lambda_handler"
-  source_code_hash               = filebase64sha256("${path.module}/cirrus-lambda-dist.zip")
+  source_code_hash               = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
   runtime                        = "python3.12"
   timeout                        = var.cirrus_process_lambda_timeout
   memory_size                    = var.cirrus_process_lambda_memory

modules/cirrus/functions/update-state.tf → modules/cirrus/builtin-functions/update-state.tf

@@ -101,12 +101,12 @@ resource "aws_iam_role_policy_attachment" "cirrus_update_state_lambda_role_polic
 }
 
 resource "aws_lambda_function" "cirrus_update_state" {
-  filename         = "${path.module}/cirrus-lambda-dist.zip"
+  filename         = "${path.module}/../cirrus-lambda-dist.zip"
   function_name    = "${var.cirrus_prefix}-update-state"
   description      = "Cirrus Update-State Lambda"
   role             = aws_iam_role.cirrus_update_state_lambda_role.arn
   handler          = "update_state.lambda_handler"
-  source_code_hash = filebase64sha256("${path.module}/cirrus-lambda-dist.zip")
+  source_code_hash = filebase64sha256("${path.module}/../cirrus-lambda-dist.zip")
   runtime          = "python3.12"
   timeout          = var.cirrus_update_state_lambda_timeout
   memory_size      = var.cirrus_update_state_lambda_memory