Remove template_variables from cirrus workflows (#92)

This commit reduces the cirrus workflow module's complexity by relying on the user to simply provide Cirrus task output attribute lookups directly in their workflow state machine definition rather than using arbitrary variable names with associated entries under a workflow config 'template_variables' attribute. The rendered JSON is now parsed via regexes to determine which resource ARNs the workflow IAM role needs permissions to execute. This also removes the need for the optional 'non_cirrus_lambda_arns' workflow config setting. The workflow IAM role's lambda and batch policy documents were merged into one with conditional statements based on whether lambda or batch resources are used by the workflow's tasks.
Element84 · Feb 5, 2025 · 587504f · 587504f
1 parent 20903e3
commit 587504f
Show file tree

Hide file tree

Showing 9 changed files with 122 additions and 286 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 
+- Cirrus workflow's `template_variables` config is removed in favor of referencing cirrus task output attributes directly
+
 ### Fixed
 
 - Fixed the Cirrus `update-state` lambda permissions to allow:

diff --git a/inputs.tf b/inputs.tf
@@ -583,13 +583,7 @@ variable "cirrus_inputs" {
     })))
     workflows = optional(list(object({
       name                   = string
-      non_cirrus_lambda_arns = optional(list(string))
-      template_filepath      = string
-      template_variables = optional(map(object({
-        task_name = string
-        task_type = string
-        task_attr = string
-      })))
+      state_machine_filepath = string
     })))
   })
   default = {

diff --git a/modules/cirrus/builtin_tasks.tf b/modules/cirrus/builtin_tasks.tf
@@ -154,20 +154,4 @@ locals {
     local.pre_batch_task_config,
     local.post_batch_task_config
   ]
-
-  # Map of pre-batch and post-batch builtin variables.
-  # Allows users to use "${PRE-BATCH}" and "${POST-BATCH}" in workflow templates
-  # without any additional config.
-  pre_batch_post_batch_task_template_variables = {
-    PRE-BATCH = {
-      task_name = local.pre_batch_task_config.name
-      task_type = "lambda"
-      task_attr = "function_arn"
-    }
-    POST-BATCH = {
-      task_name = local.post_batch_task_config.name
-      task_type = "lambda"
-      task_attr = "function_arn"
-    }
-  }
 }
diff --git a/modules/cirrus/inputs.tf b/modules/cirrus/inputs.tf
@@ -541,67 +541,32 @@ variable "cirrus_workflows" {
       - name: (required, string) Identifier for the Cirrus Workflow. Must be
         unique across all Cirrus Workflows. Valid characters are: [A-Za-z0-9-]
 
-      - non_cirrus_lambda_arns: (optional, list[string]) List of Lambda function
-        ARNs that'll be executed by the Workflow but are not managed by a Cirrus
-        task. This is necessary for granting the Workflow execution role invoke
-        permissions on these functions.
-
-      - template_filepath: (required, string) Path to an Amazon State Machine
-        definition template file. The path must be relative to the ROOT module
-        of the Terraform deployment. The template should use valid Amazon States
-        Language syntax; wherever a Cirrus Task resource ARN is needed, a
+      - state_machine_filepath: (required, string) Path to an Amazon State
+        Machine definition template file. The path must be relative to the ROOT
+        module of the Terraform deployment. The template should use valid Amazon
+        States Language syntax; wherever a Cirrus Task resource ARN is needed, a
         Terraform interpolation sequence (a "$\{...}" without the "\") may be
-        used instead. The variable name does not matter so long as there is a
-        corresponding entry in the "template_variables" argument.
-        Example template snippet:
+        used instead. The interpolation sequence should have the following form:
+          <TASK NAME>.<TASK TYPE>.<TASK ATTR>
 
-          "States": {
-            "FirstState": {
-              "Type": "Task",
-              "Resource": "$\{my-task-lambda}",  // REMOVE THE "\"
-              "Next": "SecondState",
-              ...
-            },
-
-        Cirrus may deploy and manage several builtin tasks. Resource ARNs for
-        these tasks may be referenced in a Workflow template using a predefined
-        variable name without having to supply a 'template_variable' entry.
-          - If Batch Tasks were created, the following variables may be used:
-            - PRE-BATCH: cirrus-geo pre-batch Lambda function ARN
-            - POST-BATCH: cirrus-geo post-batch Lambda function ARN
-
-      - template_variables: (optional, map[object]) A map of template variable
-        names to their corresponding Cirrus Task attributes. Assuming a Cirrus
-        Task named "my-task" with Lambda config was passed to the 'task' module,
-        the following workflow template variable config:
-
-          my-task-lambda = {
-            task_name = "my-task"
-            task_type = "lambda"
-            task_attr = "functon_arn"
-          }
-
-        when used with the example Workflow snippet above would result in the
-        following content after template interpolation:
+        Where:
+          <TASK NAME> : name of the task
+          <TASK TYPE> : one of [lambda, batch]
+          <TASK ATTR> : one of [function_arn, job_definition_arn, job_queue_arn]
 
+        Example template snippet:
           "States": {
             "FirstState": {
               "Type": "Task",
-              "Resource": "arn:aws:lambda:us-west-2:123456789012:function:my-function",
+              "Resource": "$\{my-task.lambda.function_arn}",  // REMOVE THE "\"
               "Next": "SecondState",
               ...
             },
   DESCRIPTION
 
   type = list(object({
     name                   = string
-    non_cirrus_lambda_arns = optional(list(string))
-    template_filepath      = string
-    template_variables = optional(map(object({
-      task_name = string
-      task_type = string
-      task_attr = string
-    })))
+    state_machine_filepath = string
   }))
 
   default  = []

diff --git a/modules/cirrus/tasks_and_workflows.tf b/modules/cirrus/tasks_and_workflows.tf
@@ -15,15 +15,6 @@ locals {
     local.has_batch_task ? local.pre_batch_post_batch_task_configs : []
     # ... any future builtin tasks added here ...
   )
-
-  # Construct the full map of builtin task template variables:
-  # - If at least one Batch-style task was configured, the user may reference
-  #   pre-batch and post-batch function ARNs using their builtin variable names
-  # - ...
-  builtin_task_template_variables = merge(
-    local.has_batch_task ? local.pre_batch_post_batch_task_template_variables : {}
-    # ... any future builtin task variables added here ...
-  )
 }
 
 module "task_batch_compute" {
@@ -63,8 +54,7 @@ module "workflow" {
     workflow.name => workflow
   }
 
-  cirrus_prefix                   = local.cirrus_prefix
-  cirrus_tasks                    = module.task
-  workflow_config                 = each.value
-  builtin_task_template_variables = local.builtin_task_template_variables
+  cirrus_prefix   = local.cirrus_prefix
+  cirrus_tasks    = module.task
+  workflow_config = each.value
 }
diff --git a/modules/cirrus/workflow/inputs.tf b/modules/cirrus/workflow/inputs.tf
@@ -32,113 +32,34 @@ variable "workflow_config" {
       - name: (required, string) Identifier for the Cirrus Workflow. Must be
         unique across all Cirrus Workflows. Valid characters are: [A-Za-z0-9-]
 
-      - non_cirrus_lambda_arns: (optional, list[string]) List of Lambda function
-        ARNs that'll be executed by the Workflow but are not managed by a Cirrus
-        task. This is necessary for granting the Workflow execution role invoke
-        permissions on these functions.
-
-      - template_filepath: (required, string) Path to an Amazon State Machine
-        definition template file. The path must be relative to the ROOT module
-        of the Terraform deployment. The template should use valid Amazon States
-        Language syntax; wherever a Cirrus Task resource ARN is needed, a
+      - state_machine_filepath: (required, string) Path to an Amazon State
+        Machine definition template file. The path must be relative to the ROOT
+        module of the Terraform deployment. The template should use valid Amazon
+        States Language syntax; wherever a Cirrus Task resource ARN is needed, a
         Terraform interpolation sequence (a "$\{...}" without the "\") may be
-        used instead. The variable name does not matter so long as there is a
-        corresponding entry in the "template_variables" argument.
-        Example template snippet:
-
-          "States": {
-            "FirstState": {
-              "Type": "Task",
-              "Resource": "$\{my-task-lambda}",  // REMOVE THE "\"
-              "Next": "SecondState",
-              ...
-            },
-
-        Cirrus may deploy and manage several builtin tasks. Resource ARNs for
-        these tasks may be referenced in a Workflow template using a predefined
-        variable name without having to supply a 'template_variable' entry.
-          - If Batch Tasks were created, the following variables may be used:
-            - PRE-BATCH: cirrus-geo pre-batch Lambda function ARN
-            - POST-BATCH: cirrus-geo post-batch Lambda function ARN
-
-      - template_variables: (optional, map[object]) A map of template variable
-        names to their corresponding Cirrus Task attributes. Assuming a Cirrus
-        Task named "my-task" with Lambda config was passed to the 'task' module,
-        the following workflow template variable config:
-
-          my-task-lambda = {
-            task_name = "my-task"
-            task_type = "lambda"
-            task_attr = "function_arn"
-          }
+        used instead. The interpolation sequence should have the following form:
+          <TASK NAME>.<TASK TYPE>.<TASK ATTR>
 
-        when used with the example Workflow snippet above would result in the
-        following content after template interpolation:
+        Where:
+          <TASK NAME> : name of the task
+          <TASK TYPE> : one of [lambda, batch]
+          <TASK ATTR> : one of [function_arn, job_definition_arn, job_queue_arn]
 
+        Example template snippet:
           "States": {
             "FirstState": {
               "Type": "Task",
-              "Resource": "arn:aws:lambda:us-west-2:123456789012:function:my-function",
+              "Resource": "$\{my-task.lambda.function_arn}",  // REMOVE THE "\"
               "Next": "SecondState",
               ...
             },
   DESCRIPTION
 
   type = object({
     name                   = string
-    non_cirrus_lambda_arns = optional(list(string))
-    template_filepath      = string
-    template_variables = optional(map(object({
-      task_name = string
-      task_type = string
-      task_attr = string
-    })))
+    state_machine_filepath = string
   })
 
   # Value must be provided else this module serves no purpose
   nullable = false
-
-  validation {
-    condition = (
-      var.workflow_config.template_variables != null
-      ? alltrue([
-        for _, tpl_variable in var.workflow_config.template_variables :
-        (
-          contains(
-            ["lambda", "batch"],
-            tpl_variable.task_type
-          )
-          && contains(
-            ["function_arn", "job_definition_arn", "job_queue_arn"],
-            tpl_variable.task_attr
-          )
-        )
-      ])
-      : true
-    )
-
-    error_message = <<-ERROR
-      Invalid template variable config. Each key must have a valid value:
-        - task_type => one of ["lambda", "batch"]
-        - task_attr => one of ["function_arn", "job_definition_arn", "job_queue_arn"]
-    ERROR
-  }
-}
-
-variable "builtin_task_template_variables" {
-  description = <<-DESCRIPTION
-    (optional, object) Key/value pairs of builtin task variables used during
-    workflow state machine templating. This should be set in the parent module
-    and not by user input.
-  DESCRIPTION
-
-  type = map(object({
-    task_name = string
-    task_type = string
-    task_attr = string
-  }))
-
-  # Value should always be a map (empty map is OK)
-  default  = {}
-  nullable = false
 }