Add some nice pipeline stuff.

VerticalRelevance · May 4, 2022 · f80acc4 · f80acc4
1 parent e6abd6b
commit f80acc4
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 86 deletions.
diff --git a/app.py b/app.py
@@ -1,23 +1,22 @@
 #!/usr/bin/env python3
 import os
-from pathlib import Path
-from typing import List
 
 import aws_cdk as cdk
-from aws_cdk import aws_config, aws_stepfunctions
 
 from stacks.control_broker_stack import (
     ControlBrokerStack,
 )
 from stacks.pipeline_stack import GitHubCDKPipelineStack
 from stacks.test_stack import TestStack
 from stacks.endpoint_stack import EndpointStack
+from utils.environment import is_pipeline_synth
 
 STACK_VERSION = "V0x7x0"
 
 app = cdk.App()
-continuously_deployed = app.node.try_get_context(
-    "control-broker/continuous-deployment/enabled"
+continuously_deployed = (
+    app.node.try_get_context("control-broker/continuous-deployment/enabled")
+    or is_pipeline_synth()
 )
 deploy_stage = None
 if continuously_deployed:
@@ -39,7 +38,7 @@
         f"ControlBrokerTestStack{STACK_VERSION}",
         control_broker_outer_state_machine=control_broker_stack.outer_eval_engine_state_machine,
         control_broker_roles=control_broker_stack.Input_reader_roles,
-        env=env
+        env=env,
     )
 if app.node.try_get_context("control-broker/client/enabled"):
     EndpointStack(
@@ -48,7 +47,7 @@
         control_broker_outer_state_machine=control_broker_stack.outer_eval_engine_state_machine,
         control_broker_roles=control_broker_stack.Input_reader_roles,
         control_broker_eval_results_bucket=control_broker_stack.eval_results_reports_bucket,
-        env=env
+        env=env,
     )
 
 if continuously_deployed:

diff --git a/stacks/control_broker_stack.py b/stacks/control_broker_stack.py
@@ -139,7 +139,7 @@ def deploy_utils(self):
                 restrict_public_buckets=True,
             ),
         )
-        
+
         # results reports
 
         self.bucket_eval_results_reports = aws_s3.Bucket(
@@ -212,21 +212,21 @@ def deploy_inner_sfn_lambdas(self):
             code=aws_lambda.Code.from_asset(
                 "./supplementary_files/lambdas/pac_evaluation_router"
             ),
-            environment = {
-                "PaCBucketRouting": json.dumps({
-                    "OPA":self.bucket_opa_policies.bucket_name
-                }),
-                "ConvertedInputsBucket": self.bucket_converted_inputs.bucket_name
-            }
+            environment={
+                "PaCBucketRouting": json.dumps(
+                    {"OPA": self.bucket_opa_policies.bucket_name}
+                ),
+                "ConvertedInputsBucket": self.bucket_converted_inputs.bucket_name,
+            },
         )
 
         self.lambda_pac_evaluation_router.role.add_to_policy(
             aws_iam.PolicyStatement(
                 actions=[
                     "cloudformation:ValidateTemplate",
                     "cloudformation:DescribeType",
-                    "cloudformation:Get*", #FIXME
-                    "cloudformation:Describe*", #FIXME
+                    "cloudformation:Get*",  # FIXME
+                    "cloudformation:Describe*",  # FIXME
                 ],
                 resources=["*"],
             )
@@ -235,7 +235,7 @@ def deploy_inner_sfn_lambdas(self):
             aws_iam.PolicyStatement(
                 actions=[
                     "cloudcontrol:GetResource",
-                    "cloudcontrol:*", #FIXME
+                    "cloudcontrol:*",  # FIXME
                 ],
                 resources=["*"],
             )
@@ -252,7 +252,7 @@ def deploy_inner_sfn_lambdas(self):
         )
         self.lambda_pac_evaluation_router.role.add_to_policy(
             aws_iam.PolicyStatement(
-                 # Get*, List* for all services with a cloudcontrol provisionable resource
+                # Get*, List* for all services with a cloudcontrol provisionable resource
                 # required fro cloudcontrol.get_resource()
                 actions=[
                     "acmpca:Get*",
@@ -539,7 +539,7 @@ def deploy_inner_sfn_lambdas(self):
                 resources=["*"],
             )
         )
-        
+
         # InputType CloudFormation - PaCFramework OPA - PythonSubprocess
 
         self.lambda_evaluate_cloudformation_by_opa = aws_lambda.Function(
@@ -726,11 +726,9 @@ def deploy_inner_sfn(self):
                             "Resource": "arn:aws:states:::lambda:invoke",
                             "Parameters": {
                                 "FunctionName": self.lambda_pac_evaluation_router.function_name,
-                                "Payload.$": "$"
-                            },
-                            "ResultSelector": {
-                                "Routing.$": "$.Payload.Routing"
+                                "Payload.$": "$",
                             },
+                            "ResultSelector": {"Routing.$": "$.Payload.Routing"},
                         },
                         "ChoicePaCEvaluationRouting": {
                             "Type": "Choice",
@@ -755,8 +753,8 @@ def deploy_inner_sfn(self):
                                 "FunctionName": self.lambda_evaluate_cloudformation_by_opa.function_name,
                                 "Payload": {
                                     "JsonInput": {
-                                        "Bucket.$":"$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket",
-                                        "Key.$":"$.PaCEvaluationRouter.Routing.ModifiedInput.Key",
+                                        "Bucket.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket",
+                                        "Key.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Key",
                                     },
                                     "PaC": {
                                         "Bucket.$": "$.PaCEvaluationRouter.Routing.PaC.Bucket"
@@ -818,12 +816,12 @@ def deploy_inner_sfn(self):
                                             "Payload": {
                                                 "Infraction.$": "$.Infraction",
                                                 "JsonInput": {
-                                                    "Bucket.$":"$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket",
-                                                    "Key.$":"$.PaCEvaluationRouter.Routing.ModifiedInput.Key",
+                                                    "Bucket.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket",
+                                                    "Key.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Key",
                                                 },
                                                 "OuterEvalEngineSfnExecutionId.$": "$.OuterEvalEngineSfnExecutionId",
                                                 "ConsumerMetadata.$": "$.InvokedByApigw.ControlBrokerConsumerInputs.ConsumerMetadata",
-                                            }
+                                            },
                                         },
                                         "ResultSelector": {"Payload.$": "$.Payload"},
                                     },
@@ -986,7 +984,7 @@ def deploy_outer_sfn(self):
                             "ResultPath": "$.ForEachInput",
                             "ItemsPath": "$.InvokedByApigw.ControlBrokerConsumerInputs.InputKeys",
                             "Parameters": {
-                                "InvokedByApigw.$":"$.InvokedByApigw",
+                                "InvokedByApigw.$": "$.InvokedByApigw",
                                 "ControlBrokerConsumerInputKey.$": "$$.Map.Item.Value",
                             },
                             "Iterator": {
@@ -1000,9 +998,9 @@ def deploy_outer_sfn(self):
                                         "Parameters": {
                                             "StateMachineArn": self.sfn_inner_eval_engine.attr_arn,
                                             "Input": {
-                                                "ControlBrokerConsumerInputKey.$":"$.ControlBrokerConsumerInputKey",
+                                                "ControlBrokerConsumerInputKey.$": "$.ControlBrokerConsumerInputKey",
                                                 "ControlBrokerConsumerInputs.$": "$.InvokedByApigw.ControlBrokerConsumerInputs",
-                                                "OuterEvalEngineSfnExecutionId.$":"$$.Execution.Id",
+                                                "OuterEvalEngineSfnExecutionId.$": "$$.Execution.Id",
                                             },
                                         },
                                     },
@@ -1031,9 +1029,9 @@ def deploy_outer_sfn(self):
                                 "FunctionName": self.lambda_write_results_report.function_name,
                                 "Payload": {
                                     "OuterEvalEngineSfnExecutionId.$": "$$.Execution.Id",
-                                    "ResultsReportS3Uri.$":"$.ResultsReportS3Uri",
-                                    "ForEachInput.$":"$.ForEachInput",
-                                }
+                                    "ResultsReportS3Uri.$": "$.ResultsReportS3Uri",
+                                    "ForEachInput.$": "$.ForEachInput",
+                                },
                             },
                             "ResultSelector": {"Payload.$": "$.Payload"},
                         },

diff --git a/stacks/endpoint_stack.py b/stacks/endpoint_stack.py
@@ -6,23 +6,14 @@
 from aws_cdk import (
     Duration,
     Stack,
-    RemovalPolicy,
     CfnOutput,
-    aws_config,
-    aws_dynamodb,
     aws_s3,
-    aws_sqs,
-    aws_s3_deployment,
     aws_lambda,
     aws_stepfunctions,
     aws_iam,
-    aws_logs,
-    aws_events,
-    aws_apigatewayv2,
-    aws_apigatewayv2_alpha, # experimental as of 4.25.22
-    aws_apigatewayv2_integrations_alpha, # experimental as of 4.25.22
-    aws_apigatewayv2_authorizers_alpha, # experimental as of 4.25.22
-    aws_lambda_python_alpha, # experimental as of 4.25.22
+    aws_apigatewayv2_alpha,  # experimental as of 4.25.22
+    aws_apigatewayv2_integrations_alpha,  # experimental as of 4.25.22
+    aws_apigatewayv2_authorizers_alpha,  # experimental as of 4.25.22
 )
 from constructs import Construct
 
@@ -46,16 +37,16 @@ def __init__(
         :type control_broker_eval_results_bucket: aws_s3.Bcuket
         """
         super().__init__(*args, **kwargs)
-    
+
         self.control_broker_outer_state_machine = control_broker_outer_state_machine
         self.control_broker_eval_results_bucket = control_broker_eval_results_bucket
-    
+
         self.endpoint()
-        
+
     def endpoint(self):
-        
+
         # auth - lambda
-        
+
         lambda_authorizer = aws_lambda.Function(
             self,
             "ControlBrokerClientAuthorizer",
@@ -67,23 +58,25 @@ def endpoint(self):
                 "./supplementary_files/lambdas/apigw_authorizer"
             ),
         )
-    
+
         authorizer_lambda = aws_apigatewayv2_authorizers_alpha.HttpLambdaAuthorizer(
             "ControlBrokerClientAuthorizer",
             lambda_authorizer,
-            response_types=[aws_apigatewayv2_authorizers_alpha.HttpLambdaResponseType.SIMPLE],
-            results_cache_ttl = Duration.seconds(0),
-            identity_source = [
-                "$request.header.Authorization", # Authorization must be present in headers or 401, e.g. r = requests.post(url,auth = auth, ...)
-            ]
+            response_types=[
+                aws_apigatewayv2_authorizers_alpha.HttpLambdaResponseType.SIMPLE
+            ],
+            results_cache_ttl=Duration.seconds(0),
+            identity_source=[
+                "$request.header.Authorization",  # Authorization must be present in headers or 401, e.g. r = requests.post(url,auth = auth, ...)
+            ],
         )
-        
+
         # auth - iam
-        
+
         authorizer_iam = aws_apigatewayv2_authorizers_alpha.HttpIamAuthorizer()
 
         # integration
-        
+
         lambda_invoked_by_apigw = aws_lambda.Function(
             self,
             "InvokedByApigw",
@@ -94,48 +87,45 @@ def endpoint(self):
             code=aws_lambda.Code.from_asset(
                 "./supplementary_files/lambdas/invoked_by_apigw"
             ),
-            environment = {
-                "ControlBrokerOuterSfnArn" : self.control_broker_outer_state_machine.state_machine_arn,
-                "ControlBrokerEvalResultsReportsBucket": self.control_broker_eval_results_bucket.bucket_name
-            }
+            environment={
+                "ControlBrokerOuterSfnArn": self.control_broker_outer_state_machine.state_machine_arn,
+                "ControlBrokerEvalResultsReportsBucket": self.control_broker_eval_results_bucket.bucket_name,
+            },
         )
-        
+
         lambda_invoked_by_apigw.role.add_to_policy(
             aws_iam.PolicyStatement(
                 actions=[
                     "states:StartExecution",
                 ],
-                resources=[
-                    self.control_broker_outer_state_machine.state_machine_arn
-                ],
+                resources=[self.control_broker_outer_state_machine.state_machine_arn],
             )
         )
 
         integration = aws_apigatewayv2_integrations_alpha.HttpLambdaIntegration(
-            "ControlBrokerClient",
-            lambda_invoked_by_apigw
+            "ControlBrokerClient", lambda_invoked_by_apigw
         )
-    
+
         # api
-    
+
         self.http_api = aws_apigatewayv2_alpha.HttpApi(
             self,
             "ControlBrokerClient",
             # default_authorizer = authorizer
         )
-        
+
         self.path = "/"
-        
+
         routes = self.http_api.add_routes(
             path=self.path,
-            methods=[
-                aws_apigatewayv2_alpha.HttpMethod.POST
-            ],
+            methods=[aws_apigatewayv2_alpha.HttpMethod.POST],
             integration=integration,
             authorizer=authorizer_lambda
             # authorizer=authorizer_iam
         )
-
-        self.apigw_full_invoke_url = path.join(self.http_api.url.rstrip("/"),self.path.strip('/'))
-
-        CfnOutput(self, "ApigwInvokeUrl", value=self.apigw_full_invoke_url)
+
+        self.apigw_full_invoke_url = path.join(
+            self.http_api.url.rstrip("/"), self.path.strip("/")
+        )
+
+        CfnOutput(self, "ApigwInvokeUrl", value=self.apigw_full_invoke_url)
diff --git a/stacks/pipeline_stack.py b/stacks/pipeline_stack.py
@@ -112,12 +112,13 @@ def __init__(
                 "pip install -r requirements.txt",  # Instructs Codebuild to install required packages
                 "npx cdk synth",
             ],
+            env={"PIPELINE_SYNTH": "true"}
         )
 
         self.pipeline = pipelines.CodePipeline(
             self,
             "Pipeline",
             synth=pipeline_synth_action,
             publish_assets_in_parallel=False,
-            docker_enabled_for_synth=True
+            docker_enabled_for_synth=True,
         )
diff --git a/stacks/test_stack.py b/stacks/test_stack.py
@@ -38,7 +38,10 @@ def __init__(
             removal_policy=RemovalPolicy.DESTROY,
         )
         for principal in control_broker_roles:
-            canary_bucket.grant_read(aws_iam.ArnPrincipal(principal.role_arn), f"{CANARY_TEST_TEMPLATE_DEST}/*")
+            canary_bucket.grant_read(
+                aws_iam.ArnPrincipal(principal.role_arn),
+                f"{CANARY_TEST_TEMPLATE_DEST}/*",
+            )
         control_broker_consumer_policy = aws_iam.ManagedPolicy(
             self,
             "ControlBrokerConsumerPolicy",
@@ -134,8 +137,6 @@ def __init__(
             ),
         )
 
-
-
         """
         TODO.1
         
@@ -163,4 +164,4 @@ def __init__(
         assert that for each non-read-only property, the API response is the same
         
         
-        """
+        """
diff --git a/utils/environment.py b/utils/environment.py
@@ -0,0 +1,4 @@
+import os
+
+def is_pipeline_synth():
+    return "PIPELINE_SYNTH" in os.environ