Feature/tdp 188 v2 inf blocks (#196)

* feat(common-utils): This implements a library to be leveraged for shared code. The module added for this task is for deployment of docker images and Prefect filesystems README.md has lots of details * ci(.circleci): adding the build step for common-utils * ci(.circleci): poetry does not need to be installed * style(.aws): fixing style check findings * ci(.circleci): moving common-utils build to separate workflow * ci(.circleci): setting up proper stdout and fixing black command * ci(.circleci): trying to get output for commands to troubleshoot errors * fix: fixing build by not create venv and fixing bug with cli:main where no args (--help for example) was throwing error * ci(.circleci): removing poetry run commands * ci: adding verbose logging to poetry * ci: moving build to using Prefect image since this is the environment this package will run on * Add --no-ansie flag as per python-poetry/poetry#7184 --------- Co-authored-by: Brett Kochendorfer <[email protected]>
Pocket · Feb 27, 2023 · f79e790 · f79e790
1 parent d212622
commit f79e790
Show file tree

Hide file tree

Showing 19 changed files with 3,875 additions and 1,651 deletions.
diff --git a/.aws/src/iam.ts b/.aws/src/iam.ts
@@ -1,9 +1,14 @@
 // Module to abstract IAM elements needed for Prefect v2
 import { DataAwsCallerIdentity } from '@cdktf/provider-aws/lib/data-aws-caller-identity';
-import { DataAwsIamPolicyDocumentStatement } from '@cdktf/provider-aws/lib/data-aws-iam-policy-document';
+import {
+  DataAwsIamPolicyDocument,
+  DataAwsIamPolicyDocumentStatement
+} from '@cdktf/provider-aws/lib/data-aws-iam-policy-document';
 import { DataAwsSecretsmanagerSecret } from '@cdktf/provider-aws/lib/data-aws-secretsmanager-secret';
 import { Construct } from 'constructs';
 import { DataAwsRegion } from '@cdktf/provider-aws/lib/data-aws-region';
+import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket';
+import { IamRole } from '@cdktf/provider-aws/lib/iam-role';
 
 // Custom construct to create IAM roles needed for a Prefect v2 Agent
 export class AgentIamPolicies extends Construct {
@@ -86,3 +91,112 @@ export class AgentIamPolicies extends Construct {
     };
   }
 }
+
+export class DataFlowsIamRoles extends Construct {
+  private readonly fileSystem: S3Bucket;
+  private readonly caller: DataAwsCallerIdentity;
+  private readonly region: DataAwsRegion;
+  constructor(
+    scope: Construct,
+    name: string,
+    fileSystem: S3Bucket,
+    caller: DataAwsCallerIdentity,
+    region: DataAwsRegion,
+    environment: string,
+    deploymentType: string
+  ) {
+    super(scope, name);
+    this.caller = caller;
+    this.region = region;
+    this.fileSystem = fileSystem;
+    // create an inline policy doc for the execution role that can be combined with AWS managed policy
+    const flowExecutionPolicyStatements = [
+      {
+        actions: [
+          'kms:Decrypt',
+          'secretsmanager:GetSecretValue',
+          'ssm:GetParameters'
+        ],
+        effect: 'Allow',
+        resources: [
+          `arn:aws:secretsmanager:${this.region.name}:${this.caller.accountId}:secret:dpt/${environment}/data_flows_prefect_*`
+        ]
+      }
+    ];
+
+    // build the policy document for the Task role using Policy statement functions
+    const flowTaskPolicyStatements = [
+      this.getFlowS3BucketAccess(),
+      this.getFlowS3ObjectAccess()
+    ];
+
+    this.createFlowIamRole(
+      `data-flows-prefect-${deploymentType}-exec-role`,
+      flowExecutionPolicyStatements
+    );
+    this.createFlowIamRole(
+      `data-flows-prefect-${deploymentType}-task-role`,
+      flowTaskPolicyStatements
+    );
+  }
+  // build policy statment for S3 bucket access
+  private getFlowS3BucketAccess(): DataAwsIamPolicyDocumentStatement {
+    return {
+      actions: ['s3:ListBucket'],
+      effect: 'Allow',
+      resources: [this.fileSystem.arn]
+    };
+  }
+  // build policy statment for S3 object access
+  private getFlowS3ObjectAccess(): DataAwsIamPolicyDocumentStatement {
+    return {
+      actions: ['s3:*Object'],
+      effect: 'Allow',
+      resources: [`${this.fileSystem.arn}/data/*`]
+    };
+  }
+  // build policy statment for S3 object access
+  private getFlowAssumeRoleAccess(role_name: string): DataAwsIamPolicyDocument {
+    return new DataAwsIamPolicyDocument(this, `${role_name}TrustPolicy`, {
+      version: '2012-10-17',
+      statement: [
+        {
+          effect: 'Allow',
+          actions: ['sts:AssumeRole'],
+          principals: [
+            {
+              identifiers: ['ecs-tasks.amazonaws.com'],
+              type: 'Service'
+            }
+          ]
+        }
+      ]
+    });
+  }
+  private getFlowRolePolicy(
+    role_name: string,
+    policyStatements: DataAwsIamPolicyDocumentStatement[]
+  ): DataAwsIamPolicyDocument {
+    return new DataAwsIamPolicyDocument(this, `${role_name}AccessPolicy`, {
+      version: '2012-10-17',
+      statement: policyStatements
+    });
+  }
+
+  private createFlowIamRole(
+    name: string,
+    policy: DataAwsIamPolicyDocumentStatement[]
+  ): IamRole {
+    const inline_policy = this.getFlowRolePolicy(name, policy);
+    return new IamRole(this, name, {
+      name: name,
+      assumeRolePolicy: this.getFlowAssumeRoleAccess(name).json,
+      inlinePolicy: [
+        {
+          name: `${name}-access-policy`,
+          policy: inline_policy.json
+        }
+      ]
+    });
+  }
+}
diff --git a/.aws/src/main.ts b/.aws/src/main.ts
@@ -7,9 +7,15 @@ import { config } from './config';
 import { App, TerraformStack, CloudBackend, NamedCloudWorkspace } from 'cdktf';
 import { DataAwsRegion } from '@cdktf/provider-aws/lib/data-aws-region';
 import { DataAwsCallerIdentity } from '@cdktf/provider-aws/lib/data-aws-caller-identity';
-import { AgentIamPolicies } from './iam';
+import { AgentIamPolicies, DataFlowsIamRoles } from './iam';
 import { DataAwsSecretsmanagerSecret } from '@cdktf/provider-aws/lib/data-aws-secretsmanager-secret';
-import { PocketECSApplication } from '@pocket-tools/terraform-modules';
+import {
+  ApplicationECR,
+  PocketECSApplication
+} from '@pocket-tools/terraform-modules';
+import { S3Bucket } from '@cdktf/provider-aws/lib/s3-bucket';
+import { S3BucketVersioningA } from '@cdktf/provider-aws/lib/s3-bucket-versioning';
+import { S3BucketPublicAccessBlock } from '@cdktf/provider-aws/lib/s3-bucket-public-access-block';
 
 // main Terraform Stack object for Prefect V2 infrastructure
 class PrefectV2 extends TerraformStack {
@@ -50,6 +56,69 @@ class PrefectV2 extends TerraformStack {
     // we need an agent per queue...so we create 2 agent services
     this.getAgentService('test');
     this.getAgentService('live');
+
+    // create new data-flows-prefect-envs ECR Repo
+    new ApplicationECR(this, 'data-flows-prefect-envs-ecr', {
+      name: 'data-flows-prefect-envs'
+    });
+
+    // create live and test filesystems
+    const testFS = this.createDataFlowsBucket('test');
+    const liveFS = this.createDataFlowsBucket('live');
+
+    // create live and test iam roles
+    new DataFlowsIamRoles(
+      this,
+      'dataFlowTestRoles',
+      testFS,
+      this.caller,
+      this.region,
+      config.tags.environment,
+      'test'
+    );
+    new DataFlowsIamRoles(
+      this,
+      'dataFlowLiveRoles',
+      liveFS,
+      this.caller,
+      this.region,
+      config.tags.environment,
+      'live'
+    );
+  }
+
+  // create new data-flows-prefect-filesystem S3 buckets
+  // this is used for flow artifacts and staging as needed
+  private createDataFlowsBucket(deploymentType: string): S3Bucket {
+    const artifactsBucket = new S3Bucket(
+      this,
+      `dataFlowsPrefectFs${deploymentType}`,
+      {
+        bucket: `data-flows-prefect-fs-${config.tags.environment}-${deploymentType}`
+      }
+    );
+    new S3BucketVersioningA(
+      this,
+      `dataFlowsPrefectFsVConfig${deploymentType}`,
+      {
+        bucket: artifactsBucket.id,
+        versioningConfiguration: {
+          status: 'Enabled'
+        }
+      }
+    );
+    new S3BucketPublicAccessBlock(
+      this,
+      `dataFlowsPrefectFsPublicAccess${deploymentType}`,
+      {
+        bucket: artifactsBucket.id,
+        blockPublicAcls: true,
+        blockPublicPolicy: true,
+        ignorePublicAcls: true,
+        restrictPublicBuckets: true
+      }
+    );
+    return artifactsBucket;
   }
 
   // create a task definition and service using private methods and params

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,5 +10,6 @@ workflows:
           config-path: .circleci/workflows.yml
           mapping: |
             [.]aws/.* aws_changes true
+            common-utils/.* common_utils_changes true
 
 # VS Code Extension Version: 1.5.1
diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml
@@ -5,6 +5,9 @@ parameters:
   aws_changes:
     type: boolean
     default: false
+  common_utils_changes:
+    type: boolean
+    default: false
 
 # Workflow and job shortcuts
 only_main_v2: &only_main_v2
@@ -29,6 +32,9 @@ only_dev_v2: &only_dev_v2
 has_aws_changes: &has_aws_changes
   when: << pipeline.parameters.aws_changes >>
 
+has_common_utils_changes: &has_common_utils_changes
+  when: << pipeline.parameters.common_utils_changes >>
+
 # Reusable commands for jobs
 commands:
   no_op:
@@ -113,7 +119,30 @@ jobs:
                   name: terraform deploy
         - no_op
       working_directory: ~/repo/.aws
-
+  common_utils_changes_build:
+    docker:
+      - image: prefecthq/prefect:2-python3.10
+    steps:
+      - when: 
+          condition: has_common_utils_changes
+          steps:
+          - checkout:
+              path: ~/repo
+          - run:
+              command: |
+                export PATH=/root/.local/bin:$PATH
+                apt update
+                apt install -y curl
+                curl -sSL https://install.python-poetry.org | python3
+                apt clean && apt autoremove -y
+                rm -rf /var/lib/apt/lists/*
+                poetry config virtualenvs.create false
+                poetry install --no-ansi --only main,dev
+                black --check .
+                python -m pytest --cov=src/common --cov-report term-missing --cov-fail-under=100
+              name: Install dependencies and run tests
+      - no_op
+    working_directory: ~/repo/common-utils
 workflows:
   aws_changes_workflow:
     jobs: 
@@ -141,6 +170,11 @@ workflows:
           node_env: production
           runner_resource_class: pocket/default-prod
           name: terraform apply production
+  common_utils_changes_workflow:
+    jobs: 
+      - common_utils_changes_build:
+          <<: *not_dev_main_v2
+          name: common-utils build
 
 
 # VS Code Extension Version: 1.5.1
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,9 @@
 .idea
 *__pycache__
 
+# Python
 .env
-*.env
 .venv
+.pytest_cache
+dist
+.coverage
diff --git a/Pipfile b/Pipfile