* use terragrunt, refactor, add readme, bootstrap and environments

* bump version of pre-commit-terraform to fix antonbabenko/pre-commit-terraform#171

.pre-commit-config.yaml

@@ -43,6 +43,6 @@ repos:
     -   id: check-merge-conflict
     -   id: check-case-conflict
 -   repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.45.0
+    rev: v1.46.0
     hooks:
-    -   id: terraform_validate
+    -   id: terraform_validate

infra/README.rst

@@ -0,0 +1,44 @@
+Preqequisites:
+  * terragrunt
+  * an aws admin account that will be used to assume role of the different deploy environments
+    * have an aws profile setup for this account, e.g. `curi_admin`
+  * aws account(s) profile for each environment with iam user and api access (access_key/secret_key)
+    Currently the environments use these profile names, e.g.
+
+      .. code:: bash
+      $ cat ~/.aws.credentials
+      [curi_modl]
+      aws_access_key_id=<curi_modl access key>
+      aws_secret_access_key=<curi_modl secret access key>
+
+      [curi_prod]
+      aws_access_key_id=<curi_prod access key>
+      aws_secret_access_key=<curi_prod secret access key>
+
+      [curi_test]
+      aws_access_key_id=<curi_prod access key>
+      aws_secret_access_key=<curi_prod secret access key>
+      ```
+
+To bootstrap infrastructure for each environment:
+  * set `TF_VAR_account_id` to the value of the admin account, e.g. `export TF_VAR_account_id=123456789012`
+  * `cd ./bootstrap/<env>`
+  * check ./bootstrap/<env>/terragrunt.hcl is using the correct profile, change if you want to use different names
+  * run `terragrunt plan --terragrunt-non-interactive`
+  * verify the plan makes sense
+  * run `terragrunt apply --terragrunt-non-interactive`
+  * Note the arn output value, for each env add the arn string to
+    `./environment/<env>/terragrunt.hcl` for the `role_arn` key in the `assume_role` block for the aws provider
+
+
+To provision an environment, after it has been bootstrapped:
+  * `cd ./bootstrap/<env>`
+  * optionally you can export the `AWS_PROFILE` environment variable to the aws profile you plan to deploy with,
+    `export AWS_PROFILE=curi_admin` or provide that value when running the terragrunt command.
+  * run `AWS_PROFILE=curi_admin terragrunt run-all plan --terragrunt-non-interactive`
+  * verify plan looks good
+  * run `AWS_PROFILE=curi_admin terragrunt run-all apply --terragrunt-non-interactive`
+
+
+If all the command ran successfully the infrastructure is fully deployed, make a pr to check-in the changes to the `terragrunt.hcl`
+files with the new arn values.

infra/bootstrap/modl/main.tf

@@ -0,0 +1,14 @@
+provider "aws" {
+  region  = "us-east-1"
+  profile = "curi_modl"
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "iam_roles" {
+  source     = "../modules/aws/iam_role_policy"
+  account_id = var.account_id
+}

infra/bootstrap/modl/outputs.tf

@@ -0,0 +1,4 @@
+output "modl_iam_role_arn" {
+  value       = module.iam_roles.iam_role_arn
+  description = "aws_iam_role arn"
+}

infra/bootstrap/modl/terragrunt.hcl

@@ -0,0 +1,12 @@
+remote_state {
+  backend = "s3"
+  config = {
+    profile = "curi_modl"
+    bucket = "curi-modl-terraform-state"
+
+    key            = "${path_relative_to_include()}/terraform.tfstate"
+    region         = "us-east-1"
+    encrypt        = true
+    dynamodb_table = "terraform-lock-table"
+  }
+}

infra/bootstrap/modl/variables.tf

@@ -0,0 +1,4 @@
+variable "account_id" {
+  type        = string
+  description = "Root account id for setting iam_role AssumeRole in deploy environments"
+}

infra/bootstrap/modules/aws/iam_role_policy/main.tf

@@ -0,0 +1,37 @@
+resource "aws_iam_role" "terraform_role" {
+  name = "terraform_deploy_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          AWS = "arn:aws:iam::${var.account_id}:root"
+        }
+        Condition = {}
+      },
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "policy" {
+  name = "terraform_deploy_policy"
+  role = aws_iam_role.terraform_role.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "s3:*",
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      },
+    ]
+  })
+}
+

infra/bootstrap/modules/aws/iam_role_policy/outputs.tf

@@ -0,0 +1,4 @@
+output "iam_role_arn" {
+  value       = aws_iam_role.terraform_role.arn
+  description = "aws_iam_role arn"
+}

infra/bootstrap/modules/aws/iam_role_policy/variables.tf

@@ -0,0 +1,4 @@
+variable "account_id" {
+  type        = string
+  description = "Root account id for setting iam_role AssumeRole in deploy environments"
+}

infra/bootstrap/prod/main.tf

@@ -0,0 +1,14 @@
+provider "aws" {
+  region  = "us-east-1"
+  profile = "curi_prod"
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "iam_roles" {
+  source     = "../modules/aws/iam_role_policy"
+  account_id = var.account_id
+}

infra/bootstrap/prod/outputs.tf

@@ -0,0 +1,4 @@
+output "prod_iam_role_arn" {
+  value       = module.iam_roles.iam_role_arn
+  description = "aws_iam_role arn"
+}

infra/bootstrap/prod/terragrunt.hcl

@@ -0,0 +1,12 @@
+remote_state {
+  backend = "s3"
+  config = {
+    profile = "curi_prod"
+    bucket = "curi-prod-terraform-state"
+
+    key            = "${path_relative_to_include()}/terraform.tfstate"
+    region         = "us-east-1"
+    encrypt        = true
+    dynamodb_table = "terraform-lock-table"
+  }
+}

infra/bootstrap/prod/variables.tf

@@ -0,0 +1,4 @@
+variable "account_id" {
+  type        = string
+  description = "Root account id for setting iam_role AssumeRole in deploy environments"
+}

infra/bootstrap/test/main.tf

@@ -0,0 +1,14 @@
+provider "aws" {
+  region  = "us-east-1"
+  profile = "curi_test"
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "iam_roles" {
+  source     = "../modules/aws/iam_role_policy"
+  account_id = var.account_id
+}

infra/bootstrap/test/outputs.tf

@@ -0,0 +1,4 @@
+output "test_iam_role_arn" {
+  value       = module.iam_roles.iam_role_arn
+  description = "aws_iam_role arn"
+}

infra/bootstrap/test/terragrunt.hcl

@@ -0,0 +1,14 @@
+remote_state {
+  backend = "s3"
+  config = {
+    profile = "curi_test"
+    bucket  = "curi-test-terraform-state"
+
+    key                      = "${path_relative_to_include()}/terraform.tfstate"
+    region                   = "us-east-1"
+    encrypt                  = true
+    skip_bucket_versioning   = false
+    skip_bucket_ssencryption = false
+    dynamodb_table           = "terraform-lock-table"
+  }
+}

infra/bootstrap/test/variables.tf

@@ -0,0 +1,4 @@
+variable "account_id" {
+  type        = string
+  description = "Root account id for setting iam_role AssumeRole in deploy environments"
+}

infra/environments/modl/main.tf

@@ -0,0 +1,17 @@
+provider "aws" {
+  region       = "us-east-1"
+  assume_role {
+    role_arn     = "arn:aws:iam::725604423866:role/terraform_deploy_role"
+    session_name = "terraform"
+  }
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "data_ingest" {
+  source = "../../modules/curi/data_ingest"
+  data_processor_bucket = "curi-modl-data-test-bucket"
+}

infra/environments/modl/terragrunt.hcl

@@ -0,0 +1,11 @@
+remote_state {
+  backend = "s3"
+  config = {
+    bucket = "curi-terraform-state"
+
+    key            = "modl/${path_relative_to_include()}/terraform.tfstate"
+    region         = "us-east-1"
+    encrypt        = true
+    dynamodb_table = "terraform-lock-table"
+  }
+}

infra/environments/prod/main.tf

@@ -0,0 +1,17 @@
+provider "aws" {
+  region       = "us-east-1"
+  assume_role {
+    role_arn     = "arn:aws:iam::245339368379:role/terraform_deploy_role"
+    session_name = "terraform"
+  }
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "data_ingest" {
+  source = "../../modules/curi/data_ingest"
+  data_processor_bucket = "curi-prod-data-test-bucket"
+}

infra/environments/prod/terragrunt.hcl

@@ -0,0 +1,11 @@
+remote_state {
+  backend = "s3"
+  config = {
+    bucket = "curi-terraform-state"
+
+    key            = "prod/${path_relative_to_include()}/terraform.tfstate"
+    region         = "us-east-1"
+    encrypt        = true
+    dynamodb_table = "terraform-lock-table"
+  }
+}

infra/environments/test/main.tf

@@ -0,0 +1,17 @@
+provider "aws" {
+  region  = "us-east-1"
+  assume_role {
+    role_arn     = "arn:aws:iam::077346344852:role/terraform_deploy_role"
+    session_name = "terraform"
+  }
+}
+
+terraform {
+  required_version = ">= 0.14.7"
+  backend "s3" {}
+}
+
+module "data_processor" {
+  source = "../../modules/curi/data_processor"
+  data_processor_bucket = "curi-test-data-test-bucket"
+}

infra/environments/test/terragrunt.hcl

@@ -0,0 +1,11 @@
+remote_state {
+  backend = "s3"
+  config = {
+    bucket = "curi-terraform-state"
+
+    key            = "test/${path_relative_to_include()}/terraform.tfstate"
+    region         = "us-east-1"
+    encrypt        = true
+    dynamodb_table = "terraform-lock-table"
+  }
+}

infra/modules/curi/data_processor/main.tf

@@ -0,0 +1,13 @@
+resource "aws_s3_bucket" "b" {
+  bucket = var.data_processor_bucket
+  acl    = "private"
+
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm = "AES256"
+      }
+    }
+  }
+}
+

infra/modules/curi/data_processor/variables.tf

@@ -0,0 +1,4 @@
+variable "data_processor_bucket" {
+  description = "S3 data ingestion bucket"
+  type = string
+}