Replace Cloudfront with nginx reverse proxy to serve web reports

.github/workflows/run-integration-tests.yml

@@ -40,4 +40,4 @@ jobs:
           cd ./tests/integration_tests/test_data_tests
           docker build -t integration-tests:latest .
       - name: Run tests
-        run: docker run --env QA_BUCKET=integration-test-bucket --env S3_HOST=172.17.0.1 --env S3_PORT=4566 integration-tests
+        run: docker run --env BUCKET=integration-test-bucket --env S3_HOST=172.17.0.1 --env S3_PORT=4566 integration-tests

.github/workflows/terraform-deploy-data-qa-dev.yml

@@ -1,4 +1,4 @@
-name: Deploy to DataQA Dev
+name: Deploy to DataQA
 on:
   workflow_dispatch:
     inputs:
@@ -23,7 +23,7 @@ on:
 
 jobs:
   terraform:
-    name: setup-envs
+    name: "Deploy - ${{ github.event.inputs.TerraformTarget }}/${{ github.event.inputs.applyTerraform }}"
     runs-on: ubuntu-latest
     env:
       TF_VAR_slack_channel: ${{ secrets.TF_VAR_SLACK_CHANNEL }}

.gitignore

@@ -165,4 +165,5 @@ fabric.properties
 node_modules
 tmp
 
-examples/*/plan
+examples/*/plan
+*.auto.tfvars

Makefile

@@ -27,7 +27,7 @@ qa_bucket = integration-test-bucket
 
 run-integration-tests: build-data-test-img build-data-test-tests-img
 	cd $(integration_tests_dir)
-	docker run --env QA_BUCKET=$(qa_bucket) --env S3_HOST=$(host) --env S3_PORT=$(port) test_data_tests
+	docker run --env BUCKET=$(qa_bucket) --env S3_HOST=$(host) --env S3_PORT=$(port) test_data_tests
 
 prepare-unit-tests:
 	cd ./functions/data_test && \
@@ -39,7 +39,7 @@ run-unit-tests:
 	export ENVIRONMENT='local' && \
 	export S3_HOST='localhost' && \
 	export S3_PORT='4566' && \
-	export QA_BUCKET='test-bucket' && \
+	export BUCKET='test-bucket' && \
 	export AWS_DEFAULT_REGION='us-east-1' && \
 	export REDSHIFT_DB='titanic' && \
 	export REDSHIFT_SECRET='titanic' && \

examples/basic/builds/ea92b837122791f9d253e5b78b96b4331a0cd8e81879ffabe32a57680c2c3685.plan.json

@@ -0,0 +1 @@
+{"filename": "builds/ea92b837122791f9d253e5b78b96b4331a0cd8e81879ffabe32a57680c2c3685.zip", "runtime": "python3.8", "artifacts_dir": "builds", "build_plan": [["zip", ".terraform/modules/data_qa_intg.data_reports_alerting.slack_notification/functions/notify_slack.py", null]]}

examples/basic/main.tf

@@ -19,10 +19,23 @@ provider "docker" {
 data "aws_caller_identity" "current" {}
 data "aws_region" "current" {}
 data "aws_ecr_authorization_token" "token" {}
+data "aws_availability_zones" "available" {
+  state = "available"
+}
+
+module "vpc" {
+  source = "./modules/vpc"
+
+  resource_name_prefix = "provectus-infra"
+
+  cidr                 = "172.21.0.0/16"
+  private_subnets_cidr = ["172.21.16.0/20"]
+  public_subnets_cidr  = ["172.21.32.0/20"]
+  azs                  = data.aws_availability_zones.available.zone_ids
+}
 
 module "data_qa" {
-  source                     = "../../terraform"
-  cloudfront_allowed_subnets = ["255.255.255.255/32"]
+  source = "../../terraform"
 
   data_test_storage_bucket_name = "dqg-settings-dev"
   environment                   = "demo"
@@ -32,20 +45,21 @@ module "data_qa" {
   data_test_image_uri     = module.docker_image_data_test.image_uri
   push_report_image_uri   = module.docker_image_push_report.image_uri
 
-  web_acl_id = "arn:aws:wafv2:us-east-1:024975173233:global/webacl/demo-provectus-web-acl/c4517afa-629f-41ab-a4b9-a9645eb9b8dc"
-
   data_reports_notification_settings = {
     channel     = var.slack_channel
     webhook_url = var.slack_webhook_url
   }
 
-  vpc_subnet_ids         = ["subnet-034f0eb6c64a19cb3"]
-  vpc_security_group_ids = ["sg-01e57996f17b73938"]
+  lambda_private_subnet_ids = module.vpc.private_subnet_ids
+  lambda_security_group_ids = module.vpc.security_group_ids
+
+  reports_vpc_id        = module.vpc.vpc_id
+  reports_subnet_id     = module.vpc.public_subnet_ids[0]
+  reports_whitelist_ips = ["195.155.100.203/32"]
 }
 
 module "data_qa_intg" {
-  source                     = "../../terraform"
-  cloudfront_allowed_subnets = ["255.255.255.255/32"]
+  source = "../../terraform"
 
   data_test_storage_bucket_name = "dqg-settings-intg"
   environment                   = "intg"
@@ -55,15 +69,15 @@ module "data_qa_intg" {
   data_test_image_uri     = module.docker_image_data_test.image_uri
   push_report_image_uri   = module.docker_image_push_report.image_uri
 
-  web_acl_id = "arn:aws:wafv2:us-east-1:024975173233:global/webacl/demo-provectus-web-acl/c4517afa-629f-41ab-a4b9-a9645eb9b8dc"
-
   data_reports_notification_settings = {
     channel     = var.slack_channel
     webhook_url = var.slack_webhook_url
   }
 
-  vpc_to_create = {
-    cidr                 = "172.28.0.0/16"
-    private_subnets_cidr = ["172.28.16.0/20"]
-  }
+  lambda_private_subnet_ids = module.vpc.private_subnet_ids
+  lambda_security_group_ids = module.vpc.security_group_ids
+
+  reports_vpc_id        = module.vpc.vpc_id
+  reports_subnet_id     = module.vpc.public_subnet_ids[0]
+  reports_whitelist_ips = ["195.155.100.203/32"]
 }

examples/basic/modules/vpc/main.tf

@@ -0,0 +1,116 @@
+locals {
+  resource_name_prefix    = var.resource_name_prefix
+  private_route_table_ids = module.vpc.private_route_table_ids
+}
+
+data "aws_region" "current" {}
+data "aws_vpc_endpoint_service" "dynamodb" { service = "dynamodb" }
+data "aws_vpc_endpoint_service" "secretsmanager" { service = "secretsmanager" }
+data "aws_vpc_endpoint_service" "monitoring" { service = "monitoring" }
+data "aws_vpc_endpoint_service" "sns" { service = "sns" }
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "~> 3.0"
+
+  enable_dns_hostnames = true
+
+  name = "${local.resource_name_prefix}-vpc"
+  cidr = var.cidr
+
+  azs             = var.azs
+  private_subnets = var.private_subnets_cidr
+  public_subnets  = var.public_subnets_cidr
+
+  map_public_ip_on_launch = true
+}
+
+resource "aws_vpc_endpoint" "s3" {
+  vpc_id       = module.vpc.vpc_id
+  service_name = "com.amazonaws.${data.aws_region.current.name}.s3"
+}
+
+resource "aws_vpc_endpoint_route_table_association" "s3" {
+  count = length(local.private_route_table_ids)
+
+  vpc_endpoint_id = aws_vpc_endpoint.s3.id
+  route_table_id  = local.private_route_table_ids[count.index]
+}
+
+resource "aws_vpc_endpoint" "dynamodb" {
+  vpc_id       = module.vpc.vpc_id
+  service_name = data.aws_vpc_endpoint_service.dynamodb.service_name
+}
+
+resource "aws_vpc_endpoint_route_table_association" "private_dynamodb" {
+  count = length(local.private_route_table_ids)
+
+  vpc_endpoint_id = aws_vpc_endpoint.dynamodb.id
+  route_table_id  = local.private_route_table_ids[count.index]
+}
+
+resource "aws_vpc_endpoint" "secretsmanager" {
+  vpc_id            = module.vpc.vpc_id
+  service_name      = data.aws_vpc_endpoint_service.secretsmanager.service_name
+  vpc_endpoint_type = "Interface"
+
+  security_group_ids  = [aws_security_group.inbound_ssl_tcp.id]
+  subnet_ids          = module.vpc.private_subnets
+  private_dns_enabled = true
+}
+
+resource "aws_vpc_endpoint" "monitoring" {
+  vpc_id            = module.vpc.vpc_id
+  service_name      = data.aws_vpc_endpoint_service.monitoring.service_name
+  vpc_endpoint_type = "Interface"
+
+  security_group_ids  = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id]
+  subnet_ids          = module.vpc.private_subnets
+  private_dns_enabled = true
+}
+
+resource "aws_vpc_endpoint" "sns" {
+  vpc_id            = module.vpc.vpc_id
+  service_name      = data.aws_vpc_endpoint_service.sns.service_name
+  vpc_endpoint_type = "Interface"
+
+  security_group_ids  = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id]
+  subnet_ids          = module.vpc.private_subnets
+  private_dns_enabled = true
+}
+
+resource "aws_security_group" "inbound_ssl_tcp" {
+  name   = "${local.resource_name_prefix}-endpoint-443"
+  vpc_id = module.vpc.vpc_id
+}
+
+resource "aws_security_group_rule" "secretsmanager_sg_rule_sg" {
+  type                     = "ingress"
+  security_group_id        = aws_security_group.inbound_ssl_tcp.id
+  from_port                = 443
+  to_port                  = 443
+  protocol                 = "tcp"
+  source_security_group_id = aws_security_group.lambda_security_group.id
+  description              = "Allow ingress from security group"
+}
+
+resource "aws_security_group" "lambda_security_group" {
+  name   = "${local.resource_name_prefix}-service-endpoints"
+  vpc_id = module.vpc.vpc_id
+
+  egress {
+    from_port   = 5439
+    to_port     = 5439
+    protocol    = "tcp"
+    cidr_blocks = [module.vpc.vpc_cidr_block]
+    description = "Allow out to redshift through vpc endpoint"
+  }
+
+  egress {
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+    description = "Allow out to tcp through vpc endpoint"
+  }
+}

terraform/modules/vpc/outputs.tf → examples/basic/modules/vpc/outputs.tf

@@ -9,3 +9,7 @@ output "security_group_ids" {
 output "private_subnet_ids" {
   value = module.vpc.private_subnets
 }
+
+output "public_subnet_ids" {
+  value = module.vpc.public_subnets
+}

terraform/modules/vpc/variables.tf → examples/basic/modules/vpc/variables.tf

@@ -8,6 +8,11 @@ variable "private_subnets_cidr" {
   type        = list(string)
 }
 
+variable "public_subnets_cidr" {
+  description = "List of private subnets cidr"
+  type        = list(string)
+}
+
 variable "azs" {
   description = "List of available zones in selected region"
   type        = list(string)

examples/basic/remote_state.tf

@@ -1,8 +1,8 @@
 terraform {
   backend "s3" {
-    bucket         = "fast-data-qa-terraform"
-    key            = "terraform/state/terraform.tfstate"
-    region         = "eu-west-2"
-    encrypt        = true
+    bucket  = "fast-data-qa-terraform"
+    key     = "terraform/state/terraform.tfstate"
+    region  = "eu-west-2"
+    encrypt = true
   }
 }

examples/localstack/main.tf

@@ -11,14 +11,21 @@ provider "aws" {
 }
 
 module "integration_tests_data_qa" {
-  source                     = "../../terraform"
-  cloudfront_allowed_subnets = ["255.255.255.255/32"]
+  source = "../../terraform"
 
   data_test_storage_bucket_name = "integration-test-bucket"
   environment                   = "local"
 
   allure_report_image_uri = ""
   data_test_image_uri     = ""
   push_report_image_uri   = ""
+
+  reports_subnet_id = ""
+  reports_vpc_id    = ""
+
+  lambda_private_subnet_ids = []
+  lambda_security_group_ids = []
+
+  reports_whitelist_ips = []
 }
 

functions/allure_report/make_allure_report.py

@@ -3,13 +3,13 @@
 
 
 def handler(event, context):
-    qa_bucket = os.environ['QA_BUCKET']
-    cloudfront = os.environ['QA_CLOUDFRONT']
+    qa_bucket = os.environ['BUCKET']
+    reports_web = os.environ['REPORTS_WEB']
     report = event['report'].get('Payload')
     suite = report.get('suite_name')
     folder_key = report.get('folder_key')
     validate_id = report.get('validate_id')
-    link, key = create_json_report(suite, cloudfront, folder_key, validate_id)
+    link, key = create_json_report(suite, reports_web, folder_key, validate_id)
     os.system("chmod +x generate_report.sh")
     os.system(f"sh generate_report.sh {key} {qa_bucket}")
 

functions/allure_report/mapper.py

@@ -11,7 +11,7 @@
 import boto3
 import re
 
-qa_bucket = os.environ['QA_BUCKET']
+qa_bucket = os.environ['BUCKET']
 s3 = boto3.resource('s3')
 bucket = s3.Bucket(qa_bucket)
 

functions/data_test/data_test.py

@@ -14,8 +14,8 @@ def handler(event, context):
         s3 = boto3.resource("s3", endpoint_url=endpoint_url)
     else:
         s3 = boto3.resource("s3")
-    cloudfront = os.environ['QA_CLOUDFRONT']
-    qa_bucket_name = os.environ['QA_BUCKET']
+    cloudfront = os.environ['REPORTS_WEB']
+    qa_bucket_name = os.environ['BUCKET']
     run_name = event['run_name']
     if 'engine' in event:
         engine = event['engine']

functions/data_test/datasource.py

@@ -3,7 +3,7 @@
 import pathlib
 from data_source_factory import DataSourceFactory
 
-qa_bucket_name = os.environ['QA_BUCKET']
+qa_bucket_name = os.environ['BUCKET']
 
 
 def concat_source_list(source, source_engine):

functions/data_test/profiling.py

@@ -22,7 +22,7 @@
     endpoint_url = None
     s3 = boto3.resource("s3")
 
-qa_bucket_name = os.environ['QA_BUCKET']
+qa_bucket_name = os.environ['BUCKET']
 
 
 def generic_expectations_without_null(name, summary, batch, *args):

functions/report_push/push_data_report.py

@@ -17,9 +17,9 @@
 sns = boto3.client('sns')
 
 dynamodb = boto3.resource('dynamodb')
-dynamo_table_name = os.environ['QA_DYNAMODB_TABLE']
+dynamo_table_name = os.environ['DYNAMODB_TABLE']
 table = dynamodb.Table(dynamo_table_name)
-qa_bucket = os.environ['QA_BUCKET']
+qa_bucket = os.environ['BUCKET']
 environment = os.environ['ENVIRONMENT']
 sns_bugs_topic = os.environ.get('SNS_BUGS_TOPIC_ARN', None)
 autobug = False