Use absolute paths for s3 configs

examples/basic/main.tf → examples/docker_basic/main.tf

@@ -46,38 +46,23 @@ module "data_qa" {
   push_report_image_uri   = module.docker_image_push_report.image_uri
 
   data_reports_notification_settings = {
-    channel     = var.slack_channel
-    webhook_url = var.slack_webhook_url
+    channel     = "DataQASlackChannel"
+    webhook_url = "https://hooks.slack.com/services/........"
   }
 
   lambda_private_subnet_ids = module.vpc.private_subnet_ids
   lambda_security_group_ids = module.vpc.security_group_ids
 
   reports_vpc_id        = module.vpc.vpc_id
   reports_subnet_id     = module.vpc.public_subnet_ids[0]
-  reports_whitelist_ips = ["195.155.100.203/32"]
-}
-
-module "data_qa_intg" {
-  source = "../../terraform"
-
-  data_test_storage_bucket_name = "dqg-settings-intg"
-  environment                   = "intg"
-  project                       = "provectus"
-
-  allure_report_image_uri = module.docker_image_allure_report.image_uri
-  data_test_image_uri     = module.docker_image_data_test.image_uri
-  push_report_image_uri   = module.docker_image_push_report.image_uri
-
-  data_reports_notification_settings = {
-    channel     = var.slack_channel
-    webhook_url = var.slack_webhook_url
-  }
-
-  lambda_private_subnet_ids = module.vpc.private_subnet_ids
-  lambda_security_group_ids = module.vpc.security_group_ids
-
-  reports_vpc_id        = module.vpc.vpc_id
-  reports_subnet_id     = module.vpc.public_subnet_ids[0]
-  reports_whitelist_ips = ["195.155.100.203/32"]
+  reports_whitelist_ips = ["0.0.0.0/0"] # Available from everywhere
+
+  test_coverage_path     = "../../configs/test_coverage.json"
+  pipeline_config_path   = "../../configs/pipeline.json"
+  pks_path               = "../../configs/pks.json"
+  sort_keys_path         = "../../configs/sort_keys.json"
+  mapping_path           = "../../configs/mapping.json"
+  manifest_path          = "../../configs/manifest.json"
+  great_expectation_path = "../../templates/great_expectations.yml"
+  expectations_store     = "../../expectations_store"
 }

terraform/modules.tf

@@ -31,7 +31,7 @@ module "basic_slack_alerting" {
   slack_sns_topic_name = "dqg-alerting-${var.environment}"
   slack_username       = "DQG-alerting"
 
-  step_functions_to_monitor = ["${local.resource_name_prefix}-fast-data-qa"]
+  step_functions_to_monitor = [aws_sfn_state_machine.fast_data_qa.name]
 
   resource_name_prefix = local.resource_name_prefix
 }

terraform/modules/alerting/main.tf

@@ -9,7 +9,7 @@ resource "aws_kms_ciphertext" "slack_url" {
 
 module "slack_notification" {
   source  = "terraform-aws-modules/notify-slack/aws"
-  version = "~> 5.0"
+  version = "6.0.0"
 
   sns_topic_name = var.slack_sns_topic_name
 

terraform/modules/s3-configs/main.tf

@@ -1,5 +1,6 @@
 resource "aws_s3_bucket" "settings_bucket" {
-  bucket = var.data_test_storage_bucket_name
+  bucket        = var.data_test_storage_bucket_name
+  force_destroy = true
 }
 
 resource "aws_s3_bucket_public_access_block" "public_access_block_fast_data_qa" {
@@ -21,67 +22,67 @@ resource "aws_s3_bucket_versioning" "fast-data-qa-bucket" {
 resource "aws_s3_object" "great_expectations_yml" {
   bucket       = aws_s3_bucket.settings_bucket.bucket
   content_type = "application/x-yaml"
-  content = templatefile("${path.module}/${var.great_expectation_path}", {
+  content = templatefile("${var.great_expectation_path}", {
     bucket = aws_s3_bucket.settings_bucket.bucket
   })
   key = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/great_expectations.yml"
-  etag = md5(templatefile("${path.module}/${var.great_expectation_path}", {
+  etag = md5(templatefile("${var.great_expectation_path}", {
     bucket = aws_s3_bucket.settings_bucket.bucket
   }))
 }
 
 resource "aws_s3_object" "test_configs" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  source = "${path.module}/${var.test_coverage_path}"
+  source = var.test_coverage_path
   key    = "test_configs/test_coverage.json"
-  etag   = filemd5("${path.module}/${var.test_coverage_path}")
+  etag   = filemd5("${var.test_coverage_path}")
 }
 
 resource "aws_s3_object" "pipeline_config" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  source = "${path.module}/${var.pipeline_config_path}"
+  source = var.pipeline_config_path
   key    = "test_configs/pipeline.json"
-  etag   = filemd5("${path.module}/${var.pipeline_config_path}")
+  etag   = filemd5("${var.pipeline_config_path}")
 }
 
 resource "aws_s3_object" "pks_config" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  source = "${path.module}/${var.pks_path}"
+  source = var.pks_path
   key    = "test_configs/pks.json"
-  etag   = filemd5("${path.module}/${var.pks_path}")
+  etag   = filemd5("${var.pks_path}")
 }
 
 resource "aws_s3_object" "sort_keys_config" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  source = "${path.module}/${var.sort_keys_path}"
+  source = var.sort_keys_path
   key    = "test_configs/sort_keys.json"
-  etag   = filemd5("${path.module}/${var.sort_keys_path}")
+  etag   = filemd5("${var.sort_keys_path}")
 }
 
 resource "aws_s3_object" "mapping_config" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  source = "${path.module}/${var.mapping_path}"
+  source = var.mapping_path
   key    = "test_configs/mapping.json"
-  etag   = filemd5("${path.module}/${var.mapping_path}")
+  etag   = filemd5("${var.mapping_path}")
 }
 
 
 resource "aws_s3_object" "expectations_store" {
-  for_each = fileset("${path.module}/${var.expectations_store}", "**")
+  for_each = fileset("${var.expectations_store}", "**")
   bucket   = aws_s3_bucket.settings_bucket.bucket
-  source   = "${path.module}/${var.expectations_store}/${each.value}"
+  source   = "${var.expectations_store}/${each.value}"
   key      = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/expectations/${each.value}"
-  etag     = filemd5("${path.module}/${var.expectations_store}/${each.value}")
+  etag     = filemd5("${var.expectations_store}/${each.value}")
 }
 
 resource "aws_s3_object" "test_config_manifest" {
   bucket = aws_s3_bucket.settings_bucket.bucket
-  etag = md5(templatefile("${path.module}/${var.manifest_path}", {
+  etag = md5(templatefile("${var.manifest_path}", {
     env_name    = var.environment,
     bucket_name = aws_s3_bucket.settings_bucket.bucket
   }))
   content_type = "application/json"
-  content = templatefile("${path.module}/${var.manifest_path}",
+  content = templatefile("${var.manifest_path}",
     {
       env_name    = var.environment,
       bucket_name = aws_s3_bucket.settings_bucket.bucket

terraform/versions.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "= 4.67.0"
+      version = "= 4.64.0"
     }
     local = {
       source  = "hashicorp/local"