diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..cb82671 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,83 @@ +name: Pre-Commit + +on: + pull_request: + branches: + - main + - master + +env: + TERRAFORM_DOCS_VERSION: v0.16.0 + TFLINT_VERSION: v0.44.1 + +jobs: + collectInputs: + name: Collect workflow inputs + runs-on: ubuntu-latest + outputs: + directories: ${{ steps.dirs.outputs.directories }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get root directories + id: dirs + uses: clowdhaus/terraform-composite-actions/directories@v1.8.3 + + preCommitMinVersions: + name: Min TF pre-commit + needs: collectInputs + runs-on: ubuntu-latest + strategy: + matrix: + directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.2.4 + with: + directory: ${{ matrix.directory }} + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory != '.' }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory == '.' }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + args: 'terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)' + + preCommitMaxVersion: + name: Max TF pre-commit + runs-on: ubuntu-latest + needs: collectInputs + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.2.4 + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.maxVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} + install-hcledit: true diff --git a/.gitignore b/.gitignore index f5a2b23..6a4eca2 100644 --- a/.gitignore +++ b/.gitignore @@ -168,4 +168,4 @@ tmp examples/*/plan *.auto.tfvars -.envrc \ No newline at end of file +.envrc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4395e67 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.79.1 + hooks: + - id: terraform_fmt + - id: terraform_validate + - id: terraform_docs + args: + - '--args=--lockfile=false' + - id: terraform_tflint + args: + - '--args=--only=terraform_deprecated_interpolation' + - '--args=--only=terraform_deprecated_index' + - '--args=--only=terraform_unused_declarations' + - '--args=--only=terraform_comment_syntax' + - '--args=--only=terraform_documented_outputs' + - '--args=--only=terraform_documented_variables' + - '--args=--only=terraform_typed_variables' + - '--args=--only=terraform_module_pinned_source' + - '--args=--only=terraform_naming_convention' + - '--args=--only=terraform_required_version' + - '--args=--only=terraform_required_providers' + - '--args=--only=terraform_standard_module_structure' + - '--args=--only=terraform_workspace_remote' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-merge-conflict + - id: end-of-file-fixer diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60400a1..afb805b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,4 +46,4 @@ By participating in this project, you agree to abide by our [Code of Conduct](ht - If you have any questions or need assistance, feel free to open an issue or reach out to the maintainers on GitHub. - We welcome any suggestions for improvements or new features. -We look forward to working with you and improving Data-Quality-Gate together! \ No newline at end of file +We look forward to working with you and improving Data-Quality-Gate together! diff --git a/LICENSE.md b/LICENSE.md index f49a4e1..261eeb9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -198,4 +198,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/configs/manifest.json b/configs/manifest.json index 90e9d81..f37030c 100644 --- a/configs/manifest.json +++ b/configs/manifest.json @@ -9,4 +9,4 @@ "globalUploadSettings":{ "format":"JSON" } -} \ No newline at end of file +} diff --git a/configs/mapping.json b/configs/mapping.json index 51f7a20..84cbeb7 100644 --- a/configs/mapping.json +++ b/configs/mapping.json @@ -3,4 +3,4 @@ "CALL ID": "call id", "new_col_added0": "year_month" } -} \ No newline at end of file +} diff --git a/configs/pipeline.json b/configs/pipeline.json index 934e5bf..5ae6cba 100644 --- a/configs/pipeline.json +++ b/configs/pipeline.json @@ -9,4 +9,4 @@ "only_failed": true } -} \ No newline at end of file +} diff --git a/configs/pks.json b/configs/pks.json index e192edf..01c8fb0 100644 --- a/configs/pks.json +++ b/configs/pks.json @@ -6,4 +6,4 @@ "RECORD_NUMBER" ] } -} \ No newline at end of file +} diff --git a/configs/sort_keys.json b/configs/sort_keys.json index 3beab44..347368f 100644 --- a/configs/sort_keys.json +++ b/configs/sort_keys.json @@ -4,4 +4,4 @@ "update_dt" ] } -} \ No newline at end of file +} diff --git a/configs/test_coverage.json b/configs/test_coverage.json index 294cd41..7a76a66 100644 --- a/configs/test_coverage.json +++ b/configs/test_coverage.json @@ -2,4 +2,4 @@ "sample": { "complexSuite": true } -} \ No newline at end of file +} diff --git a/examples/docker_basic/README.md b/examples/basic/README.md similarity index 100% rename from examples/docker_basic/README.md rename to examples/basic/README.md diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..37a199c --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,43 @@ +provider "aws" { + region = "eu-west-2" + + default_tags { + tags = { + Terraform = true + } + } +} + +module "data_qa" { + source = "../../terraform" + + data_test_storage_bucket_name = "dqg-settings-dev" + s3_source_data_bucket = "data-bucket-name" + environment = "demo" + project = "provectus" + + allure_report_image_uri = "..." + data_test_image_uri = "..." + push_report_image_uri = "..." + + data_reports_notification_settings = { + channel = "DataReportSlackChannelName" + webhook_url = "https://hooks.slack.com/services/........" + } + + lambda_private_subnet_ids = ["private_subnet_id"] + lambda_security_group_ids = ["security_group_id"] + + reports_vpc_id = "some_vpc_id" + reports_subnet_id = "subnet_id" + reports_whitelist_ips = ["0.0.0.0/0"] + + test_coverage_path = "../../configs/test_coverage.json" + pipeline_config_path = "../../configs/pipeline.json" + pks_path = "../../configs/pks.json" + sort_keys_path = "../../configs/sort_keys.json" + mapping_path = "../../configs/mapping.json" + manifest_path = "../../configs/manifest.json" + great_expectation_path = "../../templates/great_expectations.yml" + expectations_store = "../../expectations_store" +} diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf new file mode 100644 index 0000000..9eb52cd --- /dev/null +++ b/examples/basic/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } +} diff --git a/examples/docker_basic/docker.tf b/examples/docker_basic/docker.tf deleted file mode 100644 index 7b1e53b..0000000 --- a/examples/docker_basic/docker.tf +++ /dev/null @@ -1,60 +0,0 @@ -resource "random_uuid" "allure_report" { - keepers = { - for filename in setunion( - fileset("../../functions/allure_report/", "*.py"), - fileset("../../functions/allure_report/", "requirements.txt"), - fileset("../../functions/allure_report/", "Dockerfile"), - fileset("../../functions/allure_report/", "generate_report.sh") - ) : - filename => filemd5("../../functions/allure_report/${filename}") - } -} - -resource "random_uuid" "data_test" { - keepers = { - for filename in setunion( - fileset("../../functions/data_test/", "*.py"), - fileset("../../functions/data_test/", "requirements.txt"), - fileset("../../functions/data_test/", "Dockerfile") - ) : - filename => filemd5("../../functions/data_test/${filename}") - } -} - -resource "random_uuid" "push_report" { - keepers = { - for filename in setunion( - fileset("../../functions/report_push/", "*.py"), - fileset("../../functions/report_push/", "requirements.txt"), - fileset("../../functions/report_push/", "Dockerfile") - ) : - filename => filemd5("../../functions/report_push/${filename}") - } -} - -module "docker_image_push_report" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-push-report" - image_tag = random_uuid.push_report.result - source_path = "../../functions/report_push" -} - -module "docker_image_data_test" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-data-test" - image_tag = random_uuid.data_test.result - source_path = "../../functions/data_test" -} - -module "docker_image_allure_report" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-allure-report" - image_tag = random_uuid.allure_report.result - source_path = "../../functions/allure_report" -} diff --git a/examples/docker_basic/main.tf b/examples/docker_basic/main.tf deleted file mode 100644 index c71b340..0000000 --- a/examples/docker_basic/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -provider "aws" { - region = "eu-west-2" - - default_tags { - tags = { - Terraform = true - } - } -} - -provider "docker" { - registry_auth { - address = format("%v.dkr.ecr.%v.amazonaws.com", data.aws_caller_identity.current.account_id, data.aws_region.current.name) - username = data.aws_ecr_authorization_token.token.user_name - password = data.aws_ecr_authorization_token.token.password - } -} - -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} -data "aws_ecr_authorization_token" "token" {} -data "aws_availability_zones" "available" { - state = "available" -} - -module "vpc" { - source = "./modules/vpc" - - resource_name_prefix = "provectus-infra" - - cidr = "172.21.0.0/16" - private_subnets_cidr = ["172.21.16.0/20"] - public_subnets_cidr = ["172.21.32.0/20"] - azs = data.aws_availability_zones.available.zone_ids -} - -module "data_qa" { - source = "../../terraform" - - data_test_storage_bucket_name = "dqg-settings-dev" - s3_source_data_bucket = "data-bucket-name" - environment = "demo" - project = "provectus" - - allure_report_image_uri = module.docker_image_allure_report.image_uri - data_test_image_uri = module.docker_image_data_test.image_uri - push_report_image_uri = module.docker_image_push_report.image_uri - - data_reports_notification_settings = { - channel = "DataQASlackChannel" - webhook_url = "https://hooks.slack.com/services/........" - } - - lambda_private_subnet_ids = module.vpc.private_subnet_ids - lambda_security_group_ids = module.vpc.security_group_ids - - reports_vpc_id = module.vpc.vpc_id - reports_subnet_id = module.vpc.public_subnet_ids[0] - reports_whitelist_ips = ["0.0.0.0/0"] # Available from everywhere - - test_coverage_path = "../../configs/test_coverage.json" - pipeline_config_path = "../../configs/pipeline.json" - pks_path = "../../configs/pks.json" - sort_keys_path = "../../configs/sort_keys.json" - mapping_path = "../../configs/mapping.json" - manifest_path = "../../configs/manifest.json" - great_expectation_path = "../../templates/great_expectations.yml" - expectations_store = "../../expectations_store" -} diff --git a/examples/docker_basic/modules/vpc/main.tf b/examples/docker_basic/modules/vpc/main.tf deleted file mode 100644 index 273d6cb..0000000 --- a/examples/docker_basic/modules/vpc/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -locals { - resource_name_prefix = var.resource_name_prefix - private_route_table_ids = module.vpc.private_route_table_ids -} - -data "aws_region" "current" {} -data "aws_vpc_endpoint_service" "dynamodb" { service = "dynamodb" } -data "aws_vpc_endpoint_service" "secretsmanager" { service = "secretsmanager" } -data "aws_vpc_endpoint_service" "monitoring" { service = "monitoring" } -data "aws_vpc_endpoint_service" "sns" { service = "sns" } - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - enable_dns_hostnames = true - - name = "${local.resource_name_prefix}-vpc" - cidr = var.cidr - - azs = var.azs - private_subnets = var.private_subnets_cidr - public_subnets = var.public_subnets_cidr - - map_public_ip_on_launch = true -} - -resource "aws_vpc_endpoint" "s3" { - vpc_id = module.vpc.vpc_id - service_name = "com.amazonaws.${data.aws_region.current.name}.s3" -} - -resource "aws_vpc_endpoint_route_table_association" "s3" { - count = length(local.private_route_table_ids) - - vpc_endpoint_id = aws_vpc_endpoint.s3.id - route_table_id = local.private_route_table_ids[count.index] -} - -resource "aws_vpc_endpoint" "dynamodb" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.dynamodb.service_name -} - -resource "aws_vpc_endpoint_route_table_association" "private_dynamodb" { - count = length(local.private_route_table_ids) - - vpc_endpoint_id = aws_vpc_endpoint.dynamodb.id - route_table_id = local.private_route_table_ids[count.index] -} - -resource "aws_vpc_endpoint" "secretsmanager" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.secretsmanager.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_vpc_endpoint" "monitoring" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.monitoring.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_vpc_endpoint" "sns" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.sns.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_security_group" "inbound_ssl_tcp" { - name = "${local.resource_name_prefix}-endpoint-443" - vpc_id = module.vpc.vpc_id -} - -resource "aws_security_group_rule" "secretsmanager_sg_rule_sg" { - type = "ingress" - security_group_id = aws_security_group.inbound_ssl_tcp.id - from_port = 443 - to_port = 443 - protocol = "tcp" - source_security_group_id = aws_security_group.lambda_security_group.id - description = "Allow ingress from security group" -} - -resource "aws_security_group" "lambda_security_group" { - name = "${local.resource_name_prefix}-service-endpoints" - vpc_id = module.vpc.vpc_id - - egress { - from_port = 5439 - to_port = 5439 - protocol = "tcp" - cidr_blocks = [module.vpc.vpc_cidr_block] - description = "Allow out to redshift through vpc endpoint" - } - - egress { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - description = "Allow out to tcp through vpc endpoint" - } -} \ No newline at end of file diff --git a/examples/docker_basic/modules/vpc/outputs.tf b/examples/docker_basic/modules/vpc/outputs.tf deleted file mode 100644 index 2821927..0000000 --- a/examples/docker_basic/modules/vpc/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "vpc_id" { - value = module.vpc.vpc_id -} - -output "security_group_ids" { - value = [aws_security_group.lambda_security_group.id] -} - -output "private_subnet_ids" { - value = module.vpc.private_subnets -} - -output "public_subnet_ids" { - value = module.vpc.public_subnets -} diff --git a/examples/docker_basic/modules/vpc/variables.tf b/examples/docker_basic/modules/vpc/variables.tf deleted file mode 100644 index 2fc44c5..0000000 --- a/examples/docker_basic/modules/vpc/variables.tf +++ /dev/null @@ -1,24 +0,0 @@ -variable "cidr" { - description = "VPC cidr block" - type = string -} - -variable "private_subnets_cidr" { - description = "List of private subnets cidr" - type = list(string) -} - -variable "public_subnets_cidr" { - description = "List of private subnets cidr" - type = list(string) -} - -variable "azs" { - description = "List of available zones in selected region" - type = list(string) -} - -variable "resource_name_prefix" { - description = "Resource name prefix used to generate resources" - type = string -} diff --git a/examples/docker_basic/variables.tf b/examples/docker_basic/variables.tf deleted file mode 100644 index bb5f6a1..0000000 --- a/examples/docker_basic/variables.tf +++ /dev/null @@ -1,2 +0,0 @@ -variable "slack_webhook_url" { type = string } -variable "slack_channel" { type = string } \ No newline at end of file diff --git a/examples/docker_basic/versions.tf b/examples/docker_basic/versions.tf deleted file mode 100644 index ca331be..0000000 --- a/examples/docker_basic/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "= 4.67.0" - } - local = { - source = "hashicorp/local" - version = "~> 2.2.3" - } - docker = { - source = "kreuzwerker/docker" - version = "2.18.0" - } - } -} \ No newline at end of file diff --git a/functions/allure_report/requirements.txt b/functions/allure_report/requirements.txt index 1cae9ae..f3ed878 100644 --- a/functions/allure_report/requirements.txt +++ b/functions/allure_report/requirements.txt @@ -10,4 +10,4 @@ boto3==1.17.1 boto==2.49.0 pyarrow==3.0.0 fastparquet==0.8.1 -awswrangler==2.12.1 \ No newline at end of file +awswrangler==2.12.1 diff --git a/functions/data_test/Dockerfile b/functions/data_test/Dockerfile index c66c7ad..6b21d86 100755 --- a/functions/data_test/Dockerfile +++ b/functions/data_test/Dockerfile @@ -5,4 +5,4 @@ COPY ./*.py ./ COPY great_expectations ./great_expectations RUN rm -f /etc/timezone RUN python3 -c "import tzlocal; tzlocal.get_localzone()" -CMD ["data_test.handler"] \ No newline at end of file +CMD ["data_test.handler"] diff --git a/functions/data_test/great_expectations/.gitignore b/functions/data_test/great_expectations/.gitignore index 241ec61..e9ef387 100755 --- a/functions/data_test/great_expectations/.gitignore +++ b/functions/data_test/great_expectations/.gitignore @@ -1,2 +1,2 @@ uncommitted/ -allure-report \ No newline at end of file +allure-report diff --git a/functions/data_test/great_expectations/expectations/.ge_store_backend_id b/functions/data_test/great_expectations/expectations/.ge_store_backend_id index 025ba49..2d7c45d 100755 --- a/functions/data_test/great_expectations/expectations/.ge_store_backend_id +++ b/functions/data_test/great_expectations/expectations/.ge_store_backend_id @@ -1 +1 @@ -store_backend_id = f6a15c13-c249-416b-8beb-2e540a245773 \ No newline at end of file +store_backend_id = f6a15c13-c249-416b-8beb-2e540a245773 diff --git a/functions/report_push/requirements.txt b/functions/report_push/requirements.txt index 1b127f8..7ac3ad7 100644 --- a/functions/report_push/requirements.txt +++ b/functions/report_push/requirements.txt @@ -4,4 +4,4 @@ importlib-metadata==6.0.0 s3fs==0.4.2 python-dateutil==2.8.2 awswrangler==2.19.0 -jira==3.2.0 \ No newline at end of file +jira==3.2.0 diff --git a/terraform/README.md b/terraform/README.md index ad1ec4e..1b123ed 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -120,4 +120,4 @@ | [lambda\_report\_push\_arn](#output\_lambda\_report\_push\_arn) | n/a | | [report\_push\_role\_arn](#output\_report\_push\_role\_arn) | n/a | | [step\_function\_arn](#output\_step\_function\_arn) | n/a | - \ No newline at end of file + diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf index 5493fb4..039c08c 100644 --- a/terraform/dynamodb.tf +++ b/terraform/dynamodb.tf @@ -73,4 +73,4 @@ resource "aws_appautoscaling_policy" "data_qa_report_table_write_policy" { scale_out_cooldown = lookup(var.dynamodb_autoscaling_write, "scale_out_cooldown", var.dynamodb_autoscaling_defaults["scale_out_cooldown"]) target_value = lookup(var.dynamodb_autoscaling_write, "target_value", var.dynamodb_autoscaling_defaults["target_value"]) } -} \ No newline at end of file +} diff --git a/terraform/main.tf b/terraform/main.tf index 47b9b3b..1d41d64 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,10 +1,6 @@ data "aws_caller_identity" "current" {} data "aws_region" "current" {} -data "aws_availability_zones" "available" { - state = "available" -} locals { resource_name_prefix = "${var.project}-${var.environment}" - cloudwatch_prefix = replace(title(replace(local.resource_name_prefix, "-", " ")), " ", "") } diff --git a/terraform/modules.tf b/terraform/modules.tf index 5a70119..cefcd67 100644 --- a/terraform/modules.tf +++ b/terraform/modules.tf @@ -1,4 +1,4 @@ -module "athena-connector" { +module "athena_connector" { source = "./modules/athena-connector" primary_aws_region = data.aws_region.current.name diff --git a/terraform/modules/alerting/main.tf b/terraform/modules/alerting/main.tf index c6f93d5..e1f1417 100644 --- a/terraform/modules/alerting/main.tf +++ b/terraform/modules/alerting/main.tf @@ -51,4 +51,3 @@ data "aws_sfn_state_machine" "step_functions" { for_each = var.step_functions_to_monitor name = each.key } - diff --git a/terraform/modules/alerting/outputs.tf b/terraform/modules/alerting/outputs.tf index af0d246..f912c27 100644 --- a/terraform/modules/alerting/outputs.tf +++ b/terraform/modules/alerting/outputs.tf @@ -1,3 +1,4 @@ output "sns_topic_arn" { - value = module.slack_notification.slack_topic_arn + description = "Notifications topic arn" + value = module.slack_notification.slack_topic_arn } diff --git a/terraform/modules/alerting/variables.tf b/terraform/modules/alerting/variables.tf index 14cb026..b87f4f1 100644 --- a/terraform/modules/alerting/variables.tf +++ b/terraform/modules/alerting/variables.tf @@ -1,7 +1,19 @@ -variable "slack_sns_topic_name" { type = string } -variable "slack_webhook_url" { type = string } -variable "slack_channel" { type = string } -variable "slack_username" { type = string } +variable "slack_sns_topic_name" { + description = "Sns topic name to forward notifications to" + type = string +} +variable "slack_webhook_url" { + description = "Slack webhook url in form https://hooks.slack.com/services/........" + type = string +} +variable "slack_channel" { + description = "Slack channel to send notifications" + type = string +} +variable "slack_username" { + description = "Slack username which will be used as author of notifications" + type = string +} variable "resource_name_prefix" { description = "Resource name prefix used to generate resources" @@ -9,23 +21,27 @@ variable "resource_name_prefix" { } variable "step_functions_to_monitor" { - type = set(string) - default = [] + description = "List of step functions for which to create cloudwatch metrics alarm" + type = set(string) + default = [] } variable "period" { - default = 60 description = "The period in seconds over which the specified statistic is applied." + type = number + default = 60 } variable "evaluation_periods" { - default = 1 description = "The number of periods over which data is compared to the specified threshold." + type = number + default = 1 } variable "datapoints_to_alarm" { - default = 1 description = "The number of datapoints that must be breaching to trigger the alarm." + type = number + default = 1 } variable "lambda_function_vpc_security_group_ids" { diff --git a/terraform/modules/alerting/versions.tf b/terraform/modules/alerting/versions.tf new file mode 100644 index 0000000..a8ce400 --- /dev/null +++ b/terraform/modules/alerting/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } + required_version = "~> 1.1" +} diff --git a/terraform/modules/athena-connector/iam.tf b/terraform/modules/athena-connector/iam.tf index feed739..a0d85a8 100644 --- a/terraform/modules/athena-connector/iam.tf +++ b/terraform/modules/athena-connector/iam.tf @@ -81,4 +81,4 @@ resource "aws_iam_policy" "athena_connector_lambda_policy" { resource "aws_iam_role_policy_attachment" "athena_connector_basic_policy" { role = aws_iam_role.athena_connector_lambda_role.name policy_arn = aws_iam_policy.athena_connector_lambda_policy.arn -} \ No newline at end of file +} diff --git a/terraform/modules/athena-connector/main.tf b/terraform/modules/athena-connector/main.tf index b589307..61670a3 100644 --- a/terraform/modules/athena-connector/main.tf +++ b/terraform/modules/athena-connector/main.tf @@ -44,4 +44,4 @@ resource "null_resource" "delete_athena_dynamodb_connector" { } depends_on = [null_resource.athena_dynamodb_connector] -} \ No newline at end of file +} diff --git a/terraform/modules/athena-connector/outputs.tf b/terraform/modules/athena-connector/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/athena-connector/variables.tf b/terraform/modules/athena-connector/variables.tf index f02b229..fdd33b5 100644 --- a/terraform/modules/athena-connector/variables.tf +++ b/terraform/modules/athena-connector/variables.tf @@ -24,4 +24,4 @@ variable "vpc_security_group_ids" { description = "List of security group assigned to lambda. If null value, default subnet and vpc will be used" type = list(string) default = null -} \ No newline at end of file +} diff --git a/terraform/modules/athena-connector/versions.tf b/terraform/modules/athena-connector/versions.tf new file mode 100644 index 0000000..61cd52c --- /dev/null +++ b/terraform/modules/athena-connector/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + null = { + source = "hashicorp/null" + version = "~> 3.2.1" + } + } + required_version = "~> 1.1" +} diff --git a/terraform/modules/s3-configs/main.tf b/terraform/modules/s3-configs/main.tf index fe22d85..bd3f7fd 100644 --- a/terraform/modules/s3-configs/main.tf +++ b/terraform/modules/s3-configs/main.tf @@ -3,7 +3,7 @@ resource "aws_s3_bucket" "settings_bucket" { force_destroy = true } -resource "aws_s3_bucket_public_access_block" "public_access_block_fast_data_qa" { +resource "aws_s3_bucket_public_access_block" "settings_bucket_public_access_block" { bucket = aws_s3_bucket.settings_bucket.id block_public_acls = true block_public_policy = true @@ -11,7 +11,7 @@ resource "aws_s3_bucket_public_access_block" "public_access_block_fast_data_qa" restrict_public_buckets = true } -resource "aws_s3_bucket_versioning" "fast-data-qa-bucket" { +resource "aws_s3_bucket_versioning" "settings_bucket_versioning" { bucket = aws_s3_bucket.settings_bucket.id versioning_configuration { @@ -22,11 +22,11 @@ resource "aws_s3_bucket_versioning" "fast-data-qa-bucket" { resource "aws_s3_object" "great_expectations_yml" { bucket = aws_s3_bucket.settings_bucket.bucket content_type = "application/x-yaml" - content = templatefile("${var.great_expectation_path}", { + content = templatefile(var.great_expectation_path, { bucket = aws_s3_bucket.settings_bucket.bucket }) key = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/great_expectations.yml" - etag = md5(templatefile("${var.great_expectation_path}", { + etag = md5(templatefile(var.great_expectation_path, { bucket = aws_s3_bucket.settings_bucket.bucket })) } @@ -35,40 +35,40 @@ resource "aws_s3_object" "test_configs" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.test_coverage_path key = "test_configs/test_coverage.json" - etag = filemd5("${var.test_coverage_path}") + etag = filemd5(var.test_coverage_path) } resource "aws_s3_object" "pipeline_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.pipeline_config_path key = "test_configs/pipeline.json" - etag = filemd5("${var.pipeline_config_path}") + etag = filemd5(var.pipeline_config_path) } resource "aws_s3_object" "pks_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.pks_path key = "test_configs/pks.json" - etag = filemd5("${var.pks_path}") + etag = filemd5(var.pks_path) } resource "aws_s3_object" "sort_keys_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.sort_keys_path key = "test_configs/sort_keys.json" - etag = filemd5("${var.sort_keys_path}") + etag = filemd5(var.sort_keys_path) } resource "aws_s3_object" "mapping_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.mapping_path key = "test_configs/mapping.json" - etag = filemd5("${var.mapping_path}") + etag = filemd5(var.mapping_path) } resource "aws_s3_object" "expectations_store" { - for_each = fileset("${var.expectations_store}", "**") + for_each = fileset(var.expectations_store, "**") bucket = aws_s3_bucket.settings_bucket.bucket source = "${var.expectations_store}/${each.value}" key = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/expectations/${each.value}" @@ -77,12 +77,12 @@ resource "aws_s3_object" "expectations_store" { resource "aws_s3_object" "test_config_manifest" { bucket = aws_s3_bucket.settings_bucket.bucket - etag = md5(templatefile("${var.manifest_path}", { + etag = md5(templatefile(var.manifest_path, { env_name = var.environment, bucket_name = aws_s3_bucket.settings_bucket.bucket })) content_type = "application/json" - content = templatefile("${var.manifest_path}", + content = templatefile(var.manifest_path, { env_name = var.environment, bucket_name = aws_s3_bucket.settings_bucket.bucket diff --git a/terraform/modules/s3-configs/outputs.tf b/terraform/modules/s3-configs/outputs.tf index e6caf19..1cef344 100644 --- a/terraform/modules/s3-configs/outputs.tf +++ b/terraform/modules/s3-configs/outputs.tf @@ -1,3 +1,4 @@ output "bucket_name" { - value = aws_s3_bucket.settings_bucket.bucket -} \ No newline at end of file + description = "Name of s3 configs bucket" + value = aws_s3_bucket.settings_bucket.bucket +} diff --git a/terraform/modules/s3-configs/versions.tf b/terraform/modules/s3-configs/versions.tf new file mode 100644 index 0000000..a8ce400 --- /dev/null +++ b/terraform/modules/s3-configs/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } + required_version = "~> 1.1" +} diff --git a/terraform/modules/s3-gateway/outputs.tf b/terraform/modules/s3-gateway/outputs.tf index 00884ef..05221de 100644 --- a/terraform/modules/s3-gateway/outputs.tf +++ b/terraform/modules/s3-gateway/outputs.tf @@ -1,3 +1,4 @@ output "s3_gateway_address" { - value = replace(aws_instance.s3_gateway.public_dns, "https", "http") + description = "DNS http address of s3 gateway" + value = replace(aws_instance.s3_gateway.public_dns, "https", "http") } diff --git a/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh b/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh index 82bc853..29743dd 100644 --- a/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh +++ b/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh @@ -19,4 +19,4 @@ export PROXY_CACHE_VALID_FORBIDDEN=30s export CORS_ENABLED=false export S3_DEBUG=false -bash standalone_ubuntu_oss_install.sh \ No newline at end of file +bash standalone_ubuntu_oss_install.sh diff --git a/terraform/modules/s3-gateway/versions.tf b/terraform/modules/s3-gateway/versions.tf new file mode 100644 index 0000000..a8ce400 --- /dev/null +++ b/terraform/modules/s3-gateway/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } + required_version = "~> 1.1" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 239ae85..99e4a7c 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,32 +1,24 @@ output "step_function_arn" { - value = aws_sfn_state_machine.fast_data_qa.arn + description = "DataQA step function arn" + value = aws_sfn_state_machine.fast_data_qa.arn } output "lambda_allure_arn" { - value = module.lambda_allure_report.lambda_function_arn + description = "Allure reports generation lambda arn" + value = module.lambda_allure_report.lambda_function_arn } output "lambda_data_test_arn" { - value = module.lambda_data_test.lambda_function_arn + description = "Data test generation/running lambda arn" + value = module.lambda_data_test.lambda_function_arn } output "lambda_report_push_arn" { - value = module.lambda_push_report.lambda_function_arn -} - -output "allure_report_role_arn" { - value = module.lambda_allure_report.lambda_role_arn -} - -output "data_test_role_arn" { - value = module.lambda_data_test.lambda_role_arn -} - -output "report_push_role_arn" { - value = module.lambda_push_report.lambda_role_arn + description = "Report push to dynamodb lambda arn" + value = module.lambda_push_report.lambda_function_arn } output "bucket" { description = "Data quality gate bucket with settings and generated tests" value = module.s3_bucket.bucket_name -} \ No newline at end of file +} diff --git a/terraform/step_function.tf b/terraform/step_function.tf index 171248f..9ab889b 100644 --- a/terraform/step_function.tf +++ b/terraform/step_function.tf @@ -1,4 +1,4 @@ -resource "aws_cloudwatch_log_group" "state-machine-log-group" { +resource "aws_cloudwatch_log_group" "state_machine_log_group" { name = "/aws/${local.resource_name_prefix}/states/fast-data-qa-logs" retention_in_days = 0 } @@ -130,7 +130,7 @@ DEFINITION logging_configuration { include_execution_data = true level = "ALL" - log_destination = "${aws_cloudwatch_log_group.state-machine-log-group.arn}:*" + log_destination = "${aws_cloudwatch_log_group.state_machine_log_group.arn}:*" } tracing_configuration { @@ -156,15 +156,15 @@ resource "aws_iam_role" "step_functions_fast_data_qa" { ) force_detach_policies = false managed_policy_arns = [ - aws_iam_policy.CloudWatchLogsDeliveryFullAccessPolicy.arn, - aws_iam_policy.LambdaInvokeScopedAccessPolicy.arn, - aws_iam_policy.XRayAccessPolicy.arn + aws_iam_policy.cloud_watch_logs_delivery_full_access_policy.arn, + aws_iam_policy.lambda_invoke_scoped_access_policy.arn, + aws_iam_policy.xray_access_policy.arn ] max_session_duration = 3600 path = "/${var.environment}/" } -resource "aws_iam_policy" "CloudWatchLogsDeliveryFullAccessPolicy" { +resource "aws_iam_policy" "cloud_watch_logs_delivery_full_access_policy" { description = "Allows AWS Step Functions to write execution logs to CloudWatch Logs on your behalf" path = "/${var.environment}/" policy = jsonencode( @@ -190,7 +190,7 @@ resource "aws_iam_policy" "CloudWatchLogsDeliveryFullAccessPolicy" { ) } -resource "aws_iam_policy" "LambdaInvokeScopedAccessPolicy" { +resource "aws_iam_policy" "lambda_invoke_scoped_access_policy" { description = "Allow AWS Step Functions to invoke Lambda functions on your behalf" path = "/${var.environment}/" policy = jsonencode( @@ -213,7 +213,7 @@ resource "aws_iam_policy" "LambdaInvokeScopedAccessPolicy" { ) } -resource "aws_iam_policy" "XRayAccessPolicy" { +resource "aws_iam_policy" "xray_access_policy" { description = "Allow AWS Step Functions to call X-Ray daemon on your behalf" path = "/${var.environment}/" policy = jsonencode( diff --git a/terraform/variables.tf b/terraform/variables.tf index c8f984f..a8edfba 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,10 +1,3 @@ -variable "tags" { - description = "Default tags" - type = map(string) - - default = {} -} - variable "project" { description = "Project name used to build fully qualified tags and resource's names" type = string @@ -208,12 +201,6 @@ variable "push_report_image_uri" { type = string } -variable "lambda_alerts_sns_topic_arn" { - description = "SNS topic used to to publish cloudwatch alerts" - type = string - default = null -} - variable "lambda_private_subnet_ids" { description = "List of private subnets assigned to lambda" type = list(string) diff --git a/terraform/versions.tf b/terraform/versions.tf index 34239b5..0fd1610 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,12 +2,16 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.64.0" + version = "~> 4.64.0" } local = { source = "hashicorp/local" version = "~> 2.2.3" } + null = { + source = "hashicorp/null" + version = "~> 3.2.1" + } } required_version = "~> 1.1" } diff --git a/tests/integration_tests/README.md b/tests/integration_tests/README.md index 2e6abb5..03e4a7a 100644 --- a/tests/integration_tests/README.md +++ b/tests/integration_tests/README.md @@ -4,4 +4,4 @@ Requirements: Run tests: 1. Setup local env for test env by running `make run-localstack deploy-qa-infra` from project root dir -2. Run tests by running `make run-integration-tests` from from project root dir \ No newline at end of file +2. Run tests by running `make run-integration-tests` from from project root dir diff --git a/tests/integration_tests/data_test/Dockerfile b/tests/integration_tests/data_test/Dockerfile index 2ea253e..5825fe9 100644 --- a/tests/integration_tests/data_test/Dockerfile +++ b/tests/integration_tests/data_test/Dockerfile @@ -9,4 +9,4 @@ ENV AWS_SECRET_ACCESS_KEY test ENV AWS_DEFAULT_REGION us-east-1 COPY test_data ./test_data COPY ./*.py ./ -ENTRYPOINT ["pytest", "-qvs", "test_data_test.py"] \ No newline at end of file +ENTRYPOINT ["pytest", "-qvs", "test_data_test.py"] diff --git a/tests/integration_tests/data_test/test_data/titanic.json b/tests/integration_tests/data_test/test_data/titanic.json index f21fd08..6443aab 100644 --- a/tests/integration_tests/data_test/test_data/titanic.json +++ b/tests/integration_tests/data_test/test_data/titanic.json @@ -415,4 +415,4 @@ {"PassengerId":1306,"Survived":1,"Pclass":1,"Name":"Oliva y Ocana, Dona. Fermina","Sex":"female","Age":39.0,"SibSp":0,"Parch":0,"Ticket":"PC 17758","Fare":108.9,"Cabin":"C105","Embarked":"C"} {"PassengerId":1307,"Survived":0,"Pclass":3,"Name":"Saether, Mr. Simon Sivertsen","Sex":"male","Age":38.5,"SibSp":0,"Parch":0,"Ticket":"SOTON\/O.Q. 3101262","Fare":7.25,"Cabin":"","Embarked":"S"} {"PassengerId":1308,"Survived":0,"Pclass":3,"Name":"Ware, Mr. Frederick","Sex":"male","Age":null,"SibSp":0,"Parch":0,"Ticket":"359309","Fare":8.05,"Cabin":"","Embarked":"S"} -{"PassengerId":1309,"Survived":0,"Pclass":3,"Name":"Peter, Master. Michael J","Sex":"male","Age":null,"SibSp":1,"Parch":1,"Ticket":"2668","Fare":22.3583,"Cabin":"","Embarked":"C"} \ No newline at end of file +{"PassengerId":1309,"Survived":0,"Pclass":3,"Name":"Peter, Master. Michael J","Sex":"male","Age":null,"SibSp":1,"Parch":1,"Ticket":"2668","Fare":22.3583,"Cabin":"","Embarked":"C"} diff --git a/tests/integration_tests/data_test/test_data/titanic_nested.json b/tests/integration_tests/data_test/test_data/titanic_nested.json index 86bdaf6..9a53118 100644 --- a/tests/integration_tests/data_test/test_data/titanic_nested.json +++ b/tests/integration_tests/data_test/test_data/titanic_nested.json @@ -71,4 +71,4 @@ "revisionId": 7652, "timestamp": 1677742570738 } -] \ No newline at end of file +] diff --git a/tests/integration_tests/infra/README.md b/tests/integration_tests/infra/README.md index 5127af1..a7dea6d 100644 --- a/tests/integration_tests/infra/README.md +++ b/tests/integration_tests/infra/README.md @@ -2,4 +2,4 @@ 1. Navigate to `./tests/integration_tests/infra` 2. Run localstack `docker run --rm -it -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:1.3.1` 3. Run `terraform init` -4. Setup S3 environment `terraform apply -auto-approve` \ No newline at end of file +4. Setup S3 environment `terraform apply -auto-approve` diff --git a/tests/integration_tests/infra/configs/manifest.json b/tests/integration_tests/infra/configs/manifest.json index 90e9d81..f37030c 100644 --- a/tests/integration_tests/infra/configs/manifest.json +++ b/tests/integration_tests/infra/configs/manifest.json @@ -9,4 +9,4 @@ "globalUploadSettings":{ "format":"JSON" } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/mapping.json b/tests/integration_tests/infra/configs/mapping.json index 51f7a20..84cbeb7 100644 --- a/tests/integration_tests/infra/configs/mapping.json +++ b/tests/integration_tests/infra/configs/mapping.json @@ -3,4 +3,4 @@ "CALL ID": "call id", "new_col_added0": "year_month" } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/pipeline.json b/tests/integration_tests/infra/configs/pipeline.json index 934e5bf..5ae6cba 100644 --- a/tests/integration_tests/infra/configs/pipeline.json +++ b/tests/integration_tests/infra/configs/pipeline.json @@ -9,4 +9,4 @@ "only_failed": true } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/pks.json b/tests/integration_tests/infra/configs/pks.json index e192edf..01c8fb0 100644 --- a/tests/integration_tests/infra/configs/pks.json +++ b/tests/integration_tests/infra/configs/pks.json @@ -6,4 +6,4 @@ "RECORD_NUMBER" ] } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/sort_keys.json b/tests/integration_tests/infra/configs/sort_keys.json index 3beab44..347368f 100644 --- a/tests/integration_tests/infra/configs/sort_keys.json +++ b/tests/integration_tests/infra/configs/sort_keys.json @@ -4,4 +4,4 @@ "update_dt" ] } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/test_coverage.json b/tests/integration_tests/infra/configs/test_coverage.json index 294cd41..7a76a66 100644 --- a/tests/integration_tests/infra/configs/test_coverage.json +++ b/tests/integration_tests/infra/configs/test_coverage.json @@ -2,4 +2,4 @@ "sample": { "complexSuite": true } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/outputs.tf b/tests/integration_tests/infra/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/infra/variables.tf b/tests/integration_tests/infra/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration_tests/infra/versions.tf b/tests/integration_tests/infra/versions.tf index fc5a627..9eb52cd 100644 --- a/tests/integration_tests/infra/versions.tf +++ b/tests/integration_tests/infra/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.67.0" + version = "~> 4.64.0" } } }