From 73be1a39b89bce47fc68c3343b868fd3e00be226 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 13:31:47 +0300 Subject: [PATCH 01/11] Add terraform pre-commit hook --- .github/workflows/pre-commit.yml | 83 ++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 29 +++++++++++ 2 files changed, 112 insertions(+) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..cb82671 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,83 @@ +name: Pre-Commit + +on: + pull_request: + branches: + - main + - master + +env: + TERRAFORM_DOCS_VERSION: v0.16.0 + TFLINT_VERSION: v0.44.1 + +jobs: + collectInputs: + name: Collect workflow inputs + runs-on: ubuntu-latest + outputs: + directories: ${{ steps.dirs.outputs.directories }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Get root directories + id: dirs + uses: clowdhaus/terraform-composite-actions/directories@v1.8.3 + + preCommitMinVersions: + name: Min TF pre-commit + needs: collectInputs + runs-on: ubuntu-latest + strategy: + matrix: + directory: ${{ fromJson(needs.collectInputs.outputs.directories) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.2.4 + with: + directory: ${{ matrix.directory }} + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory != '.' }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + args: 'terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*' + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.minVersion }} + # Run only validate pre-commit check on min version supported + if: ${{ matrix.directory == '.' }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.minVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + args: 'terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)' + + preCommitMaxVersion: + name: Max TF pre-commit + runs-on: ubuntu-latest + needs: collectInputs + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Terraform min/max versions + id: minMax + uses: clowdhaus/terraform-min-max@v1.2.4 + + - name: Pre-commit Terraform ${{ steps.minMax.outputs.maxVersion }} + uses: clowdhaus/terraform-composite-actions/pre-commit@v1.8.3 + with: + terraform-version: ${{ steps.minMax.outputs.maxVersion }} + tflint-version: ${{ env.TFLINT_VERSION }} + terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} + install-hcledit: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4395e67 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.79.1 + hooks: + - id: terraform_fmt + - id: terraform_validate + - id: terraform_docs + args: + - '--args=--lockfile=false' + - id: terraform_tflint + args: + - '--args=--only=terraform_deprecated_interpolation' + - '--args=--only=terraform_deprecated_index' + - '--args=--only=terraform_unused_declarations' + - '--args=--only=terraform_comment_syntax' + - '--args=--only=terraform_documented_outputs' + - '--args=--only=terraform_documented_variables' + - '--args=--only=terraform_typed_variables' + - '--args=--only=terraform_module_pinned_source' + - '--args=--only=terraform_naming_convention' + - '--args=--only=terraform_required_version' + - '--args=--only=terraform_required_providers' + - '--args=--only=terraform_standard_module_structure' + - '--args=--only=terraform_workspace_remote' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-merge-conflict + - id: end-of-file-fixer From 33a664b6f6d760756cca2668c9bdad0dd2dceca3 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 13:50:38 +0300 Subject: [PATCH 02/11] Fix tflint errors --- .gitignore | 2 +- CONTRIBUTING.md | 2 +- LICENSE.md | 2 +- configs/manifest.json | 2 +- configs/mapping.json | 2 +- configs/pipeline.json | 2 +- configs/pks.json | 2 +- configs/sort_keys.json | 2 +- configs/test_coverage.json | 2 +- examples/docker_basic/modules/vpc/main.tf | 2 +- examples/docker_basic/variables.tf | 2 +- examples/docker_basic/versions.tf | 2 +- functions/allure_report/requirements.txt | 2 +- functions/data_test/Dockerfile | 2 +- functions/data_test/great_expectations/.gitignore | 2 +- .../great_expectations/expectations/.ge_store_backend_id | 2 +- functions/report_push/requirements.txt | 2 +- terraform/README.md | 2 +- terraform/dynamodb.tf | 2 +- terraform/modules/alerting/main.tf | 1 - terraform/modules/athena-connector/iam.tf | 2 +- terraform/modules/athena-connector/main.tf | 2 +- terraform/modules/athena-connector/variables.tf | 2 +- terraform/modules/s3-configs/outputs.tf | 2 +- terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh | 2 +- terraform/outputs.tf | 2 +- tests/integration_tests/README.md | 2 +- tests/integration_tests/data_test/Dockerfile | 2 +- tests/integration_tests/data_test/test_data/titanic.json | 2 +- tests/integration_tests/data_test/test_data/titanic_nested.json | 2 +- tests/integration_tests/infra/README.md | 2 +- tests/integration_tests/infra/configs/manifest.json | 2 +- tests/integration_tests/infra/configs/mapping.json | 2 +- tests/integration_tests/infra/configs/pipeline.json | 2 +- tests/integration_tests/infra/configs/pks.json | 2 +- tests/integration_tests/infra/configs/sort_keys.json | 2 +- tests/integration_tests/infra/configs/test_coverage.json | 2 +- 37 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index f5a2b23..6a4eca2 100644 --- a/.gitignore +++ b/.gitignore @@ -168,4 +168,4 @@ tmp examples/*/plan *.auto.tfvars -.envrc \ No newline at end of file +.envrc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60400a1..afb805b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,4 +46,4 @@ By participating in this project, you agree to abide by our [Code of Conduct](ht - If you have any questions or need assistance, feel free to open an issue or reach out to the maintainers on GitHub. - We welcome any suggestions for improvements or new features. -We look forward to working with you and improving Data-Quality-Gate together! \ No newline at end of file +We look forward to working with you and improving Data-Quality-Gate together! diff --git a/LICENSE.md b/LICENSE.md index f49a4e1..261eeb9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -198,4 +198,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/configs/manifest.json b/configs/manifest.json index 90e9d81..f37030c 100644 --- a/configs/manifest.json +++ b/configs/manifest.json @@ -9,4 +9,4 @@ "globalUploadSettings":{ "format":"JSON" } -} \ No newline at end of file +} diff --git a/configs/mapping.json b/configs/mapping.json index 51f7a20..84cbeb7 100644 --- a/configs/mapping.json +++ b/configs/mapping.json @@ -3,4 +3,4 @@ "CALL ID": "call id", "new_col_added0": "year_month" } -} \ No newline at end of file +} diff --git a/configs/pipeline.json b/configs/pipeline.json index 934e5bf..5ae6cba 100644 --- a/configs/pipeline.json +++ b/configs/pipeline.json @@ -9,4 +9,4 @@ "only_failed": true } -} \ No newline at end of file +} diff --git a/configs/pks.json b/configs/pks.json index e192edf..01c8fb0 100644 --- a/configs/pks.json +++ b/configs/pks.json @@ -6,4 +6,4 @@ "RECORD_NUMBER" ] } -} \ No newline at end of file +} diff --git a/configs/sort_keys.json b/configs/sort_keys.json index 3beab44..347368f 100644 --- a/configs/sort_keys.json +++ b/configs/sort_keys.json @@ -4,4 +4,4 @@ "update_dt" ] } -} \ No newline at end of file +} diff --git a/configs/test_coverage.json b/configs/test_coverage.json index 294cd41..7a76a66 100644 --- a/configs/test_coverage.json +++ b/configs/test_coverage.json @@ -2,4 +2,4 @@ "sample": { "complexSuite": true } -} \ No newline at end of file +} diff --git a/examples/docker_basic/modules/vpc/main.tf b/examples/docker_basic/modules/vpc/main.tf index 273d6cb..dfd0233 100644 --- a/examples/docker_basic/modules/vpc/main.tf +++ b/examples/docker_basic/modules/vpc/main.tf @@ -113,4 +113,4 @@ resource "aws_security_group" "lambda_security_group" { cidr_blocks = ["0.0.0.0/0"] description = "Allow out to tcp through vpc endpoint" } -} \ No newline at end of file +} diff --git a/examples/docker_basic/variables.tf b/examples/docker_basic/variables.tf index bb5f6a1..fa31ff3 100644 --- a/examples/docker_basic/variables.tf +++ b/examples/docker_basic/variables.tf @@ -1,2 +1,2 @@ variable "slack_webhook_url" { type = string } -variable "slack_channel" { type = string } \ No newline at end of file +variable "slack_channel" { type = string } diff --git a/examples/docker_basic/versions.tf b/examples/docker_basic/versions.tf index ca331be..fe6e748 100644 --- a/examples/docker_basic/versions.tf +++ b/examples/docker_basic/versions.tf @@ -15,4 +15,4 @@ terraform { version = "2.18.0" } } -} \ No newline at end of file +} diff --git a/functions/allure_report/requirements.txt b/functions/allure_report/requirements.txt index 1cae9ae..f3ed878 100644 --- a/functions/allure_report/requirements.txt +++ b/functions/allure_report/requirements.txt @@ -10,4 +10,4 @@ boto3==1.17.1 boto==2.49.0 pyarrow==3.0.0 fastparquet==0.8.1 -awswrangler==2.12.1 \ No newline at end of file +awswrangler==2.12.1 diff --git a/functions/data_test/Dockerfile b/functions/data_test/Dockerfile index c66c7ad..6b21d86 100755 --- a/functions/data_test/Dockerfile +++ b/functions/data_test/Dockerfile @@ -5,4 +5,4 @@ COPY ./*.py ./ COPY great_expectations ./great_expectations RUN rm -f /etc/timezone RUN python3 -c "import tzlocal; tzlocal.get_localzone()" -CMD ["data_test.handler"] \ No newline at end of file +CMD ["data_test.handler"] diff --git a/functions/data_test/great_expectations/.gitignore b/functions/data_test/great_expectations/.gitignore index 241ec61..e9ef387 100755 --- a/functions/data_test/great_expectations/.gitignore +++ b/functions/data_test/great_expectations/.gitignore @@ -1,2 +1,2 @@ uncommitted/ -allure-report \ No newline at end of file +allure-report diff --git a/functions/data_test/great_expectations/expectations/.ge_store_backend_id b/functions/data_test/great_expectations/expectations/.ge_store_backend_id index 025ba49..2d7c45d 100755 --- a/functions/data_test/great_expectations/expectations/.ge_store_backend_id +++ b/functions/data_test/great_expectations/expectations/.ge_store_backend_id @@ -1 +1 @@ -store_backend_id = f6a15c13-c249-416b-8beb-2e540a245773 \ No newline at end of file +store_backend_id = f6a15c13-c249-416b-8beb-2e540a245773 diff --git a/functions/report_push/requirements.txt b/functions/report_push/requirements.txt index 1b127f8..7ac3ad7 100644 --- a/functions/report_push/requirements.txt +++ b/functions/report_push/requirements.txt @@ -4,4 +4,4 @@ importlib-metadata==6.0.0 s3fs==0.4.2 python-dateutil==2.8.2 awswrangler==2.19.0 -jira==3.2.0 \ No newline at end of file +jira==3.2.0 diff --git a/terraform/README.md b/terraform/README.md index ad1ec4e..1b123ed 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -120,4 +120,4 @@ | [lambda\_report\_push\_arn](#output\_lambda\_report\_push\_arn) | n/a | | [report\_push\_role\_arn](#output\_report\_push\_role\_arn) | n/a | | [step\_function\_arn](#output\_step\_function\_arn) | n/a | - \ No newline at end of file + diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf index 5493fb4..039c08c 100644 --- a/terraform/dynamodb.tf +++ b/terraform/dynamodb.tf @@ -73,4 +73,4 @@ resource "aws_appautoscaling_policy" "data_qa_report_table_write_policy" { scale_out_cooldown = lookup(var.dynamodb_autoscaling_write, "scale_out_cooldown", var.dynamodb_autoscaling_defaults["scale_out_cooldown"]) target_value = lookup(var.dynamodb_autoscaling_write, "target_value", var.dynamodb_autoscaling_defaults["target_value"]) } -} \ No newline at end of file +} diff --git a/terraform/modules/alerting/main.tf b/terraform/modules/alerting/main.tf index c6f93d5..e1f1417 100644 --- a/terraform/modules/alerting/main.tf +++ b/terraform/modules/alerting/main.tf @@ -51,4 +51,3 @@ data "aws_sfn_state_machine" "step_functions" { for_each = var.step_functions_to_monitor name = each.key } - diff --git a/terraform/modules/athena-connector/iam.tf b/terraform/modules/athena-connector/iam.tf index feed739..a0d85a8 100644 --- a/terraform/modules/athena-connector/iam.tf +++ b/terraform/modules/athena-connector/iam.tf @@ -81,4 +81,4 @@ resource "aws_iam_policy" "athena_connector_lambda_policy" { resource "aws_iam_role_policy_attachment" "athena_connector_basic_policy" { role = aws_iam_role.athena_connector_lambda_role.name policy_arn = aws_iam_policy.athena_connector_lambda_policy.arn -} \ No newline at end of file +} diff --git a/terraform/modules/athena-connector/main.tf b/terraform/modules/athena-connector/main.tf index b589307..61670a3 100644 --- a/terraform/modules/athena-connector/main.tf +++ b/terraform/modules/athena-connector/main.tf @@ -44,4 +44,4 @@ resource "null_resource" "delete_athena_dynamodb_connector" { } depends_on = [null_resource.athena_dynamodb_connector] -} \ No newline at end of file +} diff --git a/terraform/modules/athena-connector/variables.tf b/terraform/modules/athena-connector/variables.tf index f02b229..fdd33b5 100644 --- a/terraform/modules/athena-connector/variables.tf +++ b/terraform/modules/athena-connector/variables.tf @@ -24,4 +24,4 @@ variable "vpc_security_group_ids" { description = "List of security group assigned to lambda. If null value, default subnet and vpc will be used" type = list(string) default = null -} \ No newline at end of file +} diff --git a/terraform/modules/s3-configs/outputs.tf b/terraform/modules/s3-configs/outputs.tf index e6caf19..d75e500 100644 --- a/terraform/modules/s3-configs/outputs.tf +++ b/terraform/modules/s3-configs/outputs.tf @@ -1,3 +1,3 @@ output "bucket_name" { value = aws_s3_bucket.settings_bucket.bucket -} \ No newline at end of file +} diff --git a/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh b/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh index 82bc853..29743dd 100644 --- a/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh +++ b/terraform/modules/s3-gateway/user_data/nginx_install.tmpl.sh @@ -19,4 +19,4 @@ export PROXY_CACHE_VALID_FORBIDDEN=30s export CORS_ENABLED=false export S3_DEBUG=false -bash standalone_ubuntu_oss_install.sh \ No newline at end of file +bash standalone_ubuntu_oss_install.sh diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 239ae85..8d5e8a9 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -29,4 +29,4 @@ output "report_push_role_arn" { output "bucket" { description = "Data quality gate bucket with settings and generated tests" value = module.s3_bucket.bucket_name -} \ No newline at end of file +} diff --git a/tests/integration_tests/README.md b/tests/integration_tests/README.md index 2e6abb5..03e4a7a 100644 --- a/tests/integration_tests/README.md +++ b/tests/integration_tests/README.md @@ -4,4 +4,4 @@ Requirements: Run tests: 1. Setup local env for test env by running `make run-localstack deploy-qa-infra` from project root dir -2. Run tests by running `make run-integration-tests` from from project root dir \ No newline at end of file +2. Run tests by running `make run-integration-tests` from from project root dir diff --git a/tests/integration_tests/data_test/Dockerfile b/tests/integration_tests/data_test/Dockerfile index 2ea253e..5825fe9 100644 --- a/tests/integration_tests/data_test/Dockerfile +++ b/tests/integration_tests/data_test/Dockerfile @@ -9,4 +9,4 @@ ENV AWS_SECRET_ACCESS_KEY test ENV AWS_DEFAULT_REGION us-east-1 COPY test_data ./test_data COPY ./*.py ./ -ENTRYPOINT ["pytest", "-qvs", "test_data_test.py"] \ No newline at end of file +ENTRYPOINT ["pytest", "-qvs", "test_data_test.py"] diff --git a/tests/integration_tests/data_test/test_data/titanic.json b/tests/integration_tests/data_test/test_data/titanic.json index f21fd08..6443aab 100644 --- a/tests/integration_tests/data_test/test_data/titanic.json +++ b/tests/integration_tests/data_test/test_data/titanic.json @@ -415,4 +415,4 @@ {"PassengerId":1306,"Survived":1,"Pclass":1,"Name":"Oliva y Ocana, Dona. Fermina","Sex":"female","Age":39.0,"SibSp":0,"Parch":0,"Ticket":"PC 17758","Fare":108.9,"Cabin":"C105","Embarked":"C"} {"PassengerId":1307,"Survived":0,"Pclass":3,"Name":"Saether, Mr. Simon Sivertsen","Sex":"male","Age":38.5,"SibSp":0,"Parch":0,"Ticket":"SOTON\/O.Q. 3101262","Fare":7.25,"Cabin":"","Embarked":"S"} {"PassengerId":1308,"Survived":0,"Pclass":3,"Name":"Ware, Mr. Frederick","Sex":"male","Age":null,"SibSp":0,"Parch":0,"Ticket":"359309","Fare":8.05,"Cabin":"","Embarked":"S"} -{"PassengerId":1309,"Survived":0,"Pclass":3,"Name":"Peter, Master. Michael J","Sex":"male","Age":null,"SibSp":1,"Parch":1,"Ticket":"2668","Fare":22.3583,"Cabin":"","Embarked":"C"} \ No newline at end of file +{"PassengerId":1309,"Survived":0,"Pclass":3,"Name":"Peter, Master. Michael J","Sex":"male","Age":null,"SibSp":1,"Parch":1,"Ticket":"2668","Fare":22.3583,"Cabin":"","Embarked":"C"} diff --git a/tests/integration_tests/data_test/test_data/titanic_nested.json b/tests/integration_tests/data_test/test_data/titanic_nested.json index 86bdaf6..9a53118 100644 --- a/tests/integration_tests/data_test/test_data/titanic_nested.json +++ b/tests/integration_tests/data_test/test_data/titanic_nested.json @@ -71,4 +71,4 @@ "revisionId": 7652, "timestamp": 1677742570738 } -] \ No newline at end of file +] diff --git a/tests/integration_tests/infra/README.md b/tests/integration_tests/infra/README.md index 5127af1..a7dea6d 100644 --- a/tests/integration_tests/infra/README.md +++ b/tests/integration_tests/infra/README.md @@ -2,4 +2,4 @@ 1. Navigate to `./tests/integration_tests/infra` 2. Run localstack `docker run --rm -it -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:1.3.1` 3. Run `terraform init` -4. Setup S3 environment `terraform apply -auto-approve` \ No newline at end of file +4. Setup S3 environment `terraform apply -auto-approve` diff --git a/tests/integration_tests/infra/configs/manifest.json b/tests/integration_tests/infra/configs/manifest.json index 90e9d81..f37030c 100644 --- a/tests/integration_tests/infra/configs/manifest.json +++ b/tests/integration_tests/infra/configs/manifest.json @@ -9,4 +9,4 @@ "globalUploadSettings":{ "format":"JSON" } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/mapping.json b/tests/integration_tests/infra/configs/mapping.json index 51f7a20..84cbeb7 100644 --- a/tests/integration_tests/infra/configs/mapping.json +++ b/tests/integration_tests/infra/configs/mapping.json @@ -3,4 +3,4 @@ "CALL ID": "call id", "new_col_added0": "year_month" } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/pipeline.json b/tests/integration_tests/infra/configs/pipeline.json index 934e5bf..5ae6cba 100644 --- a/tests/integration_tests/infra/configs/pipeline.json +++ b/tests/integration_tests/infra/configs/pipeline.json @@ -9,4 +9,4 @@ "only_failed": true } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/pks.json b/tests/integration_tests/infra/configs/pks.json index e192edf..01c8fb0 100644 --- a/tests/integration_tests/infra/configs/pks.json +++ b/tests/integration_tests/infra/configs/pks.json @@ -6,4 +6,4 @@ "RECORD_NUMBER" ] } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/sort_keys.json b/tests/integration_tests/infra/configs/sort_keys.json index 3beab44..347368f 100644 --- a/tests/integration_tests/infra/configs/sort_keys.json +++ b/tests/integration_tests/infra/configs/sort_keys.json @@ -4,4 +4,4 @@ "update_dt" ] } -} \ No newline at end of file +} diff --git a/tests/integration_tests/infra/configs/test_coverage.json b/tests/integration_tests/infra/configs/test_coverage.json index 294cd41..7a76a66 100644 --- a/tests/integration_tests/infra/configs/test_coverage.json +++ b/tests/integration_tests/infra/configs/test_coverage.json @@ -2,4 +2,4 @@ "sample": { "complexSuite": true } -} \ No newline at end of file +} From 3d5a9a24db16f1a8e33d5216e3db344c15205197 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 13:56:52 +0300 Subject: [PATCH 03/11] Synchronize aws provider versions --- examples/docker_basic/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/docker_basic/versions.tf b/examples/docker_basic/versions.tf index fe6e748..5a34d35 100644 --- a/examples/docker_basic/versions.tf +++ b/examples/docker_basic/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.67.0" + version = "= 4.64.0" } local = { source = "hashicorp/local" From 0cf986f1199023360dc4ee05a5c9dc9f84c68c3e Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 14:12:24 +0300 Subject: [PATCH 04/11] Cleanup basic example --- examples/{docker_basic => basic}/README.md | 0 examples/basic/main.tf | 50 ++++++++ examples/{docker_basic => basic}/variables.tf | 0 examples/basic/versions.tf | 10 ++ examples/docker_basic/docker.tf | 60 --------- examples/docker_basic/main.tf | 69 ----------- examples/docker_basic/modules/vpc/main.tf | 116 ------------------ examples/docker_basic/modules/vpc/outputs.tf | 15 --- .../docker_basic/modules/vpc/variables.tf | 24 ---- examples/docker_basic/versions.tf | 18 --- 10 files changed, 60 insertions(+), 302 deletions(-) rename examples/{docker_basic => basic}/README.md (100%) create mode 100644 examples/basic/main.tf rename examples/{docker_basic => basic}/variables.tf (100%) create mode 100644 examples/basic/versions.tf delete mode 100644 examples/docker_basic/docker.tf delete mode 100644 examples/docker_basic/main.tf delete mode 100644 examples/docker_basic/modules/vpc/main.tf delete mode 100644 examples/docker_basic/modules/vpc/outputs.tf delete mode 100644 examples/docker_basic/modules/vpc/variables.tf delete mode 100644 examples/docker_basic/versions.tf diff --git a/examples/docker_basic/README.md b/examples/basic/README.md similarity index 100% rename from examples/docker_basic/README.md rename to examples/basic/README.md diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..3830c67 --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,50 @@ +provider "aws" { + region = "eu-west-2" + + default_tags { + tags = { + Terraform = true + } + } +} + +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} +data "aws_ecr_authorization_token" "token" {} +data "aws_availability_zones" "available" { + state = "available" +} + +module "data_qa" { + source = "../../terraform" + + data_test_storage_bucket_name = "dqg-settings-dev" + s3_source_data_bucket = "data-bucket-name" + environment = "demo" + project = "provectus" + + allure_report_image_uri = "..." + data_test_image_uri = "..." + push_report_image_uri = "..." + + data_reports_notification_settings = { + channel = "DataReportSlackChannelName" + webhook_url = "https://hooks.slack.com/services/........" + } + + lambda_private_subnet_ids = ["private_subnet_id"] + lambda_security_group_ids = ["security_group_id"] + + reports_vpc_id = "some_vpc_id" + reports_subnet_id = "subnet_id" + reports_whitelist_ips = ["0.0.0.0/0"] + + test_coverage_path = "../../configs/test_coverage.json" + pipeline_config_path = "../../configs/pipeline.json" + pks_path = "../../configs/pks.json" + sort_keys_path = "../../configs/sort_keys.json" + mapping_path = "../../configs/mapping.json" + manifest_path = "../../configs/manifest.json" + great_expectation_path = "../../templates/great_expectations.yml" + expectations_store = "../../expectations_store" +} diff --git a/examples/docker_basic/variables.tf b/examples/basic/variables.tf similarity index 100% rename from examples/docker_basic/variables.tf rename to examples/basic/variables.tf diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf new file mode 100644 index 0000000..f442154 --- /dev/null +++ b/examples/basic/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "= 4.64.0" + } + } +} diff --git a/examples/docker_basic/docker.tf b/examples/docker_basic/docker.tf deleted file mode 100644 index 7b1e53b..0000000 --- a/examples/docker_basic/docker.tf +++ /dev/null @@ -1,60 +0,0 @@ -resource "random_uuid" "allure_report" { - keepers = { - for filename in setunion( - fileset("../../functions/allure_report/", "*.py"), - fileset("../../functions/allure_report/", "requirements.txt"), - fileset("../../functions/allure_report/", "Dockerfile"), - fileset("../../functions/allure_report/", "generate_report.sh") - ) : - filename => filemd5("../../functions/allure_report/${filename}") - } -} - -resource "random_uuid" "data_test" { - keepers = { - for filename in setunion( - fileset("../../functions/data_test/", "*.py"), - fileset("../../functions/data_test/", "requirements.txt"), - fileset("../../functions/data_test/", "Dockerfile") - ) : - filename => filemd5("../../functions/data_test/${filename}") - } -} - -resource "random_uuid" "push_report" { - keepers = { - for filename in setunion( - fileset("../../functions/report_push/", "*.py"), - fileset("../../functions/report_push/", "requirements.txt"), - fileset("../../functions/report_push/", "Dockerfile") - ) : - filename => filemd5("../../functions/report_push/${filename}") - } -} - -module "docker_image_push_report" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-push-report" - image_tag = random_uuid.push_report.result - source_path = "../../functions/report_push" -} - -module "docker_image_data_test" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-data-test" - image_tag = random_uuid.data_test.result - source_path = "../../functions/data_test" -} - -module "docker_image_allure_report" { - source = "terraform-aws-modules/lambda/aws//modules/docker-build" - version = "3.3.1" - create_ecr_repo = true - ecr_repo = "dqg-allure-report" - image_tag = random_uuid.allure_report.result - source_path = "../../functions/allure_report" -} diff --git a/examples/docker_basic/main.tf b/examples/docker_basic/main.tf deleted file mode 100644 index c71b340..0000000 --- a/examples/docker_basic/main.tf +++ /dev/null @@ -1,69 +0,0 @@ -provider "aws" { - region = "eu-west-2" - - default_tags { - tags = { - Terraform = true - } - } -} - -provider "docker" { - registry_auth { - address = format("%v.dkr.ecr.%v.amazonaws.com", data.aws_caller_identity.current.account_id, data.aws_region.current.name) - username = data.aws_ecr_authorization_token.token.user_name - password = data.aws_ecr_authorization_token.token.password - } -} - -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} -data "aws_ecr_authorization_token" "token" {} -data "aws_availability_zones" "available" { - state = "available" -} - -module "vpc" { - source = "./modules/vpc" - - resource_name_prefix = "provectus-infra" - - cidr = "172.21.0.0/16" - private_subnets_cidr = ["172.21.16.0/20"] - public_subnets_cidr = ["172.21.32.0/20"] - azs = data.aws_availability_zones.available.zone_ids -} - -module "data_qa" { - source = "../../terraform" - - data_test_storage_bucket_name = "dqg-settings-dev" - s3_source_data_bucket = "data-bucket-name" - environment = "demo" - project = "provectus" - - allure_report_image_uri = module.docker_image_allure_report.image_uri - data_test_image_uri = module.docker_image_data_test.image_uri - push_report_image_uri = module.docker_image_push_report.image_uri - - data_reports_notification_settings = { - channel = "DataQASlackChannel" - webhook_url = "https://hooks.slack.com/services/........" - } - - lambda_private_subnet_ids = module.vpc.private_subnet_ids - lambda_security_group_ids = module.vpc.security_group_ids - - reports_vpc_id = module.vpc.vpc_id - reports_subnet_id = module.vpc.public_subnet_ids[0] - reports_whitelist_ips = ["0.0.0.0/0"] # Available from everywhere - - test_coverage_path = "../../configs/test_coverage.json" - pipeline_config_path = "../../configs/pipeline.json" - pks_path = "../../configs/pks.json" - sort_keys_path = "../../configs/sort_keys.json" - mapping_path = "../../configs/mapping.json" - manifest_path = "../../configs/manifest.json" - great_expectation_path = "../../templates/great_expectations.yml" - expectations_store = "../../expectations_store" -} diff --git a/examples/docker_basic/modules/vpc/main.tf b/examples/docker_basic/modules/vpc/main.tf deleted file mode 100644 index dfd0233..0000000 --- a/examples/docker_basic/modules/vpc/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -locals { - resource_name_prefix = var.resource_name_prefix - private_route_table_ids = module.vpc.private_route_table_ids -} - -data "aws_region" "current" {} -data "aws_vpc_endpoint_service" "dynamodb" { service = "dynamodb" } -data "aws_vpc_endpoint_service" "secretsmanager" { service = "secretsmanager" } -data "aws_vpc_endpoint_service" "monitoring" { service = "monitoring" } -data "aws_vpc_endpoint_service" "sns" { service = "sns" } - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - enable_dns_hostnames = true - - name = "${local.resource_name_prefix}-vpc" - cidr = var.cidr - - azs = var.azs - private_subnets = var.private_subnets_cidr - public_subnets = var.public_subnets_cidr - - map_public_ip_on_launch = true -} - -resource "aws_vpc_endpoint" "s3" { - vpc_id = module.vpc.vpc_id - service_name = "com.amazonaws.${data.aws_region.current.name}.s3" -} - -resource "aws_vpc_endpoint_route_table_association" "s3" { - count = length(local.private_route_table_ids) - - vpc_endpoint_id = aws_vpc_endpoint.s3.id - route_table_id = local.private_route_table_ids[count.index] -} - -resource "aws_vpc_endpoint" "dynamodb" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.dynamodb.service_name -} - -resource "aws_vpc_endpoint_route_table_association" "private_dynamodb" { - count = length(local.private_route_table_ids) - - vpc_endpoint_id = aws_vpc_endpoint.dynamodb.id - route_table_id = local.private_route_table_ids[count.index] -} - -resource "aws_vpc_endpoint" "secretsmanager" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.secretsmanager.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_vpc_endpoint" "monitoring" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.monitoring.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_vpc_endpoint" "sns" { - vpc_id = module.vpc.vpc_id - service_name = data.aws_vpc_endpoint_service.sns.service_name - vpc_endpoint_type = "Interface" - - security_group_ids = [aws_security_group.inbound_ssl_tcp.id, aws_security_group.lambda_security_group.id] - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true -} - -resource "aws_security_group" "inbound_ssl_tcp" { - name = "${local.resource_name_prefix}-endpoint-443" - vpc_id = module.vpc.vpc_id -} - -resource "aws_security_group_rule" "secretsmanager_sg_rule_sg" { - type = "ingress" - security_group_id = aws_security_group.inbound_ssl_tcp.id - from_port = 443 - to_port = 443 - protocol = "tcp" - source_security_group_id = aws_security_group.lambda_security_group.id - description = "Allow ingress from security group" -} - -resource "aws_security_group" "lambda_security_group" { - name = "${local.resource_name_prefix}-service-endpoints" - vpc_id = module.vpc.vpc_id - - egress { - from_port = 5439 - to_port = 5439 - protocol = "tcp" - cidr_blocks = [module.vpc.vpc_cidr_block] - description = "Allow out to redshift through vpc endpoint" - } - - egress { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - description = "Allow out to tcp through vpc endpoint" - } -} diff --git a/examples/docker_basic/modules/vpc/outputs.tf b/examples/docker_basic/modules/vpc/outputs.tf deleted file mode 100644 index 2821927..0000000 --- a/examples/docker_basic/modules/vpc/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "vpc_id" { - value = module.vpc.vpc_id -} - -output "security_group_ids" { - value = [aws_security_group.lambda_security_group.id] -} - -output "private_subnet_ids" { - value = module.vpc.private_subnets -} - -output "public_subnet_ids" { - value = module.vpc.public_subnets -} diff --git a/examples/docker_basic/modules/vpc/variables.tf b/examples/docker_basic/modules/vpc/variables.tf deleted file mode 100644 index 2fc44c5..0000000 --- a/examples/docker_basic/modules/vpc/variables.tf +++ /dev/null @@ -1,24 +0,0 @@ -variable "cidr" { - description = "VPC cidr block" - type = string -} - -variable "private_subnets_cidr" { - description = "List of private subnets cidr" - type = list(string) -} - -variable "public_subnets_cidr" { - description = "List of private subnets cidr" - type = list(string) -} - -variable "azs" { - description = "List of available zones in selected region" - type = list(string) -} - -variable "resource_name_prefix" { - description = "Resource name prefix used to generate resources" - type = string -} diff --git a/examples/docker_basic/versions.tf b/examples/docker_basic/versions.tf deleted file mode 100644 index 5a34d35..0000000 --- a/examples/docker_basic/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "= 4.64.0" - } - local = { - source = "hashicorp/local" - version = "~> 2.2.3" - } - docker = { - source = "kreuzwerker/docker" - version = "2.18.0" - } - } -} From 7e0b9fe157850e0ab4260c4c9660f7ac4831eb74 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 16:37:26 +0300 Subject: [PATCH 05/11] Adding descriptions to outputs and lint changes --- examples/basic/main.tf | 7 ---- examples/basic/outputs.tf | 0 examples/basic/variables.tf | 2 -- terraform/main.tf | 4 --- terraform/modules/alerting/outputs.tf | 3 +- terraform/modules/alerting/variables.tf | 34 ++++++++++++++----- terraform/modules/athena-connector/outputs.tf | 0 terraform/modules/s3-configs/main.tf | 20 +++++------ terraform/modules/s3-configs/outputs.tf | 3 +- terraform/modules/s3-gateway/outputs.tf | 3 +- terraform/outputs.tf | 24 +++++-------- 11 files changed, 49 insertions(+), 51 deletions(-) create mode 100644 examples/basic/outputs.tf create mode 100644 terraform/modules/athena-connector/outputs.tf diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 3830c67..37a199c 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -8,13 +8,6 @@ provider "aws" { } } -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} -data "aws_ecr_authorization_token" "token" {} -data "aws_availability_zones" "available" { - state = "available" -} - module "data_qa" { source = "../../terraform" diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index fa31ff3..e69de29 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -1,2 +0,0 @@ -variable "slack_webhook_url" { type = string } -variable "slack_channel" { type = string } diff --git a/terraform/main.tf b/terraform/main.tf index 47b9b3b..1d41d64 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,10 +1,6 @@ data "aws_caller_identity" "current" {} data "aws_region" "current" {} -data "aws_availability_zones" "available" { - state = "available" -} locals { resource_name_prefix = "${var.project}-${var.environment}" - cloudwatch_prefix = replace(title(replace(local.resource_name_prefix, "-", " ")), " ", "") } diff --git a/terraform/modules/alerting/outputs.tf b/terraform/modules/alerting/outputs.tf index af0d246..f912c27 100644 --- a/terraform/modules/alerting/outputs.tf +++ b/terraform/modules/alerting/outputs.tf @@ -1,3 +1,4 @@ output "sns_topic_arn" { - value = module.slack_notification.slack_topic_arn + description = "Notifications topic arn" + value = module.slack_notification.slack_topic_arn } diff --git a/terraform/modules/alerting/variables.tf b/terraform/modules/alerting/variables.tf index 14cb026..b87f4f1 100644 --- a/terraform/modules/alerting/variables.tf +++ b/terraform/modules/alerting/variables.tf @@ -1,7 +1,19 @@ -variable "slack_sns_topic_name" { type = string } -variable "slack_webhook_url" { type = string } -variable "slack_channel" { type = string } -variable "slack_username" { type = string } +variable "slack_sns_topic_name" { + description = "Sns topic name to forward notifications to" + type = string +} +variable "slack_webhook_url" { + description = "Slack webhook url in form https://hooks.slack.com/services/........" + type = string +} +variable "slack_channel" { + description = "Slack channel to send notifications" + type = string +} +variable "slack_username" { + description = "Slack username which will be used as author of notifications" + type = string +} variable "resource_name_prefix" { description = "Resource name prefix used to generate resources" @@ -9,23 +21,27 @@ variable "resource_name_prefix" { } variable "step_functions_to_monitor" { - type = set(string) - default = [] + description = "List of step functions for which to create cloudwatch metrics alarm" + type = set(string) + default = [] } variable "period" { - default = 60 description = "The period in seconds over which the specified statistic is applied." + type = number + default = 60 } variable "evaluation_periods" { - default = 1 description = "The number of periods over which data is compared to the specified threshold." + type = number + default = 1 } variable "datapoints_to_alarm" { - default = 1 description = "The number of datapoints that must be breaching to trigger the alarm." + type = number + default = 1 } variable "lambda_function_vpc_security_group_ids" { diff --git a/terraform/modules/athena-connector/outputs.tf b/terraform/modules/athena-connector/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/s3-configs/main.tf b/terraform/modules/s3-configs/main.tf index fe22d85..8933f55 100644 --- a/terraform/modules/s3-configs/main.tf +++ b/terraform/modules/s3-configs/main.tf @@ -22,11 +22,11 @@ resource "aws_s3_bucket_versioning" "fast-data-qa-bucket" { resource "aws_s3_object" "great_expectations_yml" { bucket = aws_s3_bucket.settings_bucket.bucket content_type = "application/x-yaml" - content = templatefile("${var.great_expectation_path}", { + content = templatefile(var.great_expectation_path, { bucket = aws_s3_bucket.settings_bucket.bucket }) key = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/great_expectations.yml" - etag = md5(templatefile("${var.great_expectation_path}", { + etag = md5(templatefile(var.great_expectation_path, { bucket = aws_s3_bucket.settings_bucket.bucket })) } @@ -35,40 +35,40 @@ resource "aws_s3_object" "test_configs" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.test_coverage_path key = "test_configs/test_coverage.json" - etag = filemd5("${var.test_coverage_path}") + etag = filemd5(var.test_coverage_path) } resource "aws_s3_object" "pipeline_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.pipeline_config_path key = "test_configs/pipeline.json" - etag = filemd5("${var.pipeline_config_path}") + etag = filemd5(var.pipeline_config_path) } resource "aws_s3_object" "pks_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.pks_path key = "test_configs/pks.json" - etag = filemd5("${var.pks_path}") + etag = filemd5(var.pks_path) } resource "aws_s3_object" "sort_keys_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.sort_keys_path key = "test_configs/sort_keys.json" - etag = filemd5("${var.sort_keys_path}") + etag = filemd5(var.sort_keys_path) } resource "aws_s3_object" "mapping_config" { bucket = aws_s3_bucket.settings_bucket.bucket source = var.mapping_path key = "test_configs/mapping.json" - etag = filemd5("${var.mapping_path}") + etag = filemd5(var.mapping_path) } resource "aws_s3_object" "expectations_store" { - for_each = fileset("${var.expectations_store}", "**") + for_each = fileset(var.expectations_store, "**") bucket = aws_s3_bucket.settings_bucket.bucket source = "${var.expectations_store}/${each.value}" key = "${aws_s3_bucket.settings_bucket.bucket}/great_expectations/expectations/${each.value}" @@ -77,12 +77,12 @@ resource "aws_s3_object" "expectations_store" { resource "aws_s3_object" "test_config_manifest" { bucket = aws_s3_bucket.settings_bucket.bucket - etag = md5(templatefile("${var.manifest_path}", { + etag = md5(templatefile(var.manifest_path, { env_name = var.environment, bucket_name = aws_s3_bucket.settings_bucket.bucket })) content_type = "application/json" - content = templatefile("${var.manifest_path}", + content = templatefile(var.manifest_path, { env_name = var.environment, bucket_name = aws_s3_bucket.settings_bucket.bucket diff --git a/terraform/modules/s3-configs/outputs.tf b/terraform/modules/s3-configs/outputs.tf index d75e500..1cef344 100644 --- a/terraform/modules/s3-configs/outputs.tf +++ b/terraform/modules/s3-configs/outputs.tf @@ -1,3 +1,4 @@ output "bucket_name" { - value = aws_s3_bucket.settings_bucket.bucket + description = "Name of s3 configs bucket" + value = aws_s3_bucket.settings_bucket.bucket } diff --git a/terraform/modules/s3-gateway/outputs.tf b/terraform/modules/s3-gateway/outputs.tf index 00884ef..05221de 100644 --- a/terraform/modules/s3-gateway/outputs.tf +++ b/terraform/modules/s3-gateway/outputs.tf @@ -1,3 +1,4 @@ output "s3_gateway_address" { - value = replace(aws_instance.s3_gateway.public_dns, "https", "http") + description = "DNS http address of s3 gateway" + value = replace(aws_instance.s3_gateway.public_dns, "https", "http") } diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 8d5e8a9..99e4a7c 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,29 +1,21 @@ output "step_function_arn" { - value = aws_sfn_state_machine.fast_data_qa.arn + description = "DataQA step function arn" + value = aws_sfn_state_machine.fast_data_qa.arn } output "lambda_allure_arn" { - value = module.lambda_allure_report.lambda_function_arn + description = "Allure reports generation lambda arn" + value = module.lambda_allure_report.lambda_function_arn } output "lambda_data_test_arn" { - value = module.lambda_data_test.lambda_function_arn + description = "Data test generation/running lambda arn" + value = module.lambda_data_test.lambda_function_arn } output "lambda_report_push_arn" { - value = module.lambda_push_report.lambda_function_arn -} - -output "allure_report_role_arn" { - value = module.lambda_allure_report.lambda_role_arn -} - -output "data_test_role_arn" { - value = module.lambda_data_test.lambda_role_arn -} - -output "report_push_role_arn" { - value = module.lambda_push_report.lambda_role_arn + description = "Report push to dynamodb lambda arn" + value = module.lambda_push_report.lambda_function_arn } output "bucket" { From ae42300d3c5286924f51897bf30f4d76708effbd Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 16:45:36 +0300 Subject: [PATCH 06/11] Tflint changes --- terraform/modules/s3-configs/main.tf | 4 ++-- tests/integration_tests/infra/outputs.tf | 0 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/integration_tests/infra/outputs.tf diff --git a/terraform/modules/s3-configs/main.tf b/terraform/modules/s3-configs/main.tf index 8933f55..bd3f7fd 100644 --- a/terraform/modules/s3-configs/main.tf +++ b/terraform/modules/s3-configs/main.tf @@ -3,7 +3,7 @@ resource "aws_s3_bucket" "settings_bucket" { force_destroy = true } -resource "aws_s3_bucket_public_access_block" "public_access_block_fast_data_qa" { +resource "aws_s3_bucket_public_access_block" "settings_bucket_public_access_block" { bucket = aws_s3_bucket.settings_bucket.id block_public_acls = true block_public_policy = true @@ -11,7 +11,7 @@ resource "aws_s3_bucket_public_access_block" "public_access_block_fast_data_qa" restrict_public_buckets = true } -resource "aws_s3_bucket_versioning" "fast-data-qa-bucket" { +resource "aws_s3_bucket_versioning" "settings_bucket_versioning" { bucket = aws_s3_bucket.settings_bucket.id versioning_configuration { diff --git a/tests/integration_tests/infra/outputs.tf b/tests/integration_tests/infra/outputs.tf new file mode 100644 index 0000000..e69de29 From 68aad47063c909a9cfe921fdf9a47ff503679a78 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 16:54:07 +0300 Subject: [PATCH 07/11] Fixing tf lint --- examples/basic/versions.tf | 2 +- terraform/modules/alerting/versions.tf | 10 ++++++++++ terraform/modules/athena-connector/versions.tf | 10 ++++++++++ terraform/modules/s3-configs/versions.tf | 10 ++++++++++ terraform/modules/s3-gateway/versions.tf | 10 ++++++++++ terraform/step_function.tf | 16 ++++++++-------- terraform/versions.tf | 2 +- 7 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 terraform/modules/alerting/versions.tf create mode 100644 terraform/modules/athena-connector/versions.tf create mode 100644 terraform/modules/s3-configs/versions.tf create mode 100644 terraform/modules/s3-gateway/versions.tf diff --git a/examples/basic/versions.tf b/examples/basic/versions.tf index f442154..9eb52cd 100644 --- a/examples/basic/versions.tf +++ b/examples/basic/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.64.0" + version = "~> 4.64.0" } } } diff --git a/terraform/modules/alerting/versions.tf b/terraform/modules/alerting/versions.tf new file mode 100644 index 0000000..9eb52cd --- /dev/null +++ b/terraform/modules/alerting/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } +} diff --git a/terraform/modules/athena-connector/versions.tf b/terraform/modules/athena-connector/versions.tf new file mode 100644 index 0000000..9eb52cd --- /dev/null +++ b/terraform/modules/athena-connector/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } +} diff --git a/terraform/modules/s3-configs/versions.tf b/terraform/modules/s3-configs/versions.tf new file mode 100644 index 0000000..9eb52cd --- /dev/null +++ b/terraform/modules/s3-configs/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } +} diff --git a/terraform/modules/s3-gateway/versions.tf b/terraform/modules/s3-gateway/versions.tf new file mode 100644 index 0000000..9eb52cd --- /dev/null +++ b/terraform/modules/s3-gateway/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.64.0" + } + } +} diff --git a/terraform/step_function.tf b/terraform/step_function.tf index 171248f..9ab889b 100644 --- a/terraform/step_function.tf +++ b/terraform/step_function.tf @@ -1,4 +1,4 @@ -resource "aws_cloudwatch_log_group" "state-machine-log-group" { +resource "aws_cloudwatch_log_group" "state_machine_log_group" { name = "/aws/${local.resource_name_prefix}/states/fast-data-qa-logs" retention_in_days = 0 } @@ -130,7 +130,7 @@ DEFINITION logging_configuration { include_execution_data = true level = "ALL" - log_destination = "${aws_cloudwatch_log_group.state-machine-log-group.arn}:*" + log_destination = "${aws_cloudwatch_log_group.state_machine_log_group.arn}:*" } tracing_configuration { @@ -156,15 +156,15 @@ resource "aws_iam_role" "step_functions_fast_data_qa" { ) force_detach_policies = false managed_policy_arns = [ - aws_iam_policy.CloudWatchLogsDeliveryFullAccessPolicy.arn, - aws_iam_policy.LambdaInvokeScopedAccessPolicy.arn, - aws_iam_policy.XRayAccessPolicy.arn + aws_iam_policy.cloud_watch_logs_delivery_full_access_policy.arn, + aws_iam_policy.lambda_invoke_scoped_access_policy.arn, + aws_iam_policy.xray_access_policy.arn ] max_session_duration = 3600 path = "/${var.environment}/" } -resource "aws_iam_policy" "CloudWatchLogsDeliveryFullAccessPolicy" { +resource "aws_iam_policy" "cloud_watch_logs_delivery_full_access_policy" { description = "Allows AWS Step Functions to write execution logs to CloudWatch Logs on your behalf" path = "/${var.environment}/" policy = jsonencode( @@ -190,7 +190,7 @@ resource "aws_iam_policy" "CloudWatchLogsDeliveryFullAccessPolicy" { ) } -resource "aws_iam_policy" "LambdaInvokeScopedAccessPolicy" { +resource "aws_iam_policy" "lambda_invoke_scoped_access_policy" { description = "Allow AWS Step Functions to invoke Lambda functions on your behalf" path = "/${var.environment}/" policy = jsonencode( @@ -213,7 +213,7 @@ resource "aws_iam_policy" "LambdaInvokeScopedAccessPolicy" { ) } -resource "aws_iam_policy" "XRayAccessPolicy" { +resource "aws_iam_policy" "xray_access_policy" { description = "Allow AWS Step Functions to call X-Ray daemon on your behalf" path = "/${var.environment}/" policy = jsonencode( diff --git a/terraform/versions.tf b/terraform/versions.tf index 34239b5..96df17b 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.64.0" + version = "~> 4.64.0" } local = { source = "hashicorp/local" From e1db2beba14965da11e68ea84ec92e441723b483 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 16:56:13 +0300 Subject: [PATCH 08/11] Fix tf version for intg tests --- tests/integration_tests/infra/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/infra/versions.tf b/tests/integration_tests/infra/versions.tf index fc5a627..9eb52cd 100644 --- a/tests/integration_tests/infra/versions.tf +++ b/tests/integration_tests/infra/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 4.67.0" + version = "~> 4.64.0" } } } From 79fe8eafd01035ee909dbd1bd605a2147263ccef Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 17:00:25 +0300 Subject: [PATCH 09/11] tflint --- terraform/modules/alerting/versions.tf | 2 -- terraform/modules/athena-connector/versions.tf | 2 -- terraform/modules/s3-configs/versions.tf | 2 -- terraform/modules/s3-gateway/versions.tf | 2 -- 4 files changed, 8 deletions(-) diff --git a/terraform/modules/alerting/versions.tf b/terraform/modules/alerting/versions.tf index 9eb52cd..503557c 100644 --- a/terraform/modules/alerting/versions.tf +++ b/terraform/modules/alerting/versions.tf @@ -1,6 +1,4 @@ terraform { - required_version = ">= 1.1.7" - required_providers { aws = { source = "hashicorp/aws" diff --git a/terraform/modules/athena-connector/versions.tf b/terraform/modules/athena-connector/versions.tf index 9eb52cd..503557c 100644 --- a/terraform/modules/athena-connector/versions.tf +++ b/terraform/modules/athena-connector/versions.tf @@ -1,6 +1,4 @@ terraform { - required_version = ">= 1.1.7" - required_providers { aws = { source = "hashicorp/aws" diff --git a/terraform/modules/s3-configs/versions.tf b/terraform/modules/s3-configs/versions.tf index 9eb52cd..503557c 100644 --- a/terraform/modules/s3-configs/versions.tf +++ b/terraform/modules/s3-configs/versions.tf @@ -1,6 +1,4 @@ terraform { - required_version = ">= 1.1.7" - required_providers { aws = { source = "hashicorp/aws" diff --git a/terraform/modules/s3-gateway/versions.tf b/terraform/modules/s3-gateway/versions.tf index 9eb52cd..503557c 100644 --- a/terraform/modules/s3-gateway/versions.tf +++ b/terraform/modules/s3-gateway/versions.tf @@ -1,6 +1,4 @@ terraform { - required_version = ">= 1.1.7" - required_providers { aws = { source = "hashicorp/aws" From b40a0c1225d61ec118ca9da030c9203348a10a1f Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 17:08:52 +0300 Subject: [PATCH 10/11] tflint --- terraform/modules.tf | 2 +- terraform/modules/alerting/versions.tf | 1 + terraform/modules/athena-connector/versions.tf | 1 + terraform/modules/s3-configs/versions.tf | 1 + terraform/modules/s3-gateway/versions.tf | 1 + terraform/variables.tf | 13 ------------- terraform/versions.tf | 4 ++++ 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/terraform/modules.tf b/terraform/modules.tf index 5a70119..cefcd67 100644 --- a/terraform/modules.tf +++ b/terraform/modules.tf @@ -1,4 +1,4 @@ -module "athena-connector" { +module "athena_connector" { source = "./modules/athena-connector" primary_aws_region = data.aws_region.current.name diff --git a/terraform/modules/alerting/versions.tf b/terraform/modules/alerting/versions.tf index 503557c..a8ce400 100644 --- a/terraform/modules/alerting/versions.tf +++ b/terraform/modules/alerting/versions.tf @@ -5,4 +5,5 @@ terraform { version = "~> 4.64.0" } } + required_version = "~> 1.1" } diff --git a/terraform/modules/athena-connector/versions.tf b/terraform/modules/athena-connector/versions.tf index 503557c..a8ce400 100644 --- a/terraform/modules/athena-connector/versions.tf +++ b/terraform/modules/athena-connector/versions.tf @@ -5,4 +5,5 @@ terraform { version = "~> 4.64.0" } } + required_version = "~> 1.1" } diff --git a/terraform/modules/s3-configs/versions.tf b/terraform/modules/s3-configs/versions.tf index 503557c..a8ce400 100644 --- a/terraform/modules/s3-configs/versions.tf +++ b/terraform/modules/s3-configs/versions.tf @@ -5,4 +5,5 @@ terraform { version = "~> 4.64.0" } } + required_version = "~> 1.1" } diff --git a/terraform/modules/s3-gateway/versions.tf b/terraform/modules/s3-gateway/versions.tf index 503557c..a8ce400 100644 --- a/terraform/modules/s3-gateway/versions.tf +++ b/terraform/modules/s3-gateway/versions.tf @@ -5,4 +5,5 @@ terraform { version = "~> 4.64.0" } } + required_version = "~> 1.1" } diff --git a/terraform/variables.tf b/terraform/variables.tf index c8f984f..a8edfba 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,10 +1,3 @@ -variable "tags" { - description = "Default tags" - type = map(string) - - default = {} -} - variable "project" { description = "Project name used to build fully qualified tags and resource's names" type = string @@ -208,12 +201,6 @@ variable "push_report_image_uri" { type = string } -variable "lambda_alerts_sns_topic_arn" { - description = "SNS topic used to to publish cloudwatch alerts" - type = string - default = null -} - variable "lambda_private_subnet_ids" { description = "List of private subnets assigned to lambda" type = list(string) diff --git a/terraform/versions.tf b/terraform/versions.tf index 96df17b..0fd1610 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -8,6 +8,10 @@ terraform { source = "hashicorp/local" version = "~> 2.2.3" } + null = { + source = "hashicorp/null" + version = "~> 3.2.1" + } } required_version = "~> 1.1" } From 398b1ff6ffb17587af7d5f8926ab8a21ecdd1c18 Mon Sep 17 00:00:00 2001 From: egorodov Date: Fri, 23 Jun 2023 17:12:38 +0300 Subject: [PATCH 11/11] tflint --- terraform/modules/athena-connector/versions.tf | 4 ++++ tests/integration_tests/infra/variables.tf | 0 2 files changed, 4 insertions(+) create mode 100644 tests/integration_tests/infra/variables.tf diff --git a/terraform/modules/athena-connector/versions.tf b/terraform/modules/athena-connector/versions.tf index a8ce400..61cd52c 100644 --- a/terraform/modules/athena-connector/versions.tf +++ b/terraform/modules/athena-connector/versions.tf @@ -4,6 +4,10 @@ terraform { source = "hashicorp/aws" version = "~> 4.64.0" } + null = { + source = "hashicorp/null" + version = "~> 3.2.1" + } } required_version = "~> 1.1" } diff --git a/tests/integration_tests/infra/variables.tf b/tests/integration_tests/infra/variables.tf new file mode 100644 index 0000000..e69de29