From b1e761dcb10cc2a8502862bddf872813c5f000aa Mon Sep 17 00:00:00 2001 From: Bohdan Yurov Date: Fri, 25 Oct 2019 10:25:15 +0200 Subject: [PATCH] Fixes #12: Add testing based on new approach https://github.com/terraform-google-modules/terraform-google-memorystore/issues/12 Updated testing. --- CONTRIBUTING.md | 104 +++ Gemfile.lock | 254 ------- Makefile | 153 ++-- README.md | 39 - build/docker/kitchen_terraform/Dockerfile | 61 -- build/docker/terraform/Dockerfile | 102 --- build/int.cloudbuild.yaml | 41 ++ .../lint.cloudbuild.yaml | 13 +- examples/basic/memorystore.tf | 2 +- helpers/terraform_docs | 694 ------------------ helpers/terraform_validate | 23 - test/.gitignore | 1 + test/boilerplate/boilerplate.Dockerfile.txt | 13 - test/boilerplate/boilerplate.Makefile.txt | 13 - test/boilerplate/boilerplate.go.txt | 15 - test/boilerplate/boilerplate.py.txt | 13 - test/boilerplate/boilerplate.sh.txt | 13 - test/boilerplate/boilerplate.xml.txt | 15 - test/fixtures/minimal/README.md | 4 +- test/fixtures/minimal/main.tf | 8 +- test/fixtures/minimal/outputs.tf | 4 - .../fixtures/minimal/terraform.tfvars.example | 3 - test/fixtures/minimal/variables.tf | 5 - test/integration/minimal/controls/gcloud.rb | 5 +- test/integration/minimal/inspec.yml | 3 - test/make.sh | 149 ---- test/setup/.gitignore | 2 + test/setup/iam.tf | 39 + test/setup/main.tf | 34 + Gemfile => test/setup/make_source.sh | 17 +- test/setup/outputs.tf | 32 + test/setup/variables.tf | 27 + .../boilerplate.tf.txt => setup/versions.tf} | 12 + test/test_verify_boilerplate.py | 136 ---- test/verify_boilerplate.py | 279 ------- 35 files changed, 375 insertions(+), 1953 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 Gemfile.lock delete mode 100644 build/docker/kitchen_terraform/Dockerfile delete mode 100644 build/docker/terraform/Dockerfile create mode 100644 build/int.cloudbuild.yaml rename test/boilerplate/boilerplate.yaml.txt => build/lint.cloudbuild.yaml (62%) delete mode 100755 helpers/terraform_docs delete mode 100755 helpers/terraform_validate create mode 100644 test/.gitignore delete mode 100644 test/boilerplate/boilerplate.Dockerfile.txt delete mode 100644 test/boilerplate/boilerplate.Makefile.txt delete mode 100644 test/boilerplate/boilerplate.go.txt delete mode 100644 test/boilerplate/boilerplate.py.txt delete mode 100644 test/boilerplate/boilerplate.sh.txt delete mode 100644 test/boilerplate/boilerplate.xml.txt delete mode 100755 test/make.sh create mode 100644 test/setup/.gitignore create mode 100644 test/setup/iam.tf create mode 100644 test/setup/main.tf rename Gemfile => test/setup/make_source.sh (54%) mode change 100644 => 100755 create mode 100644 test/setup/outputs.tf create mode 100644 test/setup/variables.tf rename test/{boilerplate/boilerplate.tf.txt => setup/versions.tf} (80%) delete mode 100755 test/test_verify_boilerplate.py delete mode 100644 test/verify_boilerplate.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..4ed39b01 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,104 @@ +# Contributing + +This document provides guidelines for contributing to the module VPC service controls. + +## Dependencies + +The following dependencies must be installed on the development system: + +- [Docker Engine][docker-engine] +- [Google Cloud SDK][google-cloud-sdk] +- [make] + +## Generating Documentation for Inputs and Outputs + +The Inputs and Outputs tables in the READMEs of the root module, +submodules, and example modules are automatically generated based on +the `variables` and `outputs` of the respective modules. These tables +must be refreshed if the module interfaces are changed. + +### Execution + +Run `make generate_docs` to generate new Inputs and Outputs tables. + +## Integration Testing + +Integration tests are used to verify the behaviour of the root module, +submodules, and example modules. Additions, changes, and fixes should +be accompanied with tests. + +The integration tests are run using [Kitchen][kitchen], +[Kitchen-Terraform][kitchen-terraform], and [InSpec][inspec]. These +tools are packaged within a Docker image for convenience. + +The general strategy for these tests is to verify the behaviour of the +[example modules](./examples/), thus ensuring that the root module, +submodules, and example modules are all functionally correct. + +### Test Environment +The easiest way to test the module is in an isolated test project. The setup for such a project is defined in [test/setup](./test/setup/) directory. + +To use this setup, you need a service account with Project Creator access on a folder. Export the Service Account credentials to your environment like so: + +``` +export SERVICE_ACCOUNT_JSON=$(< credentials.json) +``` + +You will also need to set a few environment variables: +``` +export TF_VAR_org_id="your_org_id" +export TF_VAR_folder_id="your_folder_id" +export TF_VAR_billing_account="your_billing_account_id" +``` + +With these settings in place, you can prepare a test project using Docker: +``` +make docker_test_prepare +``` + +To remove Access Policy, if any exists, run funcion inside Docker: +``` +remove_gcloud_org_accesspolicy +``` + +### Noninteractive Execution + +Run `make docker_test_integration` to test all of the example modules +noninteractively, using the prepared test project. + +### Interactive Execution + +1. Run `make docker_run` to start the testing Docker container in + interactive mode. + +1. Run `kitchen_do create ` to initialize the working + directory for an example module. + +1. Run `kitchen_do converge ` to apply the example module. + +1. Run `activate_gcloud_sa && kitchen_do verify ` to test the example module. + +1. Run `kitchen_do destroy ` to destroy the example module + state. + +## Linting and Formatting + +Many of the files in the repository can be linted or formatted to +maintain a standard of quality. + +### Execution + +Run `make docker_test_lint`. + +[docker-engine]: https://www.docker.com/products/docker-engine +[flake8]: http://flake8.pycqa.org/en/latest/ +[gofmt]: https://golang.org/cmd/gofmt/ +[google-cloud-sdk]: https://cloud.google.com/sdk/install +[hadolint]: https://github.com/hadolint/hadolint +[inspec]: https://inspec.io/ +[kitchen-terraform]: https://github.com/newcontext-oss/kitchen-terraform +[kitchen]: https://kitchen.ci/ +[make]: https://en.wikipedia.org/wiki/Make_(software) +[shellcheck]: https://www.shellcheck.net/ +[terraform-docs]: https://github.com/segmentio/terraform-docs +[terraform]: https://terraform.io/ diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 36ff9595..00000000 --- a/Gemfile.lock +++ /dev/null @@ -1,254 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.5.2) - public_suffix (>= 2.0.2, < 4.0) - aws-sdk (2.11.188) - aws-sdk-resources (= 2.11.188) - aws-sdk-core (2.11.188) - aws-sigv4 (~> 1.0) - jmespath (~> 1.0) - aws-sdk-resources (2.11.188) - aws-sdk-core (= 2.11.188) - aws-sigv4 (1.0.3) - azure_graph_rbac (0.17.0) - ms_rest_azure (~> 0.11.0) - azure_mgmt_key_vault (0.17.2) - ms_rest_azure (~> 0.11.0) - azure_mgmt_resources (0.17.2) - ms_rest_azure (~> 0.11.0) - builder (3.2.3) - coderay (1.1.2) - concurrent-ruby (1.1.4) - declarative (0.0.10) - declarative-option (0.1.0) - diff-lcs (1.3) - docker-api (1.34.2) - excon (>= 0.47.0) - multi_json - domain_name (0.5.20180417) - unf (>= 0.0.5, < 1.0.0) - dry-configurable (0.7.0) - concurrent-ruby (~> 1.0) - dry-container (0.6.0) - concurrent-ruby (~> 1.0) - dry-configurable (~> 0.1, >= 0.1.3) - dry-core (0.4.7) - concurrent-ruby (~> 1.0) - dry-equalizer (0.2.1) - dry-inflector (0.1.2) - dry-logic (0.4.2) - dry-container (~> 0.2, >= 0.2.6) - dry-core (~> 0.2) - dry-equalizer (~> 0.2) - dry-types (0.13.3) - concurrent-ruby (~> 1.0) - dry-container (~> 0.3) - dry-core (~> 0.4, >= 0.4.4) - dry-equalizer (~> 0.2) - dry-inflector (~> 0.1, >= 0.1.2) - dry-logic (~> 0.4, >= 0.4.2) - dry-validation (0.12.2) - concurrent-ruby (~> 1.0) - dry-configurable (~> 0.1, >= 0.1.3) - dry-core (~> 0.2, >= 0.2.1) - dry-equalizer (~> 0.2) - dry-logic (~> 0.4, >= 0.4.0) - dry-types (~> 0.13.1) - erubis (2.7.0) - excon (0.62.0) - faraday (0.15.4) - multipart-post (>= 1.2, < 3) - faraday-cookie_jar (0.0.6) - faraday (>= 0.7.4) - http-cookie (~> 1.0.0) - faraday_middleware (0.12.2) - faraday (>= 0.7.4, < 1.0) - ffi (1.9.25) - google-api-client (0.23.9) - addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.5, < 0.7.0) - httpclient (>= 2.8.1, < 3.0) - mime-types (~> 3.0) - representable (~> 3.0) - retriable (>= 2.0, < 4.0) - signet (~> 0.9) - googleauth (0.6.7) - faraday (~> 0.12) - jwt (>= 1.4, < 3.0) - memoist (~> 0.16) - multi_json (~> 1.11) - os (>= 0.9, < 2.0) - signet (~> 0.7) - gssapi (1.2.0) - ffi (>= 1.0.1) - gyoku (1.3.1) - builder (>= 2.1.2) - hashie (3.6.0) - htmlentities (4.3.4) - http-cookie (1.0.3) - domain_name (~> 0.5) - httpclient (2.8.3) - inifile (3.0.0) - inspec (3.0.64) - addressable (~> 2.4) - faraday (>= 0.9.0) - faraday_middleware (~> 0.12.2) - hashie (~> 3.4) - htmlentities - json (>= 1.8, < 3.0) - method_source (~> 0.8) - mixlib-log - multipart-post - parallel (~> 1.9) - parslet (~> 1.5) - pry (~> 0) - rspec (~> 3) - rspec-its (~> 1.2) - rubyzip (~> 1.2, >= 1.2.2) - semverse - sslshake (~> 1.2) - term-ansicolor - thor (~> 0.20) - tomlrb (~> 1.2) - train (~> 1.5, >= 1.5.6) - jmespath (1.4.0) - json (2.1.0) - jwt (2.1.0) - kitchen-terraform (4.1.0) - dry-types (~> 0.9) - dry-validation (~> 0.10) - inspec (~> 3.0) - mixlib-shellout (~> 2.2) - test-kitchen (~> 1.23) - little-plugger (1.1.4) - logging (2.2.2) - little-plugger (~> 1.1) - multi_json (~> 1.10) - memoist (0.16.0) - method_source (0.9.2) - mime-types (3.2.2) - mime-types-data (~> 3.2015) - mime-types-data (3.2018.0812) - mixlib-install (3.11.5) - mixlib-shellout - mixlib-versioning - thor - mixlib-log (2.0.4) - mixlib-shellout (2.4.2) - mixlib-versioning (1.2.2) - ms_rest (0.7.3) - concurrent-ruby (~> 1.0) - faraday (~> 0.9) - timeliness (~> 0.3) - ms_rest_azure (0.11.0) - concurrent-ruby (~> 1.0) - faraday (~> 0.9) - faraday-cookie_jar (~> 0.0.6) - ms_rest (~> 0.7.2) - multi_json (1.13.1) - multipart-post (2.0.0) - net-scp (1.2.1) - net-ssh (>= 2.6.5) - net-ssh (4.2.0) - net-ssh-gateway (1.3.0) - net-ssh (>= 2.6.5) - nori (2.6.0) - os (1.0.0) - parallel (1.12.1) - parslet (1.8.2) - pry (0.12.2) - coderay (~> 1.1.0) - method_source (~> 0.9.0) - public_suffix (3.0.3) - representable (3.0.4) - declarative (< 0.1.0) - declarative-option (< 0.2.0) - uber (< 0.2.0) - retriable (3.1.2) - rspec (3.8.0) - rspec-core (~> 3.8.0) - rspec-expectations (~> 3.8.0) - rspec-mocks (~> 3.8.0) - rspec-core (3.8.0) - rspec-support (~> 3.8.0) - rspec-expectations (3.8.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-its (1.2.0) - rspec-core (>= 3.0.0) - rspec-expectations (>= 3.0.0) - rspec-mocks (3.8.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-support (3.8.0) - rubyntlm (0.6.2) - rubyzip (1.2.2) - semverse (3.0.0) - signet (0.11.0) - addressable (~> 2.3) - faraday (~> 0.9) - jwt (>= 1.5, < 3.0) - multi_json (~> 1.10) - sslshake (1.2.0) - term-ansicolor (1.7.0) - tins (~> 1.0) - test-kitchen (1.23.5) - mixlib-install (~> 3.6) - mixlib-shellout (>= 1.2, < 3.0) - net-scp (~> 1.1) - net-ssh (>= 2.9, < 5.0) - net-ssh-gateway (~> 1.2) - thor (~> 0.19) - winrm (~> 2.0) - winrm-elevated (~> 1.0) - winrm-fs (~> 1.1) - thor (0.20.3) - timeliness (0.3.8) - tins (1.20.2) - tomlrb (1.2.7) - train (1.5.11) - aws-sdk (~> 2) - azure_graph_rbac (~> 0.16) - azure_mgmt_key_vault (~> 0.17) - azure_mgmt_resources (~> 0.15) - docker-api (~> 1.26) - google-api-client (~> 0.23.9) - googleauth (~> 0.6.6) - inifile - json (>= 1.8, < 3.0) - mixlib-shellout (~> 2.0) - net-scp (~> 1.2) - net-ssh (>= 2.9, < 6.0) - winrm (~> 2.0) - winrm-fs (~> 1.0) - uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.7.5) - winrm (2.3.0) - builder (>= 2.1.2) - erubis (~> 2.7) - gssapi (~> 1.2) - gyoku (~> 1.0) - httpclient (~> 2.2, >= 2.2.0.2) - logging (>= 1.6.1, < 3.0) - nori (~> 2.0) - rubyntlm (~> 0.6.0, >= 0.6.1) - winrm-elevated (1.1.0) - winrm (~> 2.0) - winrm-fs (~> 1.0) - winrm-fs (1.3.1) - erubis (~> 2.7) - logging (>= 1.6.1, < 3.0) - rubyzip (~> 1.1) - winrm (~> 2.0) - -PLATFORMS - ruby - -DEPENDENCIES - kitchen-terraform (~> 4.1)! - -BUNDLED WITH - 1.17.3 diff --git a/Makefile b/Makefile index ec5de103..dcf54f4c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,121 +12,74 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). +# Please make sure to contribute relevant changes upstream! + # Make will use bash instead of sh SHELL := /usr/bin/env bash -# Docker build config variables -CREDENTIALS_PATH ?= /cft/workdir/credentials.json -DOCKER_ORG := gcr.io/cloud-foundation-cicd -DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.3.0 -DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} - -all: check_shell check_python check_golang check_terraform test_check_headers check_headers check_trailing_whitespace generate_docs ## Run all linters and update documentation - -# The .PHONY directive tells make that this isn't a real target and so -# the presence of a file named 'check_shell' won't cause this target to stop -# working -.PHONY: check_shell -check_shell: ## Lint shell scripts - @source test/make.sh && check_shell - -.PHONY: check_python -check_python: ## Lint Python source files - @source test/make.sh && check_python - -.PHONY: check_golang -check_golang: ## Lint Go source files - @source test/make.sh && golang - -.PHONY: check_terraform -check_terraform: ## Lint Terraform source files - @source test/make.sh && check_terraform - -.PHONY: check_shebangs -check_shebangs: ## Check that scripts have correct shebangs - @source test/make.sh && check_bash - -.PHONY: check_trailing_whitespace -check_trailing_whitespace: - @source test/make.sh && check_trailing_whitespace - -.PHONY: test_check_headers -test_check_headers: - @echo "Testing the validity of the header check" - @source test/make.sh && check_headers +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.6 +DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools +REGISTRY_URL := gcr.io/cloud-foundation-cicd -.PHONY: check_headers -check_headers: ## Check that source files have appropriate boilerplate - @source test/make.sh && check_headers - -# Integration tests -.PHONY: test_integration -test_integration: ## Run integration tests - bundle install - bundle exec kitchen create - bundle exec kitchen converge - bundle exec kitchen converge - bundle exec kitchen verify - bundle exec kitchen destroy - -.PHONY: generate_docs -generate_docs: ## Update README documentation for Terraform variables and outputs - @source test/make.sh && generate_docs - -# Run docker +# Enter docker container for local development .PHONY: docker_run -docker_run: ## Launch a shell within the Docker test environment +docker_run: docker run --rm -it \ -e SERVICE_ACCOUNT_JSON \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ /bin/bash -.PHONY: docker_create -docker_create: ## Run `kitchen create` within the Docker test environment +# Execute prepare tests within the docker container +.PHONY: docker_test_prepare +docker_test_prepare: docker run --rm -it \ -e SERVICE_ACCOUNT_JSON \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen create" - -.PHONY: docker_converge -docker_converge: ## Run `kitchen converge` within the Docker test environment + -e TF_VAR_org_id \ + -e TF_VAR_folder_id \ + -e TF_VAR_billing_account \ + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/execute_with_credentials.sh prepare_environment + +# Clean up test environment within the docker container +.PHONY: docker_test_cleanup +docker_test_cleanup: docker run --rm -it \ -e SERVICE_ACCOUNT_JSON \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen converge && kitchen converge" - -.PHONY: docker_verify -docker_verify: ## Run `kitchen verify` within the Docker test environment + -e TF_VAR_org_id \ + -e TF_VAR_folder_id \ + -e TF_VAR_billing_account \ + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/execute_with_credentials.sh cleanup_environment + +# Execute integration tests within the docker container +.PHONY: docker_test_integration +docker_test_integration: docker run --rm -it \ -e SERVICE_ACCOUNT_JSON \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen verify" + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/test_integration.sh -.PHONY: docker_destroy -docker_destroy: ## Run `kitchen destroy` within the Docker test environment +# Execute lint tests within the docker container +.PHONY: docker_test_lint +docker_test_lint: docker run --rm -it \ - -e SERVICE_ACCOUNT_JSON \ - -e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \ - -e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \ - -v $(CURDIR):/cft/workdir \ - ${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \ - /bin/bash -c "kitchen destroy" + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /usr/local/bin/test_lint.sh -.PHONY: test_integration_docker -test_integration_docker: docker_create docker_converge docker_verify docker_destroy ## Run a full integration test cycle - @echo "Running test-kitchen tests in docker" +# Generate documentation +.PHONY: docker_generate_docs +docker_generate_docs: + docker run --rm -it \ + -v "$(CURDIR)":/workspace \ + $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ + /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs' -help: ## Prints help for targets with comments - @grep -E '^[a-zA-Z._-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +# Alias for backwards compatibility +.PHONY: generate_docs +generate_docs: docker_generate_docs diff --git a/README.md b/README.md index fc2bfe56..c5168beb 100644 --- a/README.md +++ b/README.md @@ -63,42 +63,3 @@ The project has the following folders and files: - /variables.tf: all the variables for the module - /output.tf: the outputs of the module - /readme.md: this file - -## Testing - -### Requirements - -- Terraform is [installed](#software-dependencies) on the machine where Terraform is executed. -- An existing google cloud project -- The `redis.googleapis.com` API should be enabled -- A service account key -- Docker - -### Software Dependencies -### Terraform -- [Terraform](https://www.terraform.io/downloads.html) >= 0.12.0 -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) >= v2.5.0 - -### Running Integration Tests - -Tests are run inside of an existing google cloud project. - -1. Create tfvars file for test: - -```sh -cp test/fixtures/minimal/terraform.tfvars.example test/fixtures/minimal/terraform.tfvars -``` - -2. Edit new `test/fixtures/minimal/terraform.tfvars` and add project id to run test in. - -3. Copy a service account key (`credentials.json` file) into root of this repo. - -4. Run `make test_integration_docker`. - -## Autogeneration of documentation from .tf files - -Run: - -``` -make generate_docs -``` diff --git a/build/docker/kitchen_terraform/Dockerfile b/build/docker/kitchen_terraform/Dockerfile deleted file mode 100644 index a2dda5c8..00000000 --- a/build/docker/kitchen_terraform/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG BUILD_TERRAFORM_IMAGE -ARG BUILD_RUBY_VERSION -FROM $BUILD_TERRAFORM_IMAGE as cfkt_terraform - - - -FROM ruby:$BUILD_RUBY_VERSION-alpine - -RUN apk add --no-cache \ - bash \ - curl \ - git \ - g++ \ - jq \ - make \ - musl-dev \ - python - -SHELL ["/bin/bash", "-c"] - -ENV APP_BASE_DIR="/cftk" - -COPY --from=cfkt_terraform $APP_BASE_DIR $APP_BASE_DIR - -ENV HOME="$APP_BASE_DIR/home" -ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH -ENV GOOGLE_APPLICATION_CREDENTIALS="$CREDENTIALS_PATH" \ - CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$CREDENTIALS_PATH" - -# Fix base64 inconsistency -SHELL ["/bin/bash", "-c"] -RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc - -RUN terraform --version && \ - gcloud --version && \ - ruby --version && \ - bundle --version - -WORKDIR $APP_BASE_DIR/workdir - -COPY ./Gemfile ./ - -RUN bundle install - -RUN gcloud components install beta --quiet -RUN gcloud components install alpha --quiet - diff --git a/build/docker/terraform/Dockerfile b/build/docker/terraform/Dockerfile deleted file mode 100644 index f0b5c27d..00000000 --- a/build/docker/terraform/Dockerfile +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM alpine:3.8 as builder - -RUN apk add --no-cache \ - bash \ - git \ - go \ - make \ - musl-dev - -ENV APP_BASE_DIR="/cftk" - -RUN mkdir -p $APP_BASE_DIR/home && \ - mkdir -p $APP_BASE_DIR/bin && \ - mkdir -p $APP_BASE_DIR/workdir - -ENV GOPATH="/root/go" - -ARG BUILD_PROVIDER_GOOGLE_VERSION -ENV PROVIDER_GOOGLE_VERSION="${BUILD_PROVIDER_GOOGLE_VERSION}" - -RUN mkdir -p $APP_BASE_DIR/home/.terraform.d/plugins && \ - mkdir -p $GOPATH/src/github.com/terraform-providers && \ - cd $GOPATH/src/github.com/terraform-providers && \ - git clone https://github.com/terraform-providers/terraform-provider-google.git && \ - cd terraform-provider-google && \ - git fetch --all --tags --prune && \ - git checkout tags/v${PROVIDER_GOOGLE_VERSION} -b v${PROVIDER_GOOGLE_VERSION} && \ - make fmt && \ - make build && \ - mv $GOPATH/bin/terraform-provider-google \ - $APP_BASE_DIR/home/.terraform.d/plugins/terraform-provider-google_v${PROVIDER_GOOGLE_VERSION} - - - -FROM alpine:3.8 - -RUN apk add --no-cache \ - bash \ - curl \ - git \ - jq \ - make \ - python2 - -ENV APP_BASE_DIR="/cftk" - -COPY --from=builder $APP_BASE_DIR $APP_BASE_DIR - -ENV HOME="$APP_BASE_DIR/home" -ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH -ENV GOOGLE_APPLICATION_CREDENTIALS="$CREDENTIALS_PATH" \ - CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$CREDENTIALS_PATH" - -# Fix base64 inconsistency -SHELL ["/bin/bash", "-c"] -RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc - -ARG BUILD_CLOUD_SDK_VERSION -ENV CLOUD_SDK_VERSION="${BUILD_CLOUD_SDK_VERSION}" - -RUN cd cftk && \ - curl -LO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ - tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ - rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ - ln -s /lib /lib64 && \ - gcloud config set core/disable_usage_reporting true && \ - gcloud config set component_manager/disable_update_check true && \ - gcloud config set metrics/environment github_docker_image && \ - gcloud --version - -ARG BUILD_TERRAFORM_VERSION -ENV TERRAFORM_VERSION="${BUILD_TERRAFORM_VERSION}" - -RUN curl -LO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - mv terraform $APP_BASE_DIR/bin && \ - terraform --version - -ARG BUILD_PROVIDER_GSUITE_VERSION -ENV PROVIDER_GSUITE_VERSION="${BUILD_PROVIDER_GSUITE_VERSION}" - -RUN curl -LO https://github.com/DeviaVir/terraform-provider-gsuite/releases/download/v${PROVIDER_GSUITE_VERSION}/terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ - tar xzf terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ - rm terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ - mv terraform-provider-gsuite_v${PROVIDER_GSUITE_VERSION} $APP_BASE_DIR/home/.terraform.d/plugins/ - -WORKDIR $APP_BASE_DIR/workdir diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml new file mode 100644 index 00000000..ff44db3c --- /dev/null +++ b/build/int.cloudbuild.yaml @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +timeout: 3600s +steps: +- id: prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment'] + env: + - 'TF_VAR_org_id=$_ORG_ID' + - 'TF_VAR_folder_id=$_FOLDER_ID' + - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' +- id: create + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] +- id: converge + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge'] +- id: verify + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify'] +- id: destroy + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy'] +tags: +- 'ci' +- 'integration' +substitutions: + _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6' diff --git a/test/boilerplate/boilerplate.yaml.txt b/build/lint.cloudbuild.yaml similarity index 62% rename from test/boilerplate/boilerplate.yaml.txt rename to build/lint.cloudbuild.yaml index b0c7da3d..10c5aee6 100644 --- a/test/boilerplate/boilerplate.yaml.txt +++ b/build/lint.cloudbuild.yaml @@ -1,4 +1,4 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +steps: +- id: 'lint' + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/usr/local/bin/test_lint.sh'] +tags: +- 'ci' +- 'lint' +substitutions: + _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6' diff --git a/examples/basic/memorystore.tf b/examples/basic/memorystore.tf index 4bf69311..6ad20e39 100644 --- a/examples/basic/memorystore.tf +++ b/examples/basic/memorystore.tf @@ -19,7 +19,7 @@ provider "google" { } module "memorystore" { - source = "git::ssh://git@github.com/terraform-google-modules/terraform-google-memorystore" + source = "../../" name = "memorystore" project = "memorystore" memory_size_gb = "1" diff --git a/helpers/terraform_docs b/helpers/terraform_docs deleted file mode 100755 index 0935b69e..00000000 --- a/helpers/terraform_docs +++ /dev/null @@ -1,694 +0,0 @@ -#!/usr/bin/env bash - -set -e - -main() { - declare argv - argv=$(getopt -o a: --long args: -- "$@") || return - eval "set -- $argv" - - declare args - declare files - - for argv; do - case $argv in - (-a|--args) - shift - args="$1" - shift - ;; - (--) - shift - files="$@" - break - ;; - esac - done - - local hack_terraform_docs=$(terraform version | head -1 | grep -c 0.12) - - if [[ "$hack_terraform_docs" == "1" ]]; then - which awk 2>&1 >/dev/null || ( echo "awk is required for terraform-docs hack to work with Terraform 0.12"; exit 1) - - tmp_file_awk=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") - terraform_docs_awk "$tmp_file_awk" - terraform_docs "$tmp_file_awk" "$args" "$files" - rm -f "$tmp_file_awk" - else - terraform_docs "0" "$args" "$files" - fi - -} - -terraform_docs() { - readonly terraform_docs_awk_file="$1" - readonly args="$2" - readonly files="$3" - - declare -a paths - declare -a tfvars_files - - index=0 - - for file_with_path in $files; do - file_with_path="${file_with_path// /__REPLACED__SPACE__}" - - paths[index]=$(dirname "$file_with_path") - - if [[ "$file_with_path" == *".tfvars" ]]; then - tfvars_files+=("$file_with_path") - fi - - ((index+=1)) - done - - readonly tmp_file=$(mktemp) - readonly text_file="README.md" - - for path_uniq in $(echo "${paths[*]}" | tr ' ' '\n' | sort -u); do - path_uniq="${path_uniq//__REPLACED__SPACE__/ }" - - pushd "$path_uniq" > /dev/null - - if [[ ! -f "$text_file" ]]; then - popd > /dev/null - continue - fi - - if [[ "$terraform_docs_awk_file" == "0" ]]; then - terraform-docs $args md ./ > "$tmp_file" - else - # Can't append extension for mktemp, so renaming instead - tmp_file_docs=$(mktemp "${TMPDIR:-/tmp}/terraform-docs-XXXXXXXXXX") - mv "$tmp_file_docs" "$tmp_file_docs.tf" - tmp_file_docs_tf="$tmp_file_docs.tf" - - awk -f "$terraform_docs_awk_file" ./*.tf > "$tmp_file_docs_tf" - terraform-docs $args md "$tmp_file_docs_tf" > "$tmp_file" - rm -f "$tmp_file_docs_tf" - fi - - # Replace content between markers with the placeholder - https://stackoverflow.com/questions/1212799/how-do-i-extract-lines-between-two-line-delimiters-in-perl#1212834 - perl -i -ne 'if (/BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/../END OF PRE-COMMIT-TERRAFORM DOCS HOOK/) { print $_ if /BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK/; print "I_WANT_TO_BE_REPLACED\n$_" if /END OF PRE-COMMIT-TERRAFORM DOCS HOOK/;} else { print $_ }' "$text_file" - - # Replace placeholder with the content of the file - perl -i -e 'open(F, "'"$tmp_file"'"); $f = join "", ; while(<>){if (/I_WANT_TO_BE_REPLACED/) {print $f} else {print $_};}' "$text_file" - - rm -f "$tmp_file" - - popd > /dev/null - done -} - -terraform_docs_awk() { - readonly output_file=$1 - - cat <<"EOF" > $output_file -# This script converts Terraform 0.12 variables/outputs to something suitable for `terraform-docs` -# As of terraform-docs v0.6.0, HCL2 is not supported. This script is a *dirty hack* to get around it. -# https://github.com/segmentio/terraform-docs/ -# https://github.com/segmentio/terraform-docs/issues/62 - -# Script was originally found here: https://github.com/cloudposse/build-harness/blob/master/bin/terraform-docs.awk - -{ - if ( $0 ~ /\{/ ) { - braceCnt++ - } - - if ( $0 ~ /\}/ ) { - braceCnt-- - } - - # [START] variable or output block started - if ($0 ~ /^[[:space:]]*(variable|output)[[:space:]][[:space:]]*"(.*?)"/) { - # Normalize the braceCnt (should be 1 now) - braceCnt = 1 - # [CLOSE] "default" block - if (blockDefCnt > 0) { - blockDefCnt = 0 - } - blockCnt++ - print $0 - } - - # [START] multiline default statement started - if (blockCnt > 0) { - if ($0 ~ /^[[:space:]][[:space:]]*(default)[[:space:]][[:space:]]*=/) { - if ($3 ~ "null") { - print " default = \"null\"" - } else { - print $0 - blockDefCnt++ - blockDefStart=1 - } - } - } - - # [PRINT] single line "description" - if (blockCnt > 0) { - if (blockDefCnt == 0) { - if ($0 ~ /^[[:space:]][[:space:]]*description[[:space:]][[:space:]]*=/) { - # [CLOSE] "default" block - if (blockDefCnt > 0) { - blockDefCnt = 0 - } - print $0 - } - } - } - - # [PRINT] single line "type" - if (blockCnt > 0) { - if ($0 ~ /^[[:space:]][[:space:]]*type[[:space:]][[:space:]]*=/ ) { - # [CLOSE] "default" block - if (blockDefCnt > 0) { - blockDefCnt = 0 - } - type=$3 - if (type ~ "object") { - print " type = \"object\"" - } else { - # legacy quoted types: "string", "list", and "map" - if ($3 ~ /^[[:space:]]*"(.*?)"[[:space:]]*$/) { - print " type = " $3 - } else { - print " type = \"" $3 "\"" - } - } - } - } - - # [CLOSE] variable/output block - if (blockCnt > 0) { - if (braceCnt == 0 && blockCnt > 0) { - blockCnt-- - print $0 - } - } - - # [PRINT] Multiline "default" statement - if (blockCnt > 0 && blockDefCnt > 0) { - if (blockDefStart == 1) { - blockDefStart = 0 - } else { - print $0 - } - } -} -EOF - -} - -getopt() { - # pure-getopt, a drop-in replacement for GNU getopt in pure Bash. - # version 1.4.3 - # - # Copyright 2012-2018 Aron Griffis - # - # Permission is hereby granted, free of charge, to any person obtaining - # a copy of this software and associated documentation files (the - # "Software"), to deal in the Software without restriction, including - # without limitation the rights to use, copy, modify, merge, publish, - # distribute, sublicense, and/or sell copies of the Software, and to - # permit persons to whom the Software is furnished to do so, subject to - # the following conditions: - # - # The above copyright notice and this permission notice shall be included - # in all copies or substantial portions of the Software. - # - # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - _getopt_main() { - # Returns one of the following statuses: - # 0 success - # 1 error parsing parameters - # 2 error in getopt invocation - # 3 internal error - # 4 reserved for -T - # - # For statuses 0 and 1, generates normalized and shell-quoted - # "options -- parameters" on stdout. - - declare parsed status - declare short long name flags - declare have_short=false - - # Synopsis from getopt man-page: - # - # getopt optstring parameters - # getopt [options] [--] optstring parameters - # getopt [options] -o|--options optstring [options] [--] parameters - # - # The first form can be normalized to the third form which - # _getopt_parse() understands. The second form can be recognized after - # first parse when $short hasn't been set. - - if [[ -n ${GETOPT_COMPATIBLE+isset} || $1 == [^-]* ]]; then - # Enable compatibility mode - flags=c$flags - # Normalize first to third synopsis form - set -- -o "$1" -- "${@:2}" - fi - - # First parse always uses flags=p since getopt always parses its own - # arguments effectively in this mode. - parsed=$(_getopt_parse getopt ahl:n:o:qQs:TuV \ - alternative,help,longoptions:,name:,options:,quiet,quiet-output,shell:,test,version \ - p "$@") - status=$? - if [[ $status != 0 ]]; then - if [[ $status == 1 ]]; then - echo "Try \`getopt --help' for more information." >&2 - # Since this is the first parse, convert status 1 to 2 - status=2 - fi - return $status - fi - eval "set -- $parsed" - - while [[ $# -gt 0 ]]; do - case $1 in - (-a|--alternative) - flags=a$flags ;; - - (-h|--help) - _getopt_help - return 2 # as does GNU getopt - ;; - - (-l|--longoptions) - long="$long${long:+,}$2" - shift ;; - - (-n|--name) - name=$2 - shift ;; - - (-o|--options) - short=$2 - have_short=true - shift ;; - - (-q|--quiet) - flags=q$flags ;; - - (-Q|--quiet-output) - flags=Q$flags ;; - - (-s|--shell) - case $2 in - (sh|bash) - flags=${flags//t/} ;; - (csh|tcsh) - flags=t$flags ;; - (*) - echo 'getopt: unknown shell after -s or --shell argument' >&2 - echo "Try \`getopt --help' for more information." >&2 - return 2 ;; - esac - shift ;; - - (-u|--unquoted) - flags=u$flags ;; - - (-T|--test) - return 4 ;; - - (-V|--version) - echo "pure-getopt 1.4.3" - return 0 ;; - - (--) - shift - break ;; - esac - - shift - done - - if ! $have_short; then - # $short was declared but never set, not even to an empty string. - # This implies the second form in the synopsis. - if [[ $# == 0 ]]; then - echo 'getopt: missing optstring argument' >&2 - echo "Try \`getopt --help' for more information." >&2 - return 2 - fi - short=$1 - have_short=true - shift - fi - - if [[ $short == -* ]]; then - # Leading dash means generate output in place rather than reordering, - # unless we're already in compatibility mode. - [[ $flags == *c* ]] || flags=i$flags - short=${short#?} - elif [[ $short == +* ]]; then - # Leading plus means POSIXLY_CORRECT, unless we're already in - # compatibility mode. - [[ $flags == *c* ]] || flags=p$flags - short=${short#?} - fi - - # This should fire if POSIXLY_CORRECT is in the environment, even if - # it's an empty string. That's the difference between :+ and + - flags=${POSIXLY_CORRECT+p}$flags - - _getopt_parse "${name:-getopt}" "$short" "$long" "$flags" "$@" - } - - _getopt_parse() { - # Inner getopt parser, used for both first parse and second parse. - # Returns 0 for success, 1 for error parsing, 3 for internal error. - # In the case of status 1, still generates stdout with whatever could - # be parsed. - # - # $flags is a string of characters with the following meanings: - # a - alternative parsing mode - # c - GETOPT_COMPATIBLE - # i - generate output in place rather than reordering - # p - POSIXLY_CORRECT - # q - disable error reporting - # Q - disable normal output - # t - quote for csh/tcsh - # u - unquoted output - - declare name="$1" short="$2" long="$3" flags="$4" - shift 4 - - # Split $long on commas, prepend double-dashes, strip colons; - # for use with _getopt_resolve_abbrev - declare -a longarr - _getopt_split longarr "$long" - longarr=( "${longarr[@]/#/--}" ) - longarr=( "${longarr[@]%:}" ) - longarr=( "${longarr[@]%:}" ) - - # Parse and collect options and parameters - declare -a opts params - declare o alt_recycled=false error=0 - - while [[ $# -gt 0 ]]; do - case $1 in - (--) - params=( "${params[@]}" "${@:2}" ) - break ;; - - (--*=*) - o=${1%%=*} - if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then - error=1 - elif [[ ,"$long", == *,"${o#--}"::,* ]]; then - opts=( "${opts[@]}" "$o" "${1#*=}" ) - elif [[ ,"$long", == *,"${o#--}":,* ]]; then - opts=( "${opts[@]}" "$o" "${1#*=}" ) - elif [[ ,"$long", == *,"${o#--}",* ]]; then - if $alt_recycled; then o=${o#-}; fi - _getopt_err "$name: option '$o' doesn't allow an argument" - error=1 - else - echo "getopt: assertion failed (1)" >&2 - return 3 - fi - alt_recycled=false - ;; - - (--?*) - o=$1 - if ! o=$(_getopt_resolve_abbrev "$o" "${longarr[@]}"); then - error=1 - elif [[ ,"$long", == *,"${o#--}",* ]]; then - opts=( "${opts[@]}" "$o" ) - elif [[ ,"$long", == *,"${o#--}::",* ]]; then - opts=( "${opts[@]}" "$o" '' ) - elif [[ ,"$long", == *,"${o#--}:",* ]]; then - if [[ $# -ge 2 ]]; then - shift - opts=( "${opts[@]}" "$o" "$1" ) - else - if $alt_recycled; then o=${o#-}; fi - _getopt_err "$name: option '$o' requires an argument" - error=1 - fi - else - echo "getopt: assertion failed (2)" >&2 - return 3 - fi - alt_recycled=false - ;; - - (-*) - if [[ $flags == *a* ]]; then - # Alternative parsing mode! - # Try to handle as a long option if any of the following apply: - # 1. There's an equals sign in the mix -x=3 or -xy=3 - # 2. There's 2+ letters and an abbreviated long match -xy - # 3. There's a single letter and an exact long match - # 4. There's a single letter and no short match - o=${1::2} # temp for testing #4 - if [[ $1 == *=* || $1 == -?? || \ - ,$long, == *,"${1#-}"[:,]* || \ - ,$short, != *,"${o#-}"[:,]* ]]; then - o=$(_getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" 2>/dev/null) - case $? in - (0) - # Unambiguous match. Let the long options parser handle - # it, with a flag to get the right error message. - set -- "-$1" "${@:2}" - alt_recycled=true - continue ;; - (1) - # Ambiguous match, generate error and continue. - _getopt_resolve_abbrev "${1%%=*}" "${longarr[@]}" >/dev/null - error=1 - shift - continue ;; - (2) - # No match, fall through to single-character check. - true ;; - (*) - echo "getopt: assertion failed (3)" >&2 - return 3 ;; - esac - fi - fi - - o=${1::2} - if [[ "$short" == *"${o#-}"::* ]]; then - if [[ ${#1} -gt 2 ]]; then - opts=( "${opts[@]}" "$o" "${1:2}" ) - else - opts=( "${opts[@]}" "$o" '' ) - fi - elif [[ "$short" == *"${o#-}":* ]]; then - if [[ ${#1} -gt 2 ]]; then - opts=( "${opts[@]}" "$o" "${1:2}" ) - elif [[ $# -ge 2 ]]; then - shift - opts=( "${opts[@]}" "$o" "$1" ) - else - _getopt_err "$name: option requires an argument -- '${o#-}'" - error=1 - fi - elif [[ "$short" == *"${o#-}"* ]]; then - opts=( "${opts[@]}" "$o" ) - if [[ ${#1} -gt 2 ]]; then - set -- "$o" "-${1:2}" "${@:2}" - fi - else - if [[ $flags == *a* ]]; then - # Alternative parsing mode! Report on the entire failed - # option. GNU includes =value but we omit it for sanity with - # very long values. - _getopt_err "$name: unrecognized option '${1%%=*}'" - else - _getopt_err "$name: invalid option -- '${o#-}'" - if [[ ${#1} -gt 2 ]]; then - set -- "$o" "-${1:2}" "${@:2}" - fi - fi - error=1 - fi ;; - - (*) - # GNU getopt in-place mode (leading dash on short options) - # overrides POSIXLY_CORRECT - if [[ $flags == *i* ]]; then - opts=( "${opts[@]}" "$1" ) - elif [[ $flags == *p* ]]; then - params=( "${params[@]}" "$@" ) - break - else - params=( "${params[@]}" "$1" ) - fi - esac - - shift - done - - if [[ $flags == *Q* ]]; then - true # generate no output - else - echo -n ' ' - if [[ $flags == *[cu]* ]]; then - printf '%s -- %s' "${opts[*]}" "${params[*]}" - else - if [[ $flags == *t* ]]; then - _getopt_quote_csh "${opts[@]}" -- "${params[@]}" - else - _getopt_quote "${opts[@]}" -- "${params[@]}" - fi - fi - echo - fi - - return $error - } - - _getopt_err() { - if [[ $flags != *q* ]]; then - printf '%s\n' "$1" >&2 - fi - } - - _getopt_resolve_abbrev() { - # Resolves an abbrevation from a list of possibilities. - # If the abbreviation is unambiguous, echoes the expansion on stdout - # and returns 0. If the abbreviation is ambiguous, prints a message on - # stderr and returns 1. (For first parse this should convert to exit - # status 2.) If there is no match at all, prints a message on stderr - # and returns 2. - declare a q="$1" - declare -a matches - shift - for a; do - if [[ $q == "$a" ]]; then - # Exact match. Squash any other partial matches. - matches=( "$a" ) - break - elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q" ]]; then - # Exact alternative match. Squash any other partial matches. - matches=( "$a" ) - break - elif [[ $a == "$q"* ]]; then - # Abbreviated match. - matches=( "${matches[@]}" "$a" ) - elif [[ $flags == *a* && $q == -[^-]* && $a == -"$q"* ]]; then - # Abbreviated alternative match. - matches=( "${matches[@]}" "${a#-}" ) - fi - done - case ${#matches[@]} in - (0) - [[ $flags == *q* ]] || \ - printf "$name: unrecognized option %s\\n" >&2 \ - "$(_getopt_quote "$q")" - return 2 ;; - (1) - printf '%s' "${matches[0]}"; return 0 ;; - (*) - [[ $flags == *q* ]] || \ - printf "$name: option %s is ambiguous; possibilities: %s\\n" >&2 \ - "$(_getopt_quote "$q")" "$(_getopt_quote "${matches[@]}")" - return 1 ;; - esac - } - - _getopt_split() { - # Splits $2 at commas to build array specified by $1 - declare IFS=, - eval "$1=( \$2 )" - } - - _getopt_quote() { - # Quotes arguments with single quotes, escaping inner single quotes - declare s space q=\' - for s; do - printf "$space'%s'" "${s//$q/$q\\$q$q}" - space=' ' - done - } - - _getopt_quote_csh() { - # Quotes arguments with single quotes, escaping inner single quotes, - # bangs, backslashes and newlines - declare s i c space - for s; do - echo -n "$space'" - for ((i=0; i<${#s}; i++)); do - c=${s:i:1} - case $c in - (\\|\'|!) - echo -n "'\\$c'" ;; - ($'\n') - echo -n "\\$c" ;; - (*) - echo -n "$c" ;; - esac - done - echo -n \' - space=' ' - done - } - - _getopt_help() { - cat <<-EOT >&2 - - Usage: - getopt - getopt [options] [--] - getopt [options] -o|--options [options] [--] - - Parse command options. - - Options: - -a, --alternative allow long options starting with single - - -l, --longoptions the long options to be recognized - -n, --name the name under which errors are reported - -o, --options the short options to be recognized - -q, --quiet disable error reporting by getopt(3) - -Q, --quiet-output no normal output - -s, --shell set quoting conventions to those of - -T, --test test for getopt(1) version - -u, --unquoted do not quote the output - - -h, --help display this help and exit - -V, --version output version information and exit - - For more details see getopt(1). - EOT - } - - _getopt_version_check() { - if [[ -z $BASH_VERSION ]]; then - echo "getopt: unknown version of bash might not be compatible" >&2 - return 1 - fi - - # This is a lexical comparison that should be sufficient forever. - if [[ $BASH_VERSION < 2.05b ]]; then - echo "getopt: bash $BASH_VERSION might not be compatible" >&2 - return 1 - fi - - return 0 - } - - _getopt_version_check - _getopt_main "$@" - declare status=$? - unset -f _getopt_main _getopt_err _getopt_parse _getopt_quote \ - _getopt_quote_csh _getopt_resolve_abbrev _getopt_split _getopt_help \ - _getopt_version_check - return $status -} - -[[ $BASH_SOURCE != "$0" ]] || main "$@" diff --git a/helpers/terraform_validate b/helpers/terraform_validate deleted file mode 100755 index 7f609827..00000000 --- a/helpers/terraform_validate +++ /dev/null @@ -1,23 +0,0 @@ -#! /bin/bash -# -# Copyright 2019 Google LLC. This software is provided as-is, without warranty -# or representation for any use or purpose. Your use of it is subject to your -# agreement with Google. -# -# This script initializes modules so that terraform validate as of 0.12 behaves -# as expected and does not issue errors such as: -# -# Error: Module not installed -# -# on test/fixtures/shared_vpc_no_subnets/main.tf line 37: -# 37: module "project-factory" { -# -# This module is not yet installed. Run "terraform init" to install all modules -# required by this configuration. - -# The first and only argument to this script is the directory containing *.tf -# files to validate. This directory is assumed to be a root module. - -cd "$1" -terraform init -backend=false -terraform validate diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..d69ba0d4 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +source.sh diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt deleted file mode 100644 index b0c7da3d..00000000 --- a/test/boilerplate/boilerplate.Dockerfile.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt deleted file mode 100644 index b0c7da3d..00000000 --- a/test/boilerplate/boilerplate.Makefile.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt deleted file mode 100644 index 557e16f0..00000000 --- a/test/boilerplate/boilerplate.go.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt deleted file mode 100644 index b0c7da3d..00000000 --- a/test/boilerplate/boilerplate.py.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt deleted file mode 100644 index 2e94f3e5..00000000 --- a/test/boilerplate/boilerplate.sh.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt deleted file mode 100644 index 3d98cdc6..00000000 --- a/test/boilerplate/boilerplate.xml.txt +++ /dev/null @@ -1,15 +0,0 @@ - diff --git a/test/fixtures/minimal/README.md b/test/fixtures/minimal/README.md index e1fb6228..ebd9d1ce 100644 --- a/test/fixtures/minimal/README.md +++ b/test/fixtures/minimal/README.md @@ -7,7 +7,6 @@ This test with create a new redis instance. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| credentials\_path | Path to service account key (usually credentials.json). | string | n/a | yes | | location\_id | Zone to create test instance. | string | `"us-east1-b"` | no | | memory\_size\_gb | Memory size of test instance. | number | `"1"` | no | | name | Name of redis instance. | string | `"test-minimal"` | no | @@ -18,7 +17,6 @@ This test with create a new redis instance. | Name | Description | |------|-------------| -| credentials\_path | | | location\_id | | | memory\_size\_gb | | | name | | @@ -29,4 +27,4 @@ This test with create a new redis instance. | project\_id | | | region | | - \ No newline at end of file + diff --git a/test/fixtures/minimal/main.tf b/test/fixtures/minimal/main.tf index 7912314a..359b3207 100644 --- a/test/fixtures/minimal/main.tf +++ b/test/fixtures/minimal/main.tf @@ -14,12 +14,6 @@ * limitations under the License. */ -provider "google" { - version = "~> 2.5.0" - credentials = file(var.credentials_path) - project = var.project_id -} - module "memstore" { source = "../../.." @@ -28,7 +22,7 @@ module "memstore" { project = var.project_id region = var.region location_id = var.location_id + enable_apis = false memory_size_gb = var.memory_size_gb } - diff --git a/test/fixtures/minimal/outputs.tf b/test/fixtures/minimal/outputs.tf index 18a1adb2..f83cfd8a 100644 --- a/test/fixtures/minimal/outputs.tf +++ b/test/fixtures/minimal/outputs.tf @@ -18,10 +18,6 @@ output "project_id" { value = var.project_id } -output "credentials_path" { - value = var.credentials_path -} - output "name" { value = var.name } diff --git a/test/fixtures/minimal/terraform.tfvars.example b/test/fixtures/minimal/terraform.tfvars.example index 788bee77..30d51336 100644 --- a/test/fixtures/minimal/terraform.tfvars.example +++ b/test/fixtures/minimal/terraform.tfvars.example @@ -1,6 +1,3 @@ # The google cloud project id to create resources in. project_id="" -# The path to look for google cloud credentials. -# NOTE: /cftk/workdir is mapped to the repo root in docker. -credentials_path="/cftk/workdir/credentials.json" diff --git a/test/fixtures/minimal/variables.tf b/test/fixtures/minimal/variables.tf index 51af6180..da985d01 100644 --- a/test/fixtures/minimal/variables.tf +++ b/test/fixtures/minimal/variables.tf @@ -19,11 +19,6 @@ variable "project_id" { type = string } -variable "credentials_path" { - description = "Path to service account key (usually credentials.json)." - type = string -} - variable "name" { description = "Name of redis instance." type = string diff --git a/test/integration/minimal/controls/gcloud.rb b/test/integration/minimal/controls/gcloud.rb index 71e37fcc..d12cddaf 100644 --- a/test/integration/minimal/controls/gcloud.rb +++ b/test/integration/minimal/controls/gcloud.rb @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -16,7 +16,6 @@ name = attribute('name') region = attribute('region') location_id = attribute('location_id') -credentials_path = attribute('credentials_path') memory_size_gb = attribute('memory_size_gb') output_id = attribute('output_id') @@ -24,8 +23,6 @@ output_host = attribute('output_host') output_current_location_id = attribute('output_current_location_id') -ENV['CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE'] = credentials_path - describe 'Outputs' do it 'should reflect inputted variables' do expect(output_region).to eq region diff --git a/test/integration/minimal/inspec.yml b/test/integration/minimal/inspec.yml index 22c62807..49233e5d 100644 --- a/test/integration/minimal/inspec.yml +++ b/test/integration/minimal/inspec.yml @@ -3,9 +3,6 @@ attributes: - name: project_id required: true type: string - - name: credentials_path - required: true - type: string - name: name required: true type: string diff --git a/test/make.sh b/test/make.sh deleted file mode 100755 index ec6ea892..00000000 --- a/test/make.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# find_files is a helper to exclude .git directories and match only regular -# files to avoid double-processing symlinks. -find_files() { - local pth="$1" - shift - find "${pth}" '(' \ - -path '*/.git' \ - -o -path '*/.terraform' \ - -o -path '*/.kitchen' \ - ')' \ - -prune -o -type f "$@" -} - -# Compatibility with both GNU and BSD style xargs. -compat_xargs() { - local compat=() rval - # Test if xargs is GNU or BSD style. GNU xargs will succeed with status 0 - # when given --no-run-if-empty and no input on STDIN. BSD xargs will fail and - # exit status non-zero If xargs fails, assume it is BSD style and proceed. - # stderr is silently redirected to avoid console log spam. - if xargs --no-run-if-empty /dev/null; then - compat=("--no-run-if-empty") - fi - xargs "${compat[@]}" "$@" - rval="$?" - if [[ -z "${NOWARN:-}" ]] && [[ "${rval}" -gt 0 ]]; then - echo "Warning: compat_xargs $* failed with exit code ${rval}" >&2 - fi - return "${rval}" -} - -# This function checks to make sure that every -# shebang has a '- e' flag, which causes it -# to exit on error -function check_bash() { -find . -name "*.sh" | while IFS= read -d '' -r file; -do - if [[ "$file" != *"bash -e"* ]]; - then - echo "$file is missing shebang with -e"; - exit 1; - fi; -done; -} - -# This function runs 'terraform validate' and 'terraform fmt' -# against all directory paths which contain *.tf files. -function check_terraform() { - local rval=125 - # fmt is before validate for faster feedback, validate requires terraform - # init which takes time. - echo "Running terraform fmt" - find_files . -name "*.tf" -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u \ - | compat_xargs -t -n1 terraform fmt -diff -check=true -write=false - rval="$?" - if [[ "${rval}" -gt 0 ]]; then - echo "Error: terraform fmt failed with exit code ${rval}" >&2 - echo "Check the output for diffs and correct using terraform fmt " >&2 - return "${rval}" - fi - echo "Running terraform validate" - find_files . -not -path "./test/fixtures/shared/*" -name "*.tf" -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u \ - | compat_xargs -t -n1 helpers/terraform_validate -} - -# This function runs 'go fmt' and 'go vet' on every file -# that ends in '.go' -function golang() { - echo "Running go fmt and go vet" - find_files . -name "*.go" -print0 | compat_xargs -0 -n1 go fmt - find_files . -name "*.go" -print0 | compat_xargs -0 -n1 go vet -} - -# This function runs the flake8 linter on every file -# ending in '.py' -function check_python() { - echo "Running flake8" - find_files . -name "*.py" -print0 | compat_xargs -0 flake8 - return 0 -} - -# This function runs the shellcheck linter on every -# file ending in '.sh' -function check_shell() { - echo "Running shellcheck" - find_files . -name "*.sh" -print0 | compat_xargs -0 shellcheck -x -} - -# This function makes sure that there is no trailing whitespace -# in any files in the project. -# There are some exclusions -function check_trailing_whitespace() { - local rc - echo "Checking for trailing whitespace" - find_files . -print \ - | grep -v -E '\.(pyc|png)$' \ - | NOWARN=1 compat_xargs grep -H -n '[[:blank:]]$' - rc=$? - if [[ ${rc} -eq 0 ]]; then - return 1 - fi -} - -function generate_docs() { - echo "Generating markdown docs with terraform-docs" - local pth helper_dir rval - helper_dir="$(pwd)/helpers" - while read -r pth; do - if [[ -e "${pth}/README.md" ]]; then - (cd "${pth}" || return 3; "${helper_dir}"/terraform_docs .;) - rval="$?" - if [[ "${rval}" -gt 0 ]]; then - echo "Error: terraform_docs in ${pth} exit code: ${rval}" >&2 - return "${rval}" - fi - else - echo "Skipping ${pth} because README.md does not exist." - fi - done < <(find_files . -name '*.tf' -print0 \ - | compat_xargs -0 -n1 dirname \ - | sort -u) -} - -function check_headers() { - echo "Checking file headers" - # Use the exclusion behavior of find_files - find_files . -type f -print0 \ - | compat_xargs -0 python test/verify_boilerplate.py -} diff --git a/test/setup/.gitignore b/test/setup/.gitignore new file mode 100644 index 00000000..0e515f83 --- /dev/null +++ b/test/setup/.gitignore @@ -0,0 +1,2 @@ +terraform.tfvars +source.sh diff --git a/test/setup/iam.tf b/test/setup/iam.tf new file mode 100644 index 00000000..9255735b --- /dev/null +++ b/test/setup/iam.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + int_required_roles = [ + "roles/owner" + ] +} + +resource "google_service_account" "int_test" { + project = module.project.project_id + account_id = "ci-account" + display_name = "ci-account" +} + +resource "google_project_iam_member" "int_test" { + count = length(local.int_required_roles) + + project = module.project.project_id + role = local.int_required_roles[count.index] + member = "serviceAccount:${google_service_account.int_test.email}" +} + +resource "google_service_account_key" "int_test" { + service_account_id = google_service_account.int_test.id +} diff --git a/test/setup/main.tf b/test/setup/main.tf new file mode 100644 index 00000000..0541a108 --- /dev/null +++ b/test/setup/main.tf @@ -0,0 +1,34 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "project" { + source = "terraform-google-modules/project-factory/google" + version = "~> 3.0" + + name = "ci-memory-store" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + auto_create_network = true + + activate_apis = [ + "cloudresourcemanager.googleapis.com", + "storage-api.googleapis.com", + "serviceusage.googleapis.com", + "redis.googleapis.com", + ] +} diff --git a/Gemfile b/test/setup/make_source.sh old mode 100644 new mode 100755 similarity index 54% rename from Gemfile rename to test/setup/make_source.sh index 43cea5f0..5070f0e9 --- a/Gemfile +++ b/test/setup/make_source.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,6 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -source 'https://rubygems.org/' do - gem 'kitchen-terraform', '~> 4.1' -end +echo "#!/usr/bin/env bash" > ../source.sh + +project_id=$(terraform output project_id) +echo "export TF_VAR_project_id='$project_id'" >> ../source.sh + +sa_json=$(terraform output sa_key) +# shellcheck disable=SC2086 +echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh + +parent_id=$(terraform output parent_id) +echo "export TF_VAR_parent_id='$parent_id'" >> ../source.sh +echo "export TF_VAR_org_id='$parent_id'" >> ../source.sh diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf new file mode 100644 index 00000000..3a72d631 --- /dev/null +++ b/test/setup/outputs.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.project.project_id +} + +output "sa_key" { + value = google_service_account_key.int_test.private_key + sensitive = true +} + +output "sa_email" { + value = google_service_account.int_test.email +} + +output "parent_id" { + value = var.org_id +} diff --git a/test/setup/variables.tf b/test/setup/variables.tf new file mode 100644 index 00000000..84134fe3 --- /dev/null +++ b/test/setup/variables.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "org_id" { + description = "The numeric organization id" +} + +variable "folder_id" { + description = "The folder to deploy in" +} + +variable "billing_account" { + description = "The billing account id associated with the project, e.g. XXXXXX-YYYYYY-ZZZZZZ" +} diff --git a/test/boilerplate/boilerplate.tf.txt b/test/setup/versions.tf similarity index 80% rename from test/boilerplate/boilerplate.tf.txt rename to test/setup/versions.tf index 83185dca..efbd8ea5 100644 --- a/test/boilerplate/boilerplate.tf.txt +++ b/test/setup/versions.tf @@ -13,3 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +terraform { + required_version = ">= 0.12" +} + +provider "google" { + version = "~> 2.13.0" +} + +provider "google-beta" { + version = "~> 2.13.0" +} diff --git a/test/test_verify_boilerplate.py b/test/test_verify_boilerplate.py deleted file mode 100755 index 22a3cca0..00000000 --- a/test/test_verify_boilerplate.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -''' A simple test for the verify_boilerplate python script. -This will create a set of test files, both valid and invalid, -and confirm that the has_valid_header call returns the correct -value. - -It also checks the number of files that are found by the -get_files call. -''' -from copy import deepcopy -from tempfile import mkdtemp -from shutil import rmtree -import unittest -from verify_boilerplate import has_valid_header, get_refs, get_regexs, \ - get_args, get_files - - -class AllTestCase(unittest.TestCase): - """ - All of the setup, teardown, and tests are contained in this - class. - """ - - def write_file(self, filename, content, expected): - """ - A utility method that creates test files, and adds them to - the cases that will be tested. - - Args: - filename: (string) the file name (path) to be created. - content: (list of strings) the contents of the file. - expected: (boolean) True if the header is expected to be valid, - false if not. - """ - - file = open(filename, 'w+') - for line in content: - file.write(line + "\n") - file.close() - self.cases[filename] = expected - - def create_test_files(self, tmp_path, extension, header): - """ - Creates 2 test files for .tf, .xml, .go, etc and one for - Dockerfile, and Makefile. - - The reason for the difference is that Makefile and Dockerfile - don't have an extension. These would be substantially more - difficult to create negative test cases, unless the files - were written, deleted, and re-written. - - Args: - tmp_path: (string) the path in which to create the files - extension: (string) the file extension - header: (list of strings) the header/boilerplate content - """ - - content = "\n...blah \ncould be code or could be garbage\n" - special_cases = ["Dockerfile", "Makefile"] - header_template = deepcopy(header) - valid_filename = tmp_path + extension - valid_content = header_template.append(content) - if extension not in special_cases: - # Invalid test cases for non-*file files (.tf|.py|.sh|.yaml|.xml..) - invalid_header = [] - for line in header_template: - if "2018" in line: - invalid_header.append(line.replace('2018', 'YEAR')) - else: - invalid_header.append(line) - invalid_header.append(content) - invalid_content = invalid_header - invalid_filename = tmp_path + "invalid." + extension - self.write_file(invalid_filename, invalid_content, False) - valid_filename = tmp_path + "testfile." + extension - - valid_content = header_template - self.write_file(valid_filename, valid_content, True) - - def setUp(self): - """ - Set initial counts and values, and initializes the setup of the - test files. - """ - self.cases = {} - self.tmp_path = mkdtemp() + "/" - self.my_args = get_args() - self.my_refs = get_refs(self.my_args) - self.my_regex = get_regexs() - self.prexisting_file_count = len( - get_files(self.my_refs.keys(), self.my_args)) - for key in self.my_refs: - self.create_test_files(self.tmp_path, key, - self.my_refs.get(key)) - - def tearDown(self): - """ Delete the test directory. """ - rmtree(self.tmp_path) - - def test_files_headers(self): - """ - Confirms that the expected output of has_valid_header is correct. - """ - for case in self.cases: - if self.cases[case]: - self.assertTrue(has_valid_header(case, self.my_refs, - self.my_regex)) - else: - self.assertFalse(has_valid_header(case, self.my_refs, - self.my_regex)) - - def test_invalid_count(self): - """ - Test that the initial files found isn't zero, indicating - a problem with the code. - """ - self.assertFalse(self.prexisting_file_count == 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py deleted file mode 100644 index 3f6e0d92..00000000 --- a/test/verify_boilerplate.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Verifies that all source files contain the necessary copyright boilerplate -# snippet. -# This is based on existing work -# https://github.com/kubernetes/test-infra/blob/master/hack -# /verify_boilerplate.py -from __future__ import print_function -import argparse -import glob -import os -import re -import sys - - -def get_args(): - """Parses command line arguments. - - Configures and runs argparse.ArgumentParser to extract command line - arguments. - - Returns: - An argparse.Namespace containing the arguments parsed from the - command line - """ - parser = argparse.ArgumentParser() - parser.add_argument("filenames", - help="list of files to check, " - "all files if unspecified", - nargs='*') - rootdir = os.path.dirname(__file__) + "/../" - rootdir = os.path.abspath(rootdir) - parser.add_argument( - "--rootdir", - default=rootdir, - help="root directory to examine") - - default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") - parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) - return parser.parse_args() - - -def get_refs(args): - """Converts the directory of boilerplate files into a map keyed by file - extension. - - Reads each boilerplate file's contents into an array, then adds that array - to a map keyed by the file extension. - - Returns: - A map of boilerplate lines, keyed by file extension. For example, - boilerplate.py.txt would result in the k,v pair {".py": py_lines} where - py_lines is an array containing each line of the file. - """ - refs = {} - - # Find and iterate over the absolute path for each boilerplate template - for path in glob.glob(os.path.join( - args.boilerplate_dir, - "boilerplate.*.txt")): - extension = os.path.basename(path).split(".")[1] - ref_file = open(path, 'r') - ref = ref_file.read().splitlines() - ref_file.close() - refs[extension] = ref - return refs - - -# pylint: disable=too-many-locals -def has_valid_header(filename, refs, regexs): - """Test whether a file has the correct boilerplate header. - - Tests each file against the boilerplate stored in refs for that file type - (based on extension), or by the entire filename (eg Dockerfile, Makefile). - Some heuristics are applied to remove build tags and shebangs, but little - variance in header formatting is tolerated. - - Args: - filename: A string containing the name of the file to test - refs: A map of boilerplate headers, keyed by file extension - regexs: a map of compiled regex objects used in verifying boilerplate - - Returns: - True if the file has the correct boilerplate header, otherwise returns - False. - """ - try: - with open(filename, 'r') as fp: # pylint: disable=invalid-name - data = fp.read() - except IOError: - return False - basename = os.path.basename(filename) - extension = get_file_extension(filename) - if extension: - ref = refs[extension] - else: - ref = refs[basename] - # remove build tags from the top of Go files - if extension == "go": - con = regexs["go_build_constraints"] - (data, found) = con.subn("", data, 1) - # remove shebang - elif extension == "sh" or extension == "py": - she = regexs["shebang"] - (data, found) = she.subn("", data, 1) - data = data.splitlines() - # if our test file is smaller than the reference it surely fails! - if len(ref) > len(data): - return False - # trim our file to the same number of lines as the reference file - data = data[:len(ref)] - year = regexs["year"] - for datum in data: - if year.search(datum): - return False - - # if we don't match the reference at this point, fail - if ref != data: - return False - return True - - -def get_file_extension(filename): - """Extracts the extension part of a filename. - - Identifies the extension as everything after the last period in filename. - - Args: - filename: string containing the filename - - Returns: - A string containing the extension in lowercase - """ - return os.path.splitext(filename)[1].split(".")[-1].lower() - - -# These directories will be omitted from header checks -SKIPPED_DIRS = [ - 'Godeps', 'third_party', '_gopath', '_output', - '.git', 'vendor', '__init__.py', 'node_modules' -] - - -def normalize_files(files): - """Extracts the files that require boilerplate checking from the files - argument. - - A new list will be built. Each path from the original files argument will - be added unless it is within one of SKIPPED_DIRS. All relative paths will - be converted to absolute paths by prepending the root_dir path parsed from - the command line, or its default value. - - Args: - files: a list of file path strings - - Returns: - A modified copy of the files list where any any path in a skipped - directory is removed, and all paths have been made absolute. - """ - newfiles = [] - for pathname in files: - if any(x in pathname for x in SKIPPED_DIRS): - continue - newfiles.append(pathname) - for idx, pathname in enumerate(newfiles): - if not os.path.isabs(pathname): - newfiles[idx] = os.path.join(args.rootdir, pathname) - return newfiles - - -def get_files(extensions, args): - """Generates a list of paths whose boilerplate should be verified. - - If a list of file names has been provided on the command line, it will be - treated as the initial set to search. Otherwise, all paths within rootdir - will be discovered and used as the initial set. - - Once the initial set of files is identified, it is normalized via - normalize_files() and further stripped of any file name whose extension is - not in extensions. - - Args: - extensions: a list of file extensions indicating which file types - should have their boilerplate verified - - Returns: - A list of absolute file paths - """ - files = [] - if args.filenames: - files = args.filenames - else: - for root, dirs, walkfiles in os.walk(args.rootdir): - # don't visit certain dirs. This is just a performance improvement - # as we would prune these later in normalize_files(). But doing it - # cuts down the amount of filesystem walking we do and cuts down - # the size of the file list - for dpath in SKIPPED_DIRS: - if dpath in dirs: - dirs.remove(dpath) - for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) - files = normalize_files(files) - outfiles = [] - for pathname in files: - basename = os.path.basename(pathname) - extension = get_file_extension(pathname) - if extension in extensions or basename in extensions: - outfiles.append(pathname) - return outfiles - - -def get_regexs(): - """Builds a map of regular expressions used in boilerplate validation. - - There are two scenarios where these regexes are used. The first is in - validating the date referenced is the boilerplate, by ensuring it is an - acceptable year. The second is in identifying non-boilerplate elements, - like shebangs and compiler hints that should be ignored when validating - headers. - - Returns: - A map of compiled regular expression objects, keyed by mnemonic. - """ - regexs = {} - # Search for "YEAR" which exists in the boilerplate, but shouldn't in the - # real thing - regexs["year"] = re.compile('YEAR') - # dates can be 2014, 2015, 2016 or 2017, company holder names can be - # anything - regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') - # strip // +build \n\n build constraints - regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", - re.MULTILINE) - # strip #!.* from shell/python scripts - regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) - return regexs - - -def main(args): - """Identifies and verifies files that should have the desired boilerplate. - - Retrieves the lists of files to be validated and tests each one in turn. - If all files contain correct boilerplate, this function terminates - normally. Otherwise it prints the name of each non-conforming file and - exists with a non-zero status code. - """ - regexs = get_regexs() - refs = get_refs(args) - filenames = get_files(refs.keys(), args) - nonconforming_files = [] - for filename in filenames: - if not has_valid_header(filename, refs, regexs): - nonconforming_files.append(filename) - if nonconforming_files: - print('%d files have incorrect boilerplate headers:' % len( - nonconforming_files)) - for filename in sorted(nonconforming_files): - print(os.path.relpath(filename, args.rootdir)) - sys.exit(1) - - -if __name__ == "__main__": - args = get_args() - main(args)