1.0 - G Suite Refactor

.gitignore

@@ -43,3 +43,4 @@ terraform.tfstate.d/
 credentials.json
 .vscode/
 env/
+test/fixtures/shared/terraform.tfvars

.kitchen.yml

@@ -21,6 +21,7 @@ platforms:
 
 verifier:
   name: terraform
+  color: false
   systems:
     - name: system
       backend: local

.ruby-version

@@ -0,0 +1 @@
+2.5.3

Makefile

@@ -16,16 +16,10 @@
 SHELL := /usr/bin/env bash
 
 # Docker build config variables
-BUILD_TERRAFORM_VERSION ?= 0.11.10
-BUILD_CLOUD_SDK_VERSION ?= 216.0.0
-BUILD_PROVIDER_GOOGLE_VERSION ?= 1.19.1
-BUILD_PROVIDER_GSUITE_VERSION ?= 0.1.10
-DOCKER_IMAGE_TERRAFORM := cftk/terraform
-DOCKER_TAG_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
-BUILD_RUBY_VERSION ?= 2.5.3
-DOCKER_IMAGE_KITCHEN_TERRAFORM := cftk/kitchen_terraform
-DOCKER_TAG_KITCHEN_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
-
+CREDENTIALS_PATH ?= /cft/workdir/credentials.json
+DOCKER_ORG := gcr.io/cloud-foundation-cicd
+DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 0.11.10_216.0.0_1.19.1_0.1.10
+DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM}
 
 all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs ## Run all linters and update documentation
 
@@ -46,7 +40,7 @@ check_golang: ## Lint Go source files
 
 .PHONY: check_terraform
 check_terraform:
-	@source ## Lint Terraform source files
+	@source test/make.sh && check_terraform ## Lint Terraform source files
 
 .PHONY: check_docker
 check_docker: ## Lint Dockerfiles
@@ -92,57 +86,50 @@ generate_docs: ## Update README documentation for Terraform variables and output
 release-new-version:
 	@source helpers/release-new-version.sh
 
-# Build Docker
-.PHONY: docker_build_terraform
-docker_build_terraform:
-	docker build -f build/docker/terraform/Dockerfile \
-		--build-arg BUILD_TERRAFORM_VERSION=${BUILD_TERRAFORM_VERSION} \
-		--build-arg BUILD_CLOUD_SDK_VERSION=${BUILD_CLOUD_SDK_VERSION} \
-		--build-arg BUILD_PROVIDER_GOOGLE_VERSION=${BUILD_PROVIDER_GOOGLE_VERSION} \
-		--build-arg BUILD_PROVIDER_GSUITE_VERSION=${BUILD_PROVIDER_GSUITE_VERSION} \
-		-t ${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM} .
-
-.PHONY: docker_build_kitchen_terraform
-docker_build_kitchen_terraform:
-	docker build -f build/docker/kitchen_terraform/Dockerfile \
-		--build-arg BUILD_TERRAFORM_IMAGE="${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM}" \
-		--build-arg BUILD_RUBY_VERSION="${BUILD_RUBY_VERSION}" \
-		-t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} .
-
 # Run docker
 .PHONY: docker_run
 docker_run: ## Launch a shell within the Docker test environment
 	docker run --rm -it \
-		-v $(CURDIR):/cftk/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
 		/bin/bash
 
 .PHONY: docker_create
 docker_create: ## Run `kitchen create` within the Docker test environment
 	docker run --rm -it \
-		-v $(CURDIR):/cftk/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
 		/bin/bash -c "bundle exec kitchen create"
 
 .PHONY: docker_converge
 docker_converge: ## Run `kitchen converge` within the Docker test environment
 	docker run --rm -it \
-		-v $(CURDIR):/cftk/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
 		/bin/bash -c "bundle exec kitchen converge && bundle exec kitchen converge"
 
 .PHONY: docker_verify
 docker_verify: ## Run `kitchen verify` within the Docker test environment
 	docker run --rm -it \
-		-v $(CURDIR):/cftk/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
 		/bin/bash -c "bundle exec kitchen verify"
 
 .PHONY: docker_destroy
 docker_destroy: ## Run `kitchen destroy` within the Docker test environment
 	docker run --rm -it \
-		-v $(CURDIR):/cftk/workdir \
-		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		-e CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${CREDENTIALS_PATH} \
+		-e GOOGLE_APPLICATION_CREDENTIALS=${CREDENTIALS_PATH} \
+		-v $(CURDIR):/cft/workdir \
+		${DOCKER_REPO_BASE_KITCHEN_TERRAFORM} \
 		/bin/bash -c "bundle exec kitchen destroy"
 
 .PHONY: test_integration_docker

README.md

@@ -149,6 +149,7 @@ The project has the following folders and files:
 
 -   [Terraform](https://www.terraform.io/downloads.html) 0.10.x
 -   [terraform-provider-google] plugin 1.19.x
+-   [terraform-provider-google-beta] plugin 1.19.x
 -   [terraform-provider-gsuite] plugin 0.1.x if GSuite functionality is desired
 
 ### Permissions
@@ -326,13 +327,12 @@ test steps non-interactively.
 #### Test configuration
 
 Each test-kitchen instance is configured with a `terraform.tfvars` file in the
-test fixture directory.
+test fixture directory. For convenience, these are symlinked to a single shared file:
 
 ```sh
-for instance in full minimal; do
-  cp "test/fixtures/$instance/terraform.tfvars.example" \
-    "test/fixtures/$instance/terraform.tfvars"
-  $EDITOR "test/fixtures/$instance/terraform.tfvars"
+cp "test/fixtures/shared/terraform.tfvars.example" \
+  "test/fixtures/shared/terraform.tfvars"
+$EDITOR "test/fixtures/shared/terraform.tfvars"
 done
 ```
 
@@ -397,6 +397,7 @@ versions][release-new-version].
 
 [gsuite-enabled-module]: modules/gsuite_enabled/README.md
 [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google
+[terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta
 [terraform-provider-gsuite]: https://github.com/DeviaVir/terraform-provider-gsuite
 [glossary]: /docs/GLOSSARY.md
-[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions
+[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions

docs/upgrading_to_project_factory_v1.0.md

@@ -89,6 +89,9 @@ index d876954..ebb3b1e 100755
    org_id             = "${var.org_id}"
 ```
 
+Additionally, `org_id` is now required so you will need to add
+it as an argument if you didn't already specify it on your projects.
+
 ### Download the state migration script
 
  ```
@@ -154,7 +157,8 @@ terraform plan -state terraform.tfstate.new
 
 The G Suite refactor adds an additional IAM membership and needs to re-create
 two resources, due to how resources were split up between the `gsuite_enabled`
-and `core_project_factory` modules.
+and `core_project_factory` modules. Depending on the version
+you are upgrading from, it might also add a `null_resource` for `preconditions` checks.
 
 ```txt
 
@@ -260,6 +264,20 @@ After restoring remote state, you need to re-initialize Terraform and push your
 terraform init -force-copy
 ```
 
+### Clean up
+
+Once you are done with the migration, you can safely remove `migrate.py`.
+
+```
+rm migrate.py
+```
+
+If you are using remote state, you can also remove the local state copies.
+
+```
+rm -rf terraform.tfstate*
+```
+
 ## Troubleshooting
 
 ### Errors about invalid arguments
@@ -275,6 +293,20 @@ Error: module "project-pfactory-development": "create_group" is not a valid argu
 These are related to projects which depend on G Suite functionality.
 Make sure to update the source of such projects to point to the [G Suite module](../modules/gsuite_enabled)
 
+### Missing `org_id`
+
+If your existing configuration doesn't specify the `org_id`,
+you might see some errors on upgrade:
+
+```
+Initializing the backend...
+
+Error: module "project_factory": missing required argument "org_id"
+```
+
+The fix for this is to explicitly set the `org_id` argument
+on your projects.
+
 ### The migration script fails to run
 
 If you get an error like this when running the migration script, it means you need upgrade

examples/gke_shared_vpc/main.tf

@@ -23,6 +23,11 @@ provider "google" {
   version     = "~> 1.19"
 }
 
+provider "google-beta" {
+  credentials = "${file(local.credentials_file_path)}"
+  version     = "~> 1.19"
+}
+
 module "project-factory" {
   source             = "../../"
   random_project_id  = "true"