From af62340e5bd057df5f5e71d3c4255dd32528286b Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Wed, 19 Dec 2018 17:40:50 -0800 Subject: [PATCH 001/105] Add v1.0.0 migration document --- docs/upgrading_to_project_factory_v1.0.md | 317 ++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 docs/upgrading_to_project_factory_v1.0.md diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md new file mode 100644 index 00000000..acd3d5cc --- /dev/null +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -0,0 +1,317 @@ +# Upgrading to Project Factory v1.0 + +The v1.0 release of Project Factory is a backwards incompatible release and +features significant changes, specifically with how GSuite resources are +managed. A state migration script is provided to update your existing states +to minimize or eliminate resource re-creations. + +## Migration Instructions + +### Without GSuite + +This migration was performed with the following example configuration. + +```hcl +/// @file simple_project/main.tf + +provider "google" { + credentials = "${file(var.credentials_path)}" +} + +provider "gsuite" { + credentials = "${file(var.credentials_path)}" + impersonated_user_email = "disabled" +} + +module "project-factory" { + source = "terraform-google-modules/project-factory/google" + version = "v0.3.0" + random_project_id = "true" + name = "pf-gsuite-migrate-simple" + org_id = "${var.org_id}" + billing_account = "${var.billing_account}" + credentials_path = "${var.credentials_path}" +} +``` + +#### 1. Update the project-factory source and re-initialize Terraform + +Update the project-factory module source to the Project Factory v1.0.0 release: +```diff +diff --git i/simple_project/main.tf w/simple_project/main.tf +index d876954..ebb3b1e 100755 +--- i/simple_project/main.tf ++++ w/simple_project/main.tf +@@ -33,7 +33,7 @@ provider "gsuite" { + + module "project-factory" { + source = "terraform-google-modules/project-factory/google" +- version = "v0.3.0" ++ version = "v1.0.0" + random_project_id = "true" + name = "pf-gsuite-migrate-simple" + org_id = "${var.org_id}" +``` + +#### 2. Reinitialize Terraform + +``` +terraform init -upgrade +``` + +#### 3. Download the state migration script + +``` +curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py +chmod +x migrate.py +``` + +#### 4. Migrate the Terraform state to match the new Project Factory module structure + +``` +./migrate.py terraform.tfstate terraform.tfstate.new +``` + +Expected output: +```txt +cp terraform.tfstate terraform.tfstate.new +---- Migrating the following project-factory modules: +-- module.project-factory +Moved module.project-factory.random_id.random_project_id_suffix to module.project-factory.module.project-factory.random_id.random_project_id_suffix +Moved module.project-factory.google_project.project to module.project-factory.module.project-factory.google_project.project +Moved module.project-factory.google_project_service.project_services to module.project-factory.module.project-factory.google_project_service.project_services +Moved module.project-factory.null_resource.delete_default_compute_service_account to module.project-factory.module.project-factory.null_resource.delete_default_compute_service_account +Moved module.project-factory.google_service_account.default_service_account to module.project-factory.module.project-factory.google_service_account.default_service_account +State migration complete, verify migration with `terraform plan -state 'terraform.tfstate.new'` +``` + +#### 5. Check that terraform does not detect changes + +``` +terraform plan -state terraform.tfstate.new +``` + +Expected output: +```txt +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +random_id.random_project_id_suffix: Refreshing state... (ID: l3A) +google_project.project: Refreshing state... (ID: pf-gsuite-migrate-simple-9770) +data.google_organization.org: Refreshing state... +google_service_account.default_service_account: Refreshing state... (ID: projects/pf-gsuite-migrate-simple-9770/...te-simple-9770.iam.gserviceaccount.com) +google_project_service.project_services: Refreshing state... (ID: pf-gsuite-migrate-simple-9770/compute.googleapis.com) +data.google_compute_default_service_account.default: Refreshing state... +null_resource.delete_default_compute_service_account: Refreshing state... (ID: 4378457466829456087) + +------------------------------------------------------------------------ + +No changes. Infrastructure is up-to-date. + +This means that Terraform did not detect any differences between your +configuration and real physical resources that exist. As a result, no +actions need to be performed. +``` + +#### 6. Back up the old Terraform state and switch to the new Terraform state + +If the Terraform plan suggests no changes, replace the old state file with the new state file. + +``` +mv terraform.tfstate terraform.tfstate.pre-migrate +mv terraform.tfstate.new terraform.tfstate +``` + +### With GSuite Integration + +This migration was performed with the following example configuration. +```hcl +/// @file group_project/main.tf +provider "google" { + credentials = "${file(var.credentials_path)}" +} + +provider "gsuite" { + credentials = "${file(var.credentials_path)}" + impersonated_user_email = "${var.admin_email}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.member" + ] +} + +module "project-factory" { + source = "terraform-google-modules/project-factory/google" + version = "v0.3.0" + random_project_id = "true" + name = "pf-gsuite-migrate-group" + org_id = "${var.org_id}" + billing_account = "${var.billing_account}" + credentials_path = "${var.credentials_path}" + create_group = "true" + group_name = "${var.project_group_name}" + api_sa_group = "${var.api_sa_group}" + shared_vpc = "${var.shared_vpc}" + shared_vpc_subnets = "${var.shared_vpc_subnets}" +} +``` + +#### 1. Update the project-factory source and re-initialize Terraform + +Update the project-factory module source to the v1.0.0 release and the gsuite_enabled module: + +```diff +diff --git i/group_project/main.tf w/group_project/main.tf +index 7f8c3d0..e026b19 100755 +--- i/group_project/main.tf ++++ w/group_project/main.tf +@@ -32,8 +32,8 @@ provider "gsuite" { + } + + module "project-factory" { +- source = "terraform-google-modules/project-factory/google" +- version = "v0.3.0" ++ source = "terraform-google-modules/project-factory/google//modules/gsuite_enabled" ++ version = "v1.0.0" + random_project_id = "true" + name = "pf-gsuite-migrate-group" + org_id = "${var.org_id}" +``` + +#### 2. Reinitialize Terraform + +``` +terraform init -upgrade +``` + +#### 3. Download the state migration script + +``` +curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py +chmod +x migrate.py +``` + +#### 4. Migrate the Terraform state to match the new Project Factory module structure + +``` +./migrate.py terraform.tfstate terraform.tfstate.new +``` + +Expected output: +```txt +cp terraform.tfstate terraform.tfstate.new +---- Migrating the following project-factory modules: +-- module.project-factory +Moved module.project-factory.random_id.random_project_id_suffix to module.project-factory.module.project-factory.random_id.random_project_id_suffix +Moved module.project-factory.google_project.project to module.project-factory.module.project-factory.google_project.project +Moved module.project-factory.google_project_service.project_services to module.project-factory.module.project-factory.google_project_service.project_services +Moved module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment to module.project-factory.module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment +Moved module.project-factory.null_resource.delete_default_compute_service_account to module.project-factory.module.project-factory.null_resource.delete_default_compute_service_account +Moved module.project-factory.google_service_account.default_service_account to module.project-factory.module.project-factory.google_service_account.default_service_account +Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] +Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] +Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] +Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0] +Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1] +Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2] +Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0] +Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1] +Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2] +``` + +#### 5. Check that terraform plans for expected changes + +``` +terraform plan -state terraform.tfstate.new +``` + +The GSuite refactor adds an additional IAM membership and needs to re-create +two resources, due to how resources were split up between the `gsuite_enabled` +and `core_project_factory` modules. + +```txt + +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +random_id.random_project_id_suffix: Refreshing state... (ID: I9w) +google_project.project: Refreshing state... (ID: pf-gsuite-migrate-group-23dc) +data.google_organization.org: Refreshing state... +data.google_organization.org: Refreshing state... +gsuite_group.group: Refreshing state... (ID: 03oy7u293zvtv3v) +google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[1]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...up:pf-gsuite-migrate-group@phoogle.net) +google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[2]: Refreshing state... (ID: projects/thebo-host-c15e/regions/europe...up:pf-gsuite-migrate-group@phoogle.net) +google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[0]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...up:pf-gsuite-migrate-group@phoogle.net) +data.google_compute_default_service_account.default: Refreshing state... +gsuite_group_member.api_s_account_api_sa_group_member: Refreshing state... (ID: 118009464384327601974) +google_service_account.default_service_account: Refreshing state... (ID: projects/pf-gsuite-migrate-group-23dc/s...ate-group-23dc.iam.gserviceaccount.com) +google_project_service.project_services: Refreshing state... (ID: pf-gsuite-migrate-group-23dc/compute.googleapis.com) +google_project_iam_member.gsuite_group_role: Refreshing state... (ID: pf-gsuite-migrate-group-23dc/roles/edit...up:pf-gsuite-migrate-group@phoogle.net) +google_service_account_iam_member.service_account_grant_to_group: Refreshing state... (ID: projects/pf-gsuite-migrate-group-23dc/s...up:pf-gsuite-migrate-group@phoogle.net) +google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...ate-group-23dc.iam.gserviceaccount.com) +google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...ate-group-23dc.iam.gserviceaccount.com) +google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2]: Refreshing state... (ID: projects/thebo-host-c15e/regions/europe...ate-group-23dc.iam.gserviceaccount.com) +google_project_iam_member.controlling_group_vpc_membership[2]: Refreshing state... (ID: thebo-host-c15e/roles/compute.networkUs...3246@cloudservices.gserviceaccount.com) +google_project_iam_member.controlling_group_vpc_membership[1]: Refreshing state... (ID: thebo-host-c15e/roles/compute.networkUs...up:pf-gsuite-migrate-group@phoogle.net) +google_project_iam_member.controlling_group_vpc_membership[0]: Refreshing state... (ID: thebo-host-c15e/roles/compute.networkUs...ate-group-23dc.iam.gserviceaccount.com) +google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2]: Refreshing state... (ID: projects/thebo-host-c15e/regions/europe...3246@cloudservices.gserviceaccount.com) +google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...3246@cloudservices.gserviceaccount.com) +google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1]: Refreshing state... (ID: projects/thebo-host-c15e/regions/us-wes...3246@cloudservices.gserviceaccount.com) +google_compute_shared_vpc_service_project.shared_vpc_attachment: Refreshing state... (ID: thebo-host-c15e/pf-gsuite-migrate-group-23dc) +null_resource.delete_default_compute_service_account: Refreshing state... (ID: 3094237059262702049) + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create +-/+ destroy and then create replacement + +Terraform will perform the following actions: + ++ module.project-factory.google_project_iam_member.controlling_group_vpc_membership + id: + etag: + member: "group:pf-gsuite-migrate-group@phoogle.net" + project: "thebo-host-c15e" + role: "roles/compute.networkUser" + +-/+ module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] (new resource required) + id: "thebo-host-c15e/roles/compute.networkUser/group:pf-gsuite-migrate-group@phoogle.net" => (forces new resource) + etag: "BwV9O377mQw=" => + member: "group:pf-gsuite-migrate-group@phoogle.net" => "serviceAccount:303471683246@cloudservices.gserviceaccount.com" (forces new resource) + project: "thebo-host-c15e" => "thebo-host-c15e" + role: "roles/compute.networkUser" => "roles/compute.networkUser" + +-/+ module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] (new resource required) + id: "thebo-host-c15e/roles/compute.networkUser/serviceAccount:303471683246@cloudservices.gserviceaccount.com" => (forces new resource) + etag: "BwV9O377mQw=" => + member: "serviceAccount:303471683246@cloudservices.gserviceaccount.com" => "serviceAccount:project-service-account@pf-gsuite-migrate-group-23dc.iam.gserviceaccount.com" (forces new resource) + project: "thebo-host-c15e" => "thebo-host-c15e" + role: "roles/compute.networkUser" => "roles/compute.networkUser" +Plan: 3 to add, 0 to change, 2 to destroy. + +------------------------------------------------------------------------ + +Note: You didn't specify an "-out" parameter to save this plan, so Terraform +can't guarantee that exactly these actions will be performed if +"terraform apply" is subsequently run. +``` + +#### 6. Back up the old Terraform state and switch to the new Terraform state + +If `terraform plan` suggests the above changes, replace the old state file with the new state file. + +``` +mv terraform.tfstate terraform.tfstate.pre-migrate +mv terraform.tfstate.new terraform.tfstate +``` + +#### 7. Run Terraform to reconcile any differences + +``` +terraform apply +``` From a89d266cb1e1f3992f4ddf6540631c1c9180b4a0 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 3 Jan 2019 16:45:22 -0500 Subject: [PATCH 002/105] Work on unifying migration instructions. --- docs/upgrading_to_project_factory_v1.0.md | 56 +++++++++++++++++------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index acd3d5cc..580bcac8 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -1,18 +1,19 @@ # Upgrading to Project Factory v1.0 The v1.0 release of Project Factory is a backwards incompatible release and -features significant changes, specifically with how GSuite resources are +features significant changes, specifically with how G Suite resources are managed. A state migration script is provided to update your existing states to minimize or eliminate resource re-creations. ## Migration Instructions -### Without GSuite - This migration was performed with the following example configuration. +Note that the "project-factory" module does not require G Suite functionality, +while the "project-factory-gsuite" module does. + ```hcl -/// @file simple_project/main.tf +/// @file main.tf provider "google" { credentials = "${file(var.credentials_path)}" @@ -20,7 +21,12 @@ provider "google" { provider "gsuite" { credentials = "${file(var.credentials_path)}" - impersonated_user_email = "disabled" + impersonated_user_email = "${var.admin_email}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.member" + ] } module "project-factory" { @@ -32,18 +38,33 @@ module "project-factory" { billing_account = "${var.billing_account}" credentials_path = "${var.credentials_path}" } + +module "project-factory-gsuite" { + source = "terraform-google-modules/project-factory/google" + version = "v0.3.0" + random_project_id = "true" + name = "pf-gsuite-migrate-group" + org_id = "${var.org_id}" + billing_account = "${var.billing_account}" + credentials_path = "${var.credentials_path}" + create_group = "true" + group_name = "${var.project_group_name}" + api_sa_group = "${var.api_sa_group}" + shared_vpc = "${var.shared_vpc}" + shared_vpc_subnets = "${var.shared_vpc_subnets}" +} ``` -#### 1. Update the project-factory source and re-initialize Terraform +### 1. Update the project-factory source Update the project-factory module source to the Project Factory v1.0.0 release: ```diff -diff --git i/simple_project/main.tf w/simple_project/main.tf +diff --git i/main.tf w/main.tf index d876954..ebb3b1e 100755 ---- i/simple_project/main.tf -+++ w/simple_project/main.tf -@@ -33,7 +33,7 @@ provider "gsuite" { - +--- i/main.tf ++++ w/main.tf +@@ -14,7 +14,7 @@ provider "gsuite" { + module "project-factory" { source = "terraform-google-modules/project-factory/google" - version = "v0.3.0" @@ -51,15 +72,24 @@ index d876954..ebb3b1e 100755 random_project_id = "true" name = "pf-gsuite-migrate-simple" org_id = "${var.org_id}" +@@ -24,7 +24,7 @@ module "project-factory" { + + module "project-factory-gsuite" { + source = "terraform-google-modules/project-factory/google" +- version = "v0.3.0" ++ version = "v1.0.0" + random_project_id = "true" + name = "pf-gsuite-migrate-group" + org_id = "${var.org_id}" ``` -#### 2. Reinitialize Terraform +### 2. Reinitialize Terraform ``` terraform init -upgrade ``` -#### 3. Download the state migration script +### 3. Download the state migration script ``` curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py From 0f84b778a97eab06a906a92f51b5fbe75aa1fc5d Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 3 Jan 2019 17:15:53 -0500 Subject: [PATCH 003/105] Add some troubleshooting notes... temporarily borked --- docs/upgrading_to_project_factory_v1.0.md | 211 +++++++--------------- 1 file changed, 68 insertions(+), 143 deletions(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 580bcac8..c3cd4c5e 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -5,6 +5,8 @@ features significant changes, specifically with how G Suite resources are managed. A state migration script is provided to update your existing states to minimize or eliminate resource re-creations. +Note that upgrading requires you to have Python 3.7 installed. + ## Migration Instructions This migration was performed with the following example configuration. @@ -55,9 +57,12 @@ module "project-factory-gsuite" { } ``` -### 1. Update the project-factory source +### Update the project-factory source + +Update the project-factory module source to the Project Factory v1.0.0 release. + +Note that any projects which depend on G Suite features must be updated to point to the [gsuite-enabled submodule](../modules/gsuite_enabled). -Update the project-factory module source to the Project Factory v1.0.0 release: ```diff diff --git i/main.tf w/main.tf index d876954..ebb3b1e 100755 @@ -72,31 +77,27 @@ index d876954..ebb3b1e 100755 random_project_id = "true" name = "pf-gsuite-migrate-simple" org_id = "${var.org_id}" -@@ -24,7 +24,7 @@ module "project-factory" { +@@ -24,8 +24,8 @@ module "project-factory" { module "project-factory-gsuite" { - source = "terraform-google-modules/project-factory/google" +- source = "terraform-google-modules/project-factory/google" - version = "v0.3.0" ++ source = "terraform-google-modules/project-factory/google//modules/gsuite_enabled" + version = "v1.0.0" random_project_id = "true" name = "pf-gsuite-migrate-group" org_id = "${var.org_id}" ``` -### 2. Reinitialize Terraform - -``` -terraform init -upgrade -``` +### Locally download your Terraform state -### 3. Download the state migration script +This step is only required if you are using [remote state](https://www.terraform.io/docs/state/remote.html). ``` -curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py -chmod +x migrate.py +terraform state pull >> terraform.tfstate ``` -#### 4. Migrate the Terraform state to match the new Project Factory module structure +#### Migrate the Terraform state to match the new Project Factory module structure ``` ./migrate.py terraform.tfstate terraform.tfstate.new @@ -115,143 +116,23 @@ Moved module.project-factory.google_service_account.default_service_account to m State migration complete, verify migration with `terraform plan -state 'terraform.tfstate.new'` ``` -#### 5. Check that terraform does not detect changes +#### Temporarily disable remote state -``` -terraform plan -state terraform.tfstate.new -``` +If you have a remote state backend configured, you should temporarily disable it: -Expected output: -```txt -Refreshing Terraform state in-memory prior to plan... -The refreshed state will be used to calculate this plan, but will not be -persisted to local or remote state storage. - -random_id.random_project_id_suffix: Refreshing state... (ID: l3A) -google_project.project: Refreshing state... (ID: pf-gsuite-migrate-simple-9770) -data.google_organization.org: Refreshing state... -google_service_account.default_service_account: Refreshing state... (ID: projects/pf-gsuite-migrate-simple-9770/...te-simple-9770.iam.gserviceaccount.com) -google_project_service.project_services: Refreshing state... (ID: pf-gsuite-migrate-simple-9770/compute.googleapis.com) -data.google_compute_default_service_account.default: Refreshing state... -null_resource.delete_default_compute_service_account: Refreshing state... (ID: 4378457466829456087) - ------------------------------------------------------------------------- - -No changes. Infrastructure is up-to-date. - -This means that Terraform did not detect any differences between your -configuration and real physical resources that exist. As a result, no -actions need to be performed. -``` - -#### 6. Back up the old Terraform state and switch to the new Terraform state - -If the Terraform plan suggests no changes, replace the old state file with the new state file. - -``` -mv terraform.tfstate terraform.tfstate.pre-migrate -mv terraform.tfstate.new terraform.tfstate -``` - -### With GSuite Integration - -This migration was performed with the following example configuration. ```hcl -/// @file group_project/main.tf -provider "google" { - credentials = "${file(var.credentials_path)}" -} - -provider "gsuite" { - credentials = "${file(var.credentials_path)}" - impersonated_user_email = "${var.admin_email}" - - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.group.member" - ] -} - -module "project-factory" { - source = "terraform-google-modules/project-factory/google" - version = "v0.3.0" - random_project_id = "true" - name = "pf-gsuite-migrate-group" - org_id = "${var.org_id}" - billing_account = "${var.billing_account}" - credentials_path = "${var.credentials_path}" - create_group = "true" - group_name = "${var.project_group_name}" - api_sa_group = "${var.api_sa_group}" - shared_vpc = "${var.shared_vpc}" - shared_vpc_subnets = "${var.shared_vpc_subnets}" -} +# terraform { +# backend "gcs" { +# bucket = "my-bucket-name" +# prefix = "terraform/state/projects" +# } +# } ``` -#### 1. Update the project-factory source and re-initialize Terraform - -Update the project-factory module source to the v1.0.0 release and the gsuite_enabled module: +After commenting out your remote state configuration, you must re-initialize Terraform. -```diff -diff --git i/group_project/main.tf w/group_project/main.tf -index 7f8c3d0..e026b19 100755 ---- i/group_project/main.tf -+++ w/group_project/main.tf -@@ -32,8 +32,8 @@ provider "gsuite" { - } - - module "project-factory" { -- source = "terraform-google-modules/project-factory/google" -- version = "v0.3.0" -+ source = "terraform-google-modules/project-factory/google//modules/gsuite_enabled" -+ version = "v1.0.0" - random_project_id = "true" - name = "pf-gsuite-migrate-group" - org_id = "${var.org_id}" -``` -#### 2. Reinitialize Terraform - -``` -terraform init -upgrade -``` - -#### 3. Download the state migration script - -``` -curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py -chmod +x migrate.py -``` - -#### 4. Migrate the Terraform state to match the new Project Factory module structure - -``` -./migrate.py terraform.tfstate terraform.tfstate.new -``` - -Expected output: -```txt -cp terraform.tfstate terraform.tfstate.new ----- Migrating the following project-factory modules: --- module.project-factory -Moved module.project-factory.random_id.random_project_id_suffix to module.project-factory.module.project-factory.random_id.random_project_id_suffix -Moved module.project-factory.google_project.project to module.project-factory.module.project-factory.google_project.project -Moved module.project-factory.google_project_service.project_services to module.project-factory.module.project-factory.google_project_service.project_services -Moved module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment to module.project-factory.module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment -Moved module.project-factory.null_resource.delete_default_compute_service_account to module.project-factory.module.project-factory.null_resource.delete_default_compute_service_account -Moved module.project-factory.google_service_account.default_service_account to module.project-factory.module.project-factory.google_service_account.default_service_account -Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] -Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] -Moved module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] to module.project-factory.module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] -Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0] -Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1] -Moved module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2] -Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0] -Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1] -Moved module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2] to module.project-factory.module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2] -``` - -#### 5. Check that terraform plans for expected changes +#### Check that terraform plans for expected changes ``` terraform plan -state terraform.tfstate.new @@ -333,6 +214,15 @@ can't guarantee that exactly these actions will be performed if #### 6. Back up the old Terraform state and switch to the new Terraform state +If the Terraform plan suggests no changes, replace the old state file with the new state file. + +``` +mv terraform.tfstate terraform.tfstate.pre-migrate +mv terraform.tfstate.new terraform.tfstate +``` + +#### 6. Back up the old Terraform state and switch to the new Terraform state + If `terraform plan` suggests the above changes, replace the old state file with the new state file. ``` @@ -345,3 +235,38 @@ mv terraform.tfstate.new terraform.tfstate ``` terraform apply ``` + +## Troubleshooting + +### Errors about invalid arguments + +You might get errors about invalid arguments, such as: + +``` +Error: module "project-pfactory-development": "sa_group" is not a valid argument +Error: module "project-pfactory-development": "api_sa_group" is not a valid argument +Error: module "project-pfactory-development": "create_group" is not a valid argument +``` + +These are related to projects which depend on G Suite functionality. +Make sure to update the source of such projects to point to the [G Suite module](../modules/gsuite_enabled) + +### The migration script fails to run + +If you get an error like this when running the migration script, it means you need upgrade +your Python version to 3.7. + +``` +Traceback (most recent call last): + File "./migrate.py", line 372, in + main(sys.argv) + File "./migrate.py", line 353, in main + migrate(args.newstate, dryrun=args.dryrun) + File "./migrate.py", line 314, in migrate + for path in read_state(statefile) + File "./migrate.py", line 282, in read_state + encoding='utf-8') + File "/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/subprocess.py", line 403, in run + with Popen(*popenargs, **kwargs) as process: +TypeError: __init__() got an unexpected keyword argument 'capture_output' +``` From aee71c2a86b3386716175de0464b1f6842642706 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 3 Jan 2019 17:24:35 -0500 Subject: [PATCH 004/105] Unify instructions --- docs/upgrading_to_project_factory_v1.0.md | 67 ++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index c3cd4c5e..9974e0aa 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -89,6 +89,12 @@ index d876954..ebb3b1e 100755 org_id = "${var.org_id}" ``` +### Reinitialize Terraform + +``` +terraform init -upgrade +``` + ### Locally download your Terraform state This step is only required if you are using [remote state](https://www.terraform.io/docs/state/remote.html). @@ -97,6 +103,23 @@ This step is only required if you are using [remote state](https://www.terraform terraform state pull >> terraform.tfstate ``` +You should then disable remote state temporarily: + +```hcl +# terraform { +# backend "gcs" { +# bucket = "my-bucket-name" +# prefix = "terraform/state/projects" +# } +# } +``` + +After commenting out your remote state configuration, you must re-initialize Terraform. + +``` +terraform init +``` + #### Migrate the Terraform state to match the new Project Factory module structure ``` @@ -116,22 +139,6 @@ Moved module.project-factory.google_service_account.default_service_account to m State migration complete, verify migration with `terraform plan -state 'terraform.tfstate.new'` ``` -#### Temporarily disable remote state - -If you have a remote state backend configured, you should temporarily disable it: - -```hcl -# terraform { -# backend "gcs" { -# bucket = "my-bucket-name" -# prefix = "terraform/state/projects" -# } -# } -``` - -After commenting out your remote state configuration, you must re-initialize Terraform. - - #### Check that terraform plans for expected changes ``` @@ -212,28 +219,38 @@ can't guarantee that exactly these actions will be performed if "terraform apply" is subsequently run. ``` -#### 6. Back up the old Terraform state and switch to the new Terraform state +### Back up the old Terraform state and switch to the new Terraform state -If the Terraform plan suggests no changes, replace the old state file with the new state file. +If the Terraform plan suggests no changes or only the IAM changes above, replace the old state file with the new state file. ``` mv terraform.tfstate terraform.tfstate.pre-migrate mv terraform.tfstate.new terraform.tfstate ``` -#### 6. Back up the old Terraform state and switch to the new Terraform state - -If `terraform plan` suggests the above changes, replace the old state file with the new state file. +### Run Terraform to reconcile any differences ``` -mv terraform.tfstate terraform.tfstate.pre-migrate -mv terraform.tfstate.new terraform.tfstate +terraform apply ``` -#### 7. Run Terraform to reconcile any differences +### Re-enable remote state + +You should then re-enable remote state: +```hcl +terraform { + backend "gcs" { + bucket = "my-bucket-name" + prefix = "terraform/state/projects" + } +} ``` -terraform apply + +After restoring remote state, you need to re-initialize Terraform and push your local state to the remote: + +``` +terraform init -force-copy ``` ## Troubleshooting From e178be626e6e00c5b4e39558c036a0d93b1cc6ec Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 3 Jan 2019 17:25:30 -0500 Subject: [PATCH 005/105] Update upgrading_to_project_factory_v1.0.md --- docs/upgrading_to_project_factory_v1.0.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 9974e0aa..b6167b2a 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -120,7 +120,7 @@ After commenting out your remote state configuration, you must re-initialize Ter terraform init ``` -#### Migrate the Terraform state to match the new Project Factory module structure +### Migrate the Terraform state to match the new Project Factory module structure ``` ./migrate.py terraform.tfstate terraform.tfstate.new @@ -139,13 +139,13 @@ Moved module.project-factory.google_service_account.default_service_account to m State migration complete, verify migration with `terraform plan -state 'terraform.tfstate.new'` ``` -#### Check that terraform plans for expected changes +### Check that terraform plans for expected changes ``` terraform plan -state terraform.tfstate.new ``` -The GSuite refactor adds an additional IAM membership and needs to re-create +The G Suite refactor adds an additional IAM membership and needs to re-create two resources, due to how resources were split up between the `gsuite_enabled` and `core_project_factory` modules. From 7fa06ffdfd8b2524df3481c31c5f2b7e318fd810 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Tue, 31 Jul 2018 12:25:11 -0700 Subject: [PATCH 006/105] adding gke shared vpc functionality and some cleanup --- examples/gke_shared_vpc/README.md | 2 +- examples/gke_shared_vpc/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/gke_shared_vpc/README.md b/examples/gke_shared_vpc/README.md index 772842ba..cb3a4772 100644 --- a/examples/gke_shared_vpc/README.md +++ b/examples/gke_shared_vpc/README.md @@ -32,4 +32,4 @@ More information about GKE with Shared VPC can be found here: https://cloud.goog | shared\_vpc | The ID of the host project which hosts the shared VPC | string | - | yes | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$PROJECT_ID/regions/$REGION/subnetworks/$SUBNET_ID) | list | `` | no | -[^]: (autogen_docs_end) \ No newline at end of file +[^]: (autogen_docs_end) diff --git a/examples/gke_shared_vpc/main.tf b/examples/gke_shared_vpc/main.tf index 738c4b46..8948a26e 100644 --- a/examples/gke_shared_vpc/main.tf +++ b/examples/gke_shared_vpc/main.tf @@ -20,7 +20,7 @@ locals { provider "google" { credentials = "${file(local.credentials_file_path)}" - version = "~> 1.19" + version = "~> 1.19" } module "project-factory" { From 56322b22c983c360d747bddc2b9157917e2b909e Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Thu, 23 Aug 2018 14:00:31 -0700 Subject: [PATCH 007/105] first pass at making gsuite optional --- examples/group_project/main.tf | 14 +- examples/group_project/variables.tf | 5 + main.tf | 365 +---------------- modules/core_project_factory/README.md | 230 +++++++++++ modules/core_project_factory/main.tf | 368 ++++++++++++++++++ modules/core_project_factory/outputs.tf | 73 ++++ .../scripts}/add-iam-policy-to-subnet.sh | 0 .../scripts}/delete-default-network.sh | 0 .../scripts}/delete-service-account.sh | 0 .../scripts}/get-org-id.sh | 0 .../scripts}/get-subnets-regions.sh | 0 .../scripts}/preconditions.sh | 0 .../scripts}/preconditions/preconditions.py | 0 .../scripts}/preconditions/requirements.txt | 0 modules/core_project_factory/variables.tf | 126 ++++++ modules/gsuite_enabled/README.md | 0 .../gsuite_enabled/main.tf | 65 +++- modules/gsuite_enabled/variables.tf | 126 ++++++ outputs.tf | 24 +- variables.tf | 15 - 20 files changed, 1017 insertions(+), 394 deletions(-) mode change 100755 => 100644 examples/group_project/main.tf create mode 100644 modules/core_project_factory/README.md create mode 100644 modules/core_project_factory/main.tf create mode 100644 modules/core_project_factory/outputs.tf rename {scripts => modules/core_project_factory/scripts}/add-iam-policy-to-subnet.sh (100%) rename {scripts => modules/core_project_factory/scripts}/delete-default-network.sh (100%) rename {scripts => modules/core_project_factory/scripts}/delete-service-account.sh (100%) rename {scripts => modules/core_project_factory/scripts}/get-org-id.sh (100%) rename {scripts => modules/core_project_factory/scripts}/get-subnets-regions.sh (100%) rename {scripts => modules/core_project_factory/scripts}/preconditions.sh (100%) rename {scripts => modules/core_project_factory/scripts}/preconditions/preconditions.py (100%) rename {scripts => modules/core_project_factory/scripts}/preconditions/requirements.txt (100%) create mode 100644 modules/core_project_factory/variables.tf create mode 100644 modules/gsuite_enabled/README.md rename gsuite_override.tf => modules/gsuite_enabled/main.tf (53%) create mode 100644 modules/gsuite_enabled/variables.tf diff --git a/examples/group_project/main.tf b/examples/group_project/main.tf old mode 100755 new mode 100644 index 3a2ff1ad..60db215a --- a/examples/group_project/main.tf +++ b/examples/group_project/main.tf @@ -23,19 +23,7 @@ locals { *****************************************/ provider "google" { credentials = "${file(local.credentials_file_path)}" - version = "~> 1.19" -} - -provider "gsuite" { - credentials = "${file(local.credentials_file_path)}" - impersonated_user_email = "${var.admin_email}" - - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.group.member", - ] - - version = "~> 0.1.9" + version = "~> 1.19" } module "project-factory" { diff --git a/examples/group_project/variables.tf b/examples/group_project/variables.tf index bed6cd45..ac9e5df1 100755 --- a/examples/group_project/variables.tf +++ b/examples/group_project/variables.tf @@ -16,22 +16,27 @@ variable "admin_email" { description = "Admin user email on Gsuite" + default = "terraform-forseti-project-1@zamir-forseti-test.iam.gserviceaccount.com" } variable "credentials_file_path" { description = "Service account json auth path" + default = "/Users/cpearring/testing/serviceAccountKeys/orgserviceaccount.json" } variable "organization_id" { description = "The organization id for the associated services" + default = "826592752744" } variable "billing_account" { description = "The ID of the billing account to associate this project with" + default = "01E8A0-35F760-5CF02A" } variable "api_sa_group" { description = "An existing GSuite group email to place the Google APIs Service Account for the project in" + default = "api_sa_group@phoogle.net" } variable "project_group_name" { diff --git a/main.tf b/main.tf index 7972f966..1b6d9226 100755 --- a/main.tf +++ b/main.tf @@ -14,355 +14,26 @@ * limitations under the License. */ -/****************************************** - Project random id suffix configuration - *****************************************/ -resource "random_id" "random_project_id_suffix" { - byte_length = 2 -} - -/****************************************** - Locals configuration - *****************************************/ -locals { - project_id = "${google_project.project.project_id}" - project_number = "${google_project.project.number}" - project_org_id = "${var.folder_id != "" ? "" : var.org_id}" - project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" - temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" - domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" - s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" - api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" - api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" - gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" - gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" - gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" - project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" - create_bucket = "${var.bucket_project != "" ? "true" : "false"}" - gsuite_group = "${var.group_name != "" || var.create_group}" - app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - - shared_vpc_users = "${compact(list(local.s_account_fmt, data.null_data_source.data_group_email_format.outputs["group_fmt"], local.api_s_account_fmt, local.gke_s_account_fmt))}" - shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 - - app_engine_config = { - enabled = "${list(var.app_engine)}" - disabled = "${list()}" - } -} - -resource "null_resource" "args_missing" { - count = "${local.args_missing}" - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true -} - -/****************************************** - Group email to be used on resources - *****************************************/ -data "null_data_source" "data_final_group_email" { - inputs { - final_group_email = "${var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : ""}" - } -} - -/****************************************** - Group email formatting - *****************************************/ -data "null_data_source" "data_group_email_format" { - inputs { - group_fmt = "${data.null_data_source.data_final_group_email.outputs["final_group_email"] != "" ? format("group:%s", data.null_data_source.data_final_group_email.outputs["final_group_email"]) : ""}" - } -} +module "project-factory" { + source = "modules/core_project_factory" -/****************************************** - Organization info retrieval - *****************************************/ -data "google_organization" "org" { - count = "${var.org_id == "" ? 0 : 1}" - organization = "${var.org_id}" -} - -resource "null_resource" "preconditions" { - triggers { - credentials_path = "${var.credentials_path}" - billing_account = "${var.billing_account}" - org_id = "${var.org_id}" - folder_id = "${var.folder_id}" - shared_vpc = "${var.shared_vpc}" - } - - provisioner "local-exec" { - command = < 0)) ? local.shared_vpc_users_length : 0}" - - project = "${var.shared_vpc}" - role = "roles/compute.networkUser" - member = "${element(local.shared_vpc_users, count.index)}" - - depends_on = ["google_project_service.project_services"] -} - -/************************************************************************************* - compute.networkUser role granted to Project Service Account on vpc subnets - *************************************************************************************/ -resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" - - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" - role = "roles/compute.networkUser" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - project = "${var.shared_vpc}" - member = "${local.s_account_fmt}" -} - -/************************************************************************************* - compute.networkUser role granted to GSuite group on vpc subnets - *************************************************************************************/ -resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && local.gsuite_group ? length(var.shared_vpc_subnets) : 0 }" - - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" - role = "roles/compute.networkUser" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - project = "${var.shared_vpc}" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" -} - -/************************************************************************************* - compute.networkUser role granted to APIs Service Account on vpc subnets - *************************************************************************************/ -resource "google_compute_subnetwork_iam_member" "apis_service_account_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" - - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" - role = "roles/compute.networkUser" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - project = "${var.shared_vpc}" - member = "${local.api_s_account_fmt}" - - depends_on = ["google_project_service.project_services"] -} - -/*********************************************** - Usage report export (to bucket) configuration - ***********************************************/ -resource "google_project_usage_export_bucket" "usage_report_export" { - count = "${var.usage_bucket_name != "" ? 1 : 0}" - - project = "${local.project_id}" - bucket_name = "${var.usage_bucket_name}" - prefix = "${var.usage_bucket_prefix != "" ? var.usage_bucket_prefix : "usage-${local.project_id}"}" - - depends_on = ["google_project_service.project_services"] -} - -/*********************************************** - Project's bucket creation - ***********************************************/ -resource "google_storage_bucket" "project_bucket" { - count = "${local.create_bucket ? 1 : 0}" - - name = "${local.project_bucket_name}" - project = "${var.bucket_project}" -} - -/*********************************************** - Project's bucket storage.admin granting to group - ***********************************************/ -resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { - count = "${local.create_bucket && local.gsuite_group ? 1 : 0}" - - bucket = "${google_storage_bucket.project_bucket.name}" - role = "roles/storage.admin" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" -} - -/*********************************************** - Project's bucket storage.admin granting to default compute service account - ***********************************************/ -resource "google_storage_bucket_iam_member" "s_account_storage_admin_on_project_bucket" { - count = "${local.create_bucket ? 1 : 0}" - - bucket = "${google_storage_bucket.project_bucket.name}" - role = "roles/storage.admin" - member = "${local.s_account_fmt}" -} - -/*********************************************** - Project's bucket storage.admin granting to Google APIs service account - ***********************************************/ -resource "google_storage_bucket_iam_member" "api_s_account_storage_admin_on_project_bucket" { - count = "${local.create_bucket ? 1 : 0}" - - bucket = "${google_storage_bucket.project_bucket.name}" - role = "roles/storage.admin" - member = "${local.api_s_account_fmt}" - - depends_on = ["google_project_service.project_services"] -} - -/****************************************** - compute.networkUser role granted to GKE service account for GKE on shared VPC subnets - *****************************************/ -resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { - count = "${local.gke_shared_vpc_enabled && length(compact(var.shared_vpc_subnets)) != 0 ? length(var.shared_vpc_subnets) : 0}" - - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" - role = "roles/compute.networkUser" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - project = "${var.shared_vpc}" - member = "${local.gke_s_account_fmt}" - - depends_on = ["google_project_service.project_services"] -} - -/****************************************** - container.hostServiceAgentUser role granted to GKE service account for GKE on shared VPC - *****************************************/ -resource "google_project_iam_member" "gke_host_agent" { - count = "${local.gke_shared_vpc_enabled ? 1 : 0}" - - project = "${var.shared_vpc}" - role = "roles/container.hostServiceAgentUser" - member = "${local.gke_s_account_fmt}" - - depends_on = ["google_project_service.project_services"] + app_engine = "${var.app_engine}" } diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md new file mode 100644 index 00000000..5c85888f --- /dev/null +++ b/modules/core_project_factory/README.md @@ -0,0 +1,230 @@ +# Terraform Project Factory + +This module handles opinionated Google Cloud Platform project creation and configuration with Shared VPC, IAM, APIs, etc. + +## Requirements +### Terraform plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.10.x +- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.8.0 +- [terraform-provider-gsuite](https://github.com/DeviaVir/terraform-provider-gsuite) plugin + +### Configure a Service Account +In order to execute this module you must have a Service Account with the following: +#### Roles +The service account with the following roles: +- roles/resourcemanager.folderViewer on the folder that you want to create the project in +- roles/resourcemanager.organizationViewer on the organization +- roles/resourcemanager.projectCreator on the organization +- roles/billing.user on the organization +- roles/compute.xpnAdmin on the organization (if using a shared_vpc) +- roles/compute.networkAdmin on the organization +- roles/iam.serviceAccountAdmin on the organization +- roles/browser on the host project (if using a shared_vpc) +- roles/resourcemanager.projectIamAdmin on the host project (if using a shared_vpc) +- roles/storage.admin on bucket_project + +#### GSuite domain delegation +- Enable the G Suite Domain-wide Delegation on your service account + +#### Usage report +- Give enough permissions to Service Account on the bucket for reading and writing objects. + +### Enable API's +In order to operate with the Service Account you must activate the following API's on the base project where the Service Account was created: + +- Cloud Resource Manager API - cloudresourcemanager.googleapis.com +- Cloud Billing API - cloudbilling.googleapis.com +- Identity and Access Management API - iam.googleapis.com +- Admin SDK - admin.googleapis.com +- Google App Engine Admin API - appengine.googleapis.com + +### GSuite +#### Admin +- You will need an admin on your Google Admin site. + +#### API Client access +Give API client access to Service Account on your Google Admin site +- Go to Security > Settings > Advanced Settings > Manage API client access +- Add the client ID of your Service Account and the following scopes: + - https://www.googleapis.com/auth/admin.directory.group + - https://www.googleapis.com/auth/admin.directory.group.member + +## Install + +### Terraform +Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: +- https://releases.hashicorp.com/terraform/ + +### Terraform plugins +Be sure you have the compiled plugins on $HOME/.terraform.d/plugins/ + +- [terraform-provider-gsuite](https://github.com/DeviaVir/terraform-provider-gsuite) plugin 0.1.0 (there are not compatible releases, you have to compile it from master branch) + +See each plugin page for more information about how to compile and use them + +## Fast install (optional) +For a fast install, please configure the variables on init_centos.sh or init_debian.sh script in the helpers directory and then launch it. + +The script will do: +- Environment variables setting +- Installation of base packages like wget, curl, unzip, gcloud, etc. +- Installation of go 1.9.0 +- Installation of Terraform 0.10.x +- Download the terraform-provider-gsuite plugin +- Compile the terraform-provider-gsuite plugin +- Move the terraform-provider-gsuite to the right location + +## Scripted Setting of Permissions on Host Project Service Account (optional) +This will grant the service account in your host project the needed permissions. + +To use the script, run the following with the appropriate values: +- `helpers/set-host-sa-permissions.sh ''` + +## Usage +You can go to the examples folder, however the usage of the module could be like this in your own main.tf file: + +*Configure the provider here before the module invocation, see the examples folder* + + locals { + credentials_file_path = "" + } + + provider "google" { + credentials = "${file(local.credentials_file_path)}" + } + + provider "gsuite" { + credentials = "${file(local.credentials_file_path)}" + impersonated_user_email = "" + oauth_scopes = [ + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.member" + ] + } + module "project-factory" { + source = "" + name = "pf-test-1" + random_project_id = "true" + org_id = "1234567890" + usage_bucket_name = "pf-test-1-usage-report-bucket" + billing_account = "ABCDEF-ABCDEF-ABCDEF" + group_role = "roles/editor" + shared_vpc = "shared_vpc_host_name" + sa_group = "test_sa_group@yourdomain.com" + credentials_path = "${local.credentials_file_path}" + shared_vpc_subnets = [ + "projects/base-project-196723/regions/us-east1/subnetworks/default", + "projects/base-project-196723/regions/us-central1/subnetworks/default", + "projects/base-project-196723/regions/us-central1/subnetworks/subnet-1", + ] + } + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + +#### Variables +Please refer the /variables.tf file for the required and optional variables. + +#### Outputs +Please refer the /outputs.tf file for the outputs that you can get with the `terraform output` command + +## Infrastructure +The resources/services/activations/deletions that this module will create/trigger are: +- A Google project +- Specified API's activation on the project +- Shared VPC configuration (if provided) +- A default Service Account creation +- Make the Service Account member of your provided `sa_group` +- Deletion of the default compute service (via bash script) +- Deletion of the default network (via bash script) +- Creation of a GSuite group with your provided `group_name` if `created_group` is true +- Storage the usage reports to the specified bucket +- Bucket `bucket_name` creation on `bucket_project` + +The roles granted for the specific resources are: +- Service Account + - compute.networkUser on host project or specific subnets + - compute.instanceAdmin.v1 on project + - MEMBER on `sa_group` + - storage.admin on `bucket_name` + +- `group_name` + - compute.networkUser on host project or specific subnets + - `group_role` on project + - iam.serviceAccountUser on Service Account + - storage.admin on `bucket_name` + +- APIs Service Account + - compute.networkUser on host project or specific subnets + - MEMBER on `api_sa_group` + - storage.admin on `bucket_name` + +## File structure +The project has the following folders and files: + +- /: root folder +- /examples: examples for using this module +- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file) +- /test: Folders with files for testing the module (see Testing section on this file) +- /helpers: Optional helper scripts for ease of use +- /main.tf: main file for this module, contains all the resources to create +- /variables.tf: all the variables for the module +- /output.tf: the outputs of the module +- /readme.MD: this file + +## Testing + +### Requirements +- [bats](https://github.com/sstephenson/bats) 0.4.0 +- [jq](https://stedolan.github.io/jq/) 1.5 + +### Integration test +##### Terraform integration tests +The integration tests for this module are built with bats, basically the test checks the following: +- Perform `terraform init` command +- Perform `terraform get` command +- Perform `terraform plan` command and check that it'll create *n* resources, modify 0 resources and delete 0 resources +- Perform `terraform apply -auto-approve` command and check that it has created the *n* resources, modified 0 resources and deleted 0 resources +- Perform several `gcloud` commands and check the infrastructure is in the desired state +- Perform `terraform destroy -force` command and check that it has destroyed the *n* resources + +You can use the following command to run the integration test in the folder */test/integration/gcloud-test* + + `. launch.sh` + +### Linting +The makefile in this project will lint or sometimes just format any shell, +Python, golang, Terraform, or Dockerfiles. The linters will only be run if +the makefile finds files with the appropriate file extension. + +All of the linter checks are in the default make target, so you just have to +run + +``` +make -s +``` + +The -s is for 'silent'. Successful output looks like this + +``` +Running shellcheck +Running flake8 +Running gofmt +Running terraform validate +Running hadolint on Dockerfiles +Test passed - Verified all file Apache 2 headers +``` + +The linters +are as follows: +* Shell - shellcheck. Can be found in homebrew +* Python - flake8. Can be installed with 'pip install flake8' +* Golang - gofmt. gofmt comes with the standard golang installation. golang +is a compiled language so there is no standard linter. +* Terraform - terraform has a built-in linter in the 'terraform validate' +command. +* Dockerfiles - hadolint. Can be found in homebrew diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf new file mode 100644 index 00000000..8885f460 --- /dev/null +++ b/modules/core_project_factory/main.tf @@ -0,0 +1,368 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + Project random id suffix configuration + *****************************************/ +resource "random_id" "random_project_id_suffix" { + byte_length = 2 +} + +/****************************************** + Locals configuration + *****************************************/ +locals { + project_id = "${google_project.project.project_id}" + project_number = "${google_project.project.number}" + project_org_id = "${var.folder_id != "" ? "" : var.org_id}" + project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" + temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" + domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" + args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" + s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" + api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" + api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" + gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" + gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" + gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" + project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" + create_bucket = "${var.bucket_project != "" ? "true" : "false"}" + gsuite_group = "${var.group_name != "" || var.create_group}" + app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" + + shared_vpc_users = "${compact(list(local.s_account_fmt, data.null_data_source.data_group_email_format.outputs["group_fmt"], local.api_s_account_fmt, local.gke_s_account_fmt))}" + shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 + + app_engine_config = { + enabled = "${list(var.app_engine)}" + disabled = "${list()}" + } +} + +resource "null_resource" "args_missing" { + count = "${local.args_missing}" + "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true +} + +/****************************************** + Group email to be used on resources + *****************************************/ +data "null_data_source" "data_final_group_email" { + inputs { + final_group_email = "${var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : ""}" + } +} + +/****************************************** + Group email formatting + *****************************************/ +data "null_data_source" "data_group_email_format" { + inputs { + group_fmt = "${data.null_data_source.data_final_group_email.outputs["final_group_email"] != "" ? format("group:%s", data.null_data_source.data_final_group_email.outputs["final_group_email"]) : ""}" + } +} + +/****************************************** + Organization info retrieval + *****************************************/ +data "google_organization" "org" { + count = "${var.org_id == "" ? 0 : 1}" + organization = "${var.org_id}" +} + +resource "null_resource" "preconditions" { + triggers { + credentials_path = "${var.credentials_path}" + billing_account = "${var.billing_account}" + org_id = "${var.org_id}" + folder_id = "${var.folder_id}" + shared_vpc = "${var.shared_vpc}" + } + + provisioner "local-exec" { + command = < 0)) ? local.shared_vpc_users_length : 0}" + + project = "${var.shared_vpc}" + role = "roles/compute.networkUser" + member = "${element(local.shared_vpc_users, count.index)}" + + depends_on = ["google_project_service.project_services"] +} + +/************************************************************************************* + compute.networkUser role granted to Project Service Account on vpc subnets + *************************************************************************************/ +resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_subnets" { + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" + + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" + role = "roles/compute.networkUser" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + project = "${var.shared_vpc}" + member = "${local.s_account_fmt}" +} + +/************************************************************************************* + compute.networkUser role granted to GSuite group on vpc subnets + *************************************************************************************/ +resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && local.gsuite_group ? length(var.shared_vpc_subnets) : 0 }" + + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" + role = "roles/compute.networkUser" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + project = "${var.shared_vpc}" + member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" +} + +/************************************************************************************* + compute.networkUser role granted to APIs Service Account on vpc subnets + *************************************************************************************/ +resource "google_compute_subnetwork_iam_member" "apis_service_account_role_to_vpc_subnets" { + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" + + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" + role = "roles/compute.networkUser" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + project = "${var.shared_vpc}" + member = "${local.api_s_account_fmt}" + + depends_on = ["google_project_service.project_services"] +} + +/*********************************************** + Usage report export (to bucket) configuration + ***********************************************/ +resource "google_project_usage_export_bucket" "usage_report_export" { + count = "${var.usage_bucket_name != "" ? 1 : 0}" + + project = "${local.project_id}" + bucket_name = "${var.usage_bucket_name}" + prefix = "usage-${local.project_id}" + + depends_on = ["google_project_service.project_services"] +} + +/*********************************************** + Project's bucket creation + ***********************************************/ +resource "google_storage_bucket" "project_bucket" { + count = "${local.create_bucket ? 1 : 0}" + + name = "${local.project_bucket_name}" + project = "${var.bucket_project}" +} + +/*********************************************** + Project's bucket storage.admin granting to group + ***********************************************/ +resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { + count = "${local.create_bucket && local.gsuite_group ? 1 : 0}" + + bucket = "${google_storage_bucket.project_bucket.name}" + role = "roles/storage.admin" + member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" +} + +/*********************************************** + Project's bucket storage.admin granting to default compute service account + ***********************************************/ +resource "google_storage_bucket_iam_member" "s_account_storage_admin_on_project_bucket" { + count = "${local.create_bucket ? 1 : 0}" + + bucket = "${google_storage_bucket.project_bucket.name}" + role = "roles/storage.admin" + member = "${local.s_account_fmt}" +} + +/*********************************************** + Project's bucket storage.admin granting to Google APIs service account + ***********************************************/ +resource "google_storage_bucket_iam_member" "api_s_account_storage_admin_on_project_bucket" { + count = "${local.create_bucket ? 1 : 0}" + + bucket = "${google_storage_bucket.project_bucket.name}" + role = "roles/storage.admin" + member = "${local.api_s_account_fmt}" + + depends_on = ["google_project_service.project_services"] +} + +/****************************************** + compute.networkUser role granted to GKE service account for GKE on shared VPC subnets + *****************************************/ +resource "google_compute_subnetwork_iam_member" "gke_shared_vpc_subnets" { + count = "${local.gke_shared_vpc_enabled && length(compact(var.shared_vpc_subnets)) != 0 ? length(var.shared_vpc_subnets) : 0}" + + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" + role = "roles/compute.networkUser" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + project = "${var.shared_vpc}" + member = "${local.gke_s_account_fmt}" + + depends_on = ["google_project_service.project_services"] +} + +/****************************************** + container.hostServiceAgentUser role granted to GKE service account for GKE on shared VPC + *****************************************/ +resource "google_project_iam_member" "gke_host_agent" { + count = "${local.gke_shared_vpc_enabled ? 1 : 0}" + + project = "${var.shared_vpc}" + role = "roles/container.hostServiceAgentUser" + member = "${local.gke_s_account_fmt}" + + depends_on = ["google_project_service.project_services"] +} diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf new file mode 100644 index 00000000..3428468a --- /dev/null +++ b/modules/core_project_factory/outputs.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${local.project_id}" +} + +output "project_number" { + value = "${local.project_number}" +} + +output "domain" { + value = "${local.domain}" + description = "The organization's domain" +} + +output "group_email" { + value = "${local.gsuite_group ? data.null_data_source.data_final_group_email.outputs["final_group_email"] : ""}" + description = "The email of the created GSuite group with group_name" +} + +output "service_account_id" { + value = "${google_service_account.default_service_account.account_id}" + description = "The id of the default service account" +} + +output "service_account_display_name" { + value = "${google_service_account.default_service_account.display_name}" + description = "The display name of the default service account" +} + +output "service_account_email" { + value = "${google_service_account.default_service_account.email}" + description = "The email of the default service account" +} + +output "service_account_name" { + value = "${google_service_account.default_service_account.name}" + description = "The fully-qualified name of the default service account" +} + +output "service_account_unique_id" { + value = "${google_service_account.default_service_account.unique_id}" + description = "The unique id of the default service account" +} + +output "project_bucket_self_link" { + value = "${google_storage_bucket.project_bucket.*.self_link}" + description = "Project's bucket selfLink" +} + +output "project_bucket_url" { + value = "${google_storage_bucket.project_bucket.*.url}" + description = "Project's bucket url" +} + +output "app_engine_enabled" { + value = "${local.app_engine_enabled}" + description = "Whether app engine is enabled" +} diff --git a/scripts/add-iam-policy-to-subnet.sh b/modules/core_project_factory/scripts/add-iam-policy-to-subnet.sh similarity index 100% rename from scripts/add-iam-policy-to-subnet.sh rename to modules/core_project_factory/scripts/add-iam-policy-to-subnet.sh diff --git a/scripts/delete-default-network.sh b/modules/core_project_factory/scripts/delete-default-network.sh similarity index 100% rename from scripts/delete-default-network.sh rename to modules/core_project_factory/scripts/delete-default-network.sh diff --git a/scripts/delete-service-account.sh b/modules/core_project_factory/scripts/delete-service-account.sh similarity index 100% rename from scripts/delete-service-account.sh rename to modules/core_project_factory/scripts/delete-service-account.sh diff --git a/scripts/get-org-id.sh b/modules/core_project_factory/scripts/get-org-id.sh similarity index 100% rename from scripts/get-org-id.sh rename to modules/core_project_factory/scripts/get-org-id.sh diff --git a/scripts/get-subnets-regions.sh b/modules/core_project_factory/scripts/get-subnets-regions.sh similarity index 100% rename from scripts/get-subnets-regions.sh rename to modules/core_project_factory/scripts/get-subnets-regions.sh diff --git a/scripts/preconditions.sh b/modules/core_project_factory/scripts/preconditions.sh similarity index 100% rename from scripts/preconditions.sh rename to modules/core_project_factory/scripts/preconditions.sh diff --git a/scripts/preconditions/preconditions.py b/modules/core_project_factory/scripts/preconditions/preconditions.py similarity index 100% rename from scripts/preconditions/preconditions.py rename to modules/core_project_factory/scripts/preconditions/preconditions.py diff --git a/scripts/preconditions/requirements.txt b/modules/core_project_factory/scripts/preconditions/requirements.txt similarity index 100% rename from scripts/preconditions/requirements.txt rename to modules/core_project_factory/scripts/preconditions/requirements.txt diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf new file mode 100644 index 00000000..808fc10b --- /dev/null +++ b/modules/core_project_factory/variables.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "lien" { + description = "Add a lien on the project to prevent accidental deletion" + default = "false" + type = "string" +} + +variable "random_project_id" { + description = "Enables project random id generation" + default = "false" +} + +variable "org_id" { + description = "The organization id for the associated services" +} + +variable "name" { + description = "The name for the project" +} + +variable "shared_vpc" { + description = "The ID of the host project which hosts the shared VPC" + default = "" +} + +variable "billing_account" { + description = "The ID of the billing account to associate this project with" +} + +variable "folder_id" { + description = "The ID of a folder to host this project" + default = "" +} + +variable "group_name" { + description = "A group to control the project by being assigned group_role - defaults to ${project_name}-editors" + default = "" +} + +variable "create_group" { + description = "Whether to create the group or not" + default = "false" +} + +variable "group_role" { + description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" + default = "roles/editor" +} + +variable "sa_group" { + description = "A GSuite group to place the default Service Account for the project in" + default = "" +} + +variable "sa_role" { + description = "A role to give the default Service Account for the project (defaults to none)" + default = "" +} + +variable "activate_apis" { + description = "The list of apis to activate within the project" + type = "list" + default = ["compute.googleapis.com"] +} + +variable "usage_bucket_name" { + description = "Name of a GCS bucket to store GCE usage reports in (optional)" + default = "" +} + +variable "credentials_path" { + description = "Path to a Service Account credentials file with permissions documented in the readme" +} + +variable "shared_vpc_subnets" { + description = "List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id)" + type = "list" + default = [""] +} + +variable "labels" { + description = "Map of labels for project" + type = "map" + default = {} +} + +variable "bucket_project" { + description = "A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional)" + default = "" +} + +variable "bucket_name" { + description = "A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional)" + default = "" +} + +variable "api_sa_group" { + description = "A GSuite group to place the Google APIs Service Account for the project in" + default = "" +} + +variable "auto_create_network" { + description = "Create the default network" + default = "false" +} + +variable "app_engine" { + description = "A map for app engine configuration" + type = "map" + default = {} +} diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md new file mode 100644 index 00000000..e69de29b diff --git a/gsuite_override.tf b/modules/gsuite_enabled/main.tf similarity index 53% rename from gsuite_override.tf rename to modules/gsuite_enabled/main.tf index 9d9b0bf2..9424ef7e 100644 --- a/gsuite_override.tf +++ b/modules/gsuite_enabled/main.tf @@ -14,10 +14,37 @@ * limitations under the License. */ -/* -If we don't need to work with gsuite, we can use the project-factory without installing/configuring gsuite, -it is necessary to delete this file that contains the gsuite resources. -*/ +/****************************************** + Locals configuration + *****************************************/ + +locals { + project_number = "${module.project-factory.project_number}" + api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" + api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" + domain = "${data.google_organization.org.domain}" +} + +/****************************************** + Gsuite configuration + *****************************************/ + +provider "gsuite" { + credentials = "${file(var.credentials_path)}" + impersonated_user_email = "${var.admin_email}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.member", + ] +} + +/****************************************** + Organization info retrieval + *****************************************/ +data "google_organization" "org" { + organization = "${var.org_id}" +} /****************************************** Group email construction when group already exists @@ -44,10 +71,10 @@ resource "gsuite_group_member" "service_account_sa_group_member" { count = "${var.sa_group != "" ? 1 : 0}" group = "${var.sa_group}" - email = "${google_service_account.default_service_account.email}" + email = "${module.project-factory.service_account_email}" role = "MEMBER" - depends_on = ["google_service_account.default_service_account"] + depends_on = ["module.project-factory"] } /****************************************** @@ -71,5 +98,29 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { email = "${local.api_s_account}" role = "MEMBER" - depends_on = ["google_project_service.project_services"] + depends_on = ["module.project-factory"] +} + +module "project-factory" { + source = "../core_project_factory/" + + lien = "${var.lien}" + random_project_id = "${var.random_project_id}" + org_id = "${var.org_id}" + name = "${var.name}" + shared_vpc = "${var.shared_vpc}" + billing_account = "${var.billing_account}" + folder_id = "${var.folder_id}" + group_name = "${var.create_group ? "gsuite_group.group.name" : var.group_name}" + group_role = "${var.group_role}" + sa_role = "${var.sa_role}" + activate_apis = "${var.activate_apis}" + usage_bucket_name = "${var.usage_bucket_name}" + credentials_path = "${var.credentials_path}" + shared_vpc_subnets = "${var.shared_vpc_subnets}" + labels = "${var.labels}" + bucket_project = "${var.bucket_project}" + bucket_name = "${var.bucket_name}" + auto_create_network = "${var.auto_create_network}" + app_engine = "${var.app_engine}" } diff --git a/modules/gsuite_enabled/variables.tf b/modules/gsuite_enabled/variables.tf new file mode 100644 index 00000000..808fc10b --- /dev/null +++ b/modules/gsuite_enabled/variables.tf @@ -0,0 +1,126 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "lien" { + description = "Add a lien on the project to prevent accidental deletion" + default = "false" + type = "string" +} + +variable "random_project_id" { + description = "Enables project random id generation" + default = "false" +} + +variable "org_id" { + description = "The organization id for the associated services" +} + +variable "name" { + description = "The name for the project" +} + +variable "shared_vpc" { + description = "The ID of the host project which hosts the shared VPC" + default = "" +} + +variable "billing_account" { + description = "The ID of the billing account to associate this project with" +} + +variable "folder_id" { + description = "The ID of a folder to host this project" + default = "" +} + +variable "group_name" { + description = "A group to control the project by being assigned group_role - defaults to ${project_name}-editors" + default = "" +} + +variable "create_group" { + description = "Whether to create the group or not" + default = "false" +} + +variable "group_role" { + description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" + default = "roles/editor" +} + +variable "sa_group" { + description = "A GSuite group to place the default Service Account for the project in" + default = "" +} + +variable "sa_role" { + description = "A role to give the default Service Account for the project (defaults to none)" + default = "" +} + +variable "activate_apis" { + description = "The list of apis to activate within the project" + type = "list" + default = ["compute.googleapis.com"] +} + +variable "usage_bucket_name" { + description = "Name of a GCS bucket to store GCE usage reports in (optional)" + default = "" +} + +variable "credentials_path" { + description = "Path to a Service Account credentials file with permissions documented in the readme" +} + +variable "shared_vpc_subnets" { + description = "List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id)" + type = "list" + default = [""] +} + +variable "labels" { + description = "Map of labels for project" + type = "map" + default = {} +} + +variable "bucket_project" { + description = "A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional)" + default = "" +} + +variable "bucket_name" { + description = "A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional)" + default = "" +} + +variable "api_sa_group" { + description = "A GSuite group to place the Google APIs Service Account for the project in" + default = "" +} + +variable "auto_create_network" { + description = "Create the default network" + default = "false" +} + +variable "app_engine" { + description = "A map for app engine configuration" + type = "map" + default = {} +} diff --git a/outputs.tf b/outputs.tf index 3428468a..11d0904f 100755 --- a/outputs.tf +++ b/outputs.tf @@ -15,59 +15,59 @@ */ output "project_id" { - value = "${local.project_id}" + value = "${module.project-factory.project_id}" } output "project_number" { - value = "${local.project_number}" + value = "${module.project-factory.project_number}" } output "domain" { - value = "${local.domain}" + value = "${module.project-factory.domain}" description = "The organization's domain" } output "group_email" { - value = "${local.gsuite_group ? data.null_data_source.data_final_group_email.outputs["final_group_email"] : ""}" + value = "${module.project-factory.group_email}" description = "The email of the created GSuite group with group_name" } output "service_account_id" { - value = "${google_service_account.default_service_account.account_id}" + value = "${module.project-factory.service_account_id}" description = "The id of the default service account" } output "service_account_display_name" { - value = "${google_service_account.default_service_account.display_name}" + value = "${module.project-factory.service_account_display_name}" description = "The display name of the default service account" } output "service_account_email" { - value = "${google_service_account.default_service_account.email}" + value = "${module.project-factory.service_account_email}" description = "The email of the default service account" } output "service_account_name" { - value = "${google_service_account.default_service_account.name}" + value = "${module.project-factory.service_account_name}" description = "The fully-qualified name of the default service account" } output "service_account_unique_id" { - value = "${google_service_account.default_service_account.unique_id}" + value = "${module.project-factory.service_account_unique_id}" description = "The unique id of the default service account" } output "project_bucket_self_link" { - value = "${google_storage_bucket.project_bucket.*.self_link}" + value = "${module.project-factory.project_bucket_self_link}" description = "Project's bucket selfLink" } output "project_bucket_url" { - value = "${google_storage_bucket.project_bucket.*.url}" + value = "${module.project-factory.project_bucket_url}" description = "Project's bucket url" } output "app_engine_enabled" { - value = "${local.app_engine_enabled}" + value = "${module.project-factory.app_engine_enabled}" description = "Whether app engine is enabled" } diff --git a/variables.tf b/variables.tf index 077ebf23..6db45303 100755 --- a/variables.tf +++ b/variables.tf @@ -52,21 +52,11 @@ variable "group_name" { default = "" } -variable "create_group" { - description = "Whether to create the group or not" - default = "false" -} - variable "group_role" { description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" default = "roles/editor" } -variable "sa_group" { - description = "A GSuite group to place the default Service Account for the project in" - default = "" -} - variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" @@ -114,11 +104,6 @@ variable "bucket_name" { default = "" } -variable "api_sa_group" { - description = "A GSuite group to place the Google APIs Service Account for the project in" - default = "" -} - variable "auto_create_network" { description = "Create the default network" default = "false" From aa6d2e5572025252495b5b2f271b74198d215efb Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Wed, 29 Aug 2018 10:25:58 -0700 Subject: [PATCH 008/105] first run at making the gsuite provider optional --- examples/app_engine/main.tf | 2 +- examples/group_project/main.tf | 15 +++- examples/group_project/variables.tf | 6 -- examples/project-hierarchy/main.tf | 4 +- examples/simple_project/main.tf | 2 +- modules/core_project_factory/main.tf | 2 +- modules/gsuite_enabled/main.tf | 16 +--- modules/gsuite_enabled/outputs.tf | 73 +++++++++++++++++++ .../add-iam-policy-to-subnet.sh | 0 .../delete-default-network.sh | 0 .../delete-service-account.sh | 0 .../scripts => scripts}/get-org-id.sh | 0 .../get-subnets-regions.sh | 0 13 files changed, 92 insertions(+), 28 deletions(-) create mode 100644 modules/gsuite_enabled/outputs.tf rename {modules/core_project_factory/scripts => scripts}/add-iam-policy-to-subnet.sh (100%) rename {modules/core_project_factory/scripts => scripts}/delete-default-network.sh (100%) rename {modules/core_project_factory/scripts => scripts}/delete-service-account.sh (100%) rename {modules/core_project_factory/scripts => scripts}/get-org-id.sh (100%) rename {modules/core_project_factory/scripts => scripts}/get-subnets-regions.sh (100%) diff --git a/examples/app_engine/main.tf b/examples/app_engine/main.tf index db529d9c..7b744918 100755 --- a/examples/app_engine/main.tf +++ b/examples/app_engine/main.tf @@ -28,7 +28,7 @@ provider "gsuite" { } module "project-factory" { - source = "../../" + source = "../../modules/gsuite_enabled" random_project_id = "true" name = "appeng-sample" org_id = "${var.organization_id}" diff --git a/examples/group_project/main.tf b/examples/group_project/main.tf index 60db215a..5da580cd 100644 --- a/examples/group_project/main.tf +++ b/examples/group_project/main.tf @@ -15,7 +15,8 @@ */ locals { - credentials_file_path = "${path.module}/sa-key.json" + // credentials_file_path = "${path.module}/sa-key.json" + credentials_file_path = "${var.credentials_file_path}" } /****************************************** @@ -26,8 +27,18 @@ provider "google" { version = "~> 1.19" } +provider "gsuite" { + credentials = "${file(local.credentials_file_path)}" + impersonated_user_email = "${var.admin_email}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/admin.directory.group", + "https://www.googleapis.com/auth/admin.directory.group.member", + ] +} + module "project-factory" { - source = "../../" + source = "../../modules/gsuite_enabled" random_project_id = "true" name = "group-sample-project" org_id = "${var.organization_id}" diff --git a/examples/group_project/variables.tf b/examples/group_project/variables.tf index ac9e5df1..e67271d4 100755 --- a/examples/group_project/variables.tf +++ b/examples/group_project/variables.tf @@ -16,30 +16,24 @@ variable "admin_email" { description = "Admin user email on Gsuite" - default = "terraform-forseti-project-1@zamir-forseti-test.iam.gserviceaccount.com" } variable "credentials_file_path" { description = "Service account json auth path" - default = "/Users/cpearring/testing/serviceAccountKeys/orgserviceaccount.json" } variable "organization_id" { description = "The organization id for the associated services" - default = "826592752744" } variable "billing_account" { description = "The ID of the billing account to associate this project with" - default = "01E8A0-35F760-5CF02A" } variable "api_sa_group" { description = "An existing GSuite group email to place the Google APIs Service Account for the project in" - default = "api_sa_group@phoogle.net" } variable "project_group_name" { description = "The name of a GSuite group to create for controlling the project" - default = "group-sample-project-owners" } diff --git a/examples/project-hierarchy/main.tf b/examples/project-hierarchy/main.tf index 871fff8f..bb4a7bc2 100755 --- a/examples/project-hierarchy/main.tf +++ b/examples/project-hierarchy/main.tf @@ -44,7 +44,7 @@ resource "google_folder" "prod" { } module "project-prod-gke" { - source = "../../" + source = "../../modules/gsuite_enabled" random_project_id = "true" name = "hierarchy-sample-prod-gke" org_id = "${var.organization_id}" @@ -54,7 +54,7 @@ module "project-prod-gke" { } module "project-factory" { - source = "../../" + source = "../../modules/gsuite_enabled" random_project_id = "true" name = "hierarchy-sample-factory" org_id = "${var.organization_id}" diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index d99c2dac..2410f6bc 100755 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -39,7 +39,7 @@ provider "gsuite" { } module "project-factory" { - source = "../../" + source = "../../modules/core_project_factory" random_project_id = "true" name = "simple-sample-project" org_id = "${var.organization_id}" diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 8885f460..da10881b 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -173,7 +173,7 @@ data "google_compute_default_service_account" "default" { *****************************************/ resource "null_resource" "delete_default_compute_service_account" { provisioner "local-exec" { - command = "${path.module}/scripts/delete-service-account.sh ${local.project_id} ${var.credentials_path} ${data.google_compute_default_service_account.default.id}" + command = "${path.module}/../../scripts/delete-service-account.sh ${local.project_id} ${var.credentials_path} ${data.google_compute_default_service_account.default.id}" } triggers { diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 9424ef7e..b20f07a9 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -25,20 +25,6 @@ locals { domain = "${data.google_organization.org.domain}" } -/****************************************** - Gsuite configuration - *****************************************/ - -provider "gsuite" { - credentials = "${file(var.credentials_path)}" - impersonated_user_email = "${var.admin_email}" - - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.group.member", - ] -} - /****************************************** Organization info retrieval *****************************************/ @@ -111,7 +97,7 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_name = "${var.create_group ? "gsuite_group.group.name" : var.group_name}" + group_name = "${var.create_group ? "${gsuite_group.group.name}" : var.group_name}" group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" diff --git a/modules/gsuite_enabled/outputs.tf b/modules/gsuite_enabled/outputs.tf new file mode 100644 index 00000000..11d0904f --- /dev/null +++ b/modules/gsuite_enabled/outputs.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${module.project-factory.project_id}" +} + +output "project_number" { + value = "${module.project-factory.project_number}" +} + +output "domain" { + value = "${module.project-factory.domain}" + description = "The organization's domain" +} + +output "group_email" { + value = "${module.project-factory.group_email}" + description = "The email of the created GSuite group with group_name" +} + +output "service_account_id" { + value = "${module.project-factory.service_account_id}" + description = "The id of the default service account" +} + +output "service_account_display_name" { + value = "${module.project-factory.service_account_display_name}" + description = "The display name of the default service account" +} + +output "service_account_email" { + value = "${module.project-factory.service_account_email}" + description = "The email of the default service account" +} + +output "service_account_name" { + value = "${module.project-factory.service_account_name}" + description = "The fully-qualified name of the default service account" +} + +output "service_account_unique_id" { + value = "${module.project-factory.service_account_unique_id}" + description = "The unique id of the default service account" +} + +output "project_bucket_self_link" { + value = "${module.project-factory.project_bucket_self_link}" + description = "Project's bucket selfLink" +} + +output "project_bucket_url" { + value = "${module.project-factory.project_bucket_url}" + description = "Project's bucket url" +} + +output "app_engine_enabled" { + value = "${module.project-factory.app_engine_enabled}" + description = "Whether app engine is enabled" +} diff --git a/modules/core_project_factory/scripts/add-iam-policy-to-subnet.sh b/scripts/add-iam-policy-to-subnet.sh similarity index 100% rename from modules/core_project_factory/scripts/add-iam-policy-to-subnet.sh rename to scripts/add-iam-policy-to-subnet.sh diff --git a/modules/core_project_factory/scripts/delete-default-network.sh b/scripts/delete-default-network.sh similarity index 100% rename from modules/core_project_factory/scripts/delete-default-network.sh rename to scripts/delete-default-network.sh diff --git a/modules/core_project_factory/scripts/delete-service-account.sh b/scripts/delete-service-account.sh similarity index 100% rename from modules/core_project_factory/scripts/delete-service-account.sh rename to scripts/delete-service-account.sh diff --git a/modules/core_project_factory/scripts/get-org-id.sh b/scripts/get-org-id.sh similarity index 100% rename from modules/core_project_factory/scripts/get-org-id.sh rename to scripts/get-org-id.sh diff --git a/modules/core_project_factory/scripts/get-subnets-regions.sh b/scripts/get-subnets-regions.sh similarity index 100% rename from modules/core_project_factory/scripts/get-subnets-regions.sh rename to scripts/get-subnets-regions.sh From 4a3b236640097f57581befa7029986b1b0f6a9b5 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Wed, 29 Aug 2018 10:52:25 -0700 Subject: [PATCH 009/105] changing example --- examples/simple_project/main.tf | 2 +- examples/simple_project/variables.tf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index 2410f6bc..d99c2dac 100755 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -39,7 +39,7 @@ provider "gsuite" { } module "project-factory" { - source = "../../modules/core_project_factory" + source = "../../" random_project_id = "true" name = "simple-sample-project" org_id = "${var.organization_id}" diff --git a/examples/simple_project/variables.tf b/examples/simple_project/variables.tf index 2f6c006f..c4fdc49b 100755 --- a/examples/simple_project/variables.tf +++ b/examples/simple_project/variables.tf @@ -14,13 +14,13 @@ * limitations under the License. */ -variable "admin_email" { - description = "Admin user email on Gsuite" +variable "organization_id" { + description = "The organization id for the associated services" } -variable "organization_id" {} - -variable "billing_account" {} +variable "billing_account" { + description = "The ID of the billing account to associate this project with" +} variable "credentials_path" { description = "Path to a Service Account credentials file with permissions documented in the readme" From 759a96cfaff187116e3e761fc42d96a17bc2c1ae Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Wed, 29 Aug 2018 11:02:15 -0700 Subject: [PATCH 010/105] cleanup --- modules/core_project_factory/README.md | 230 ---------------------- modules/core_project_factory/variables.tf | 15 -- 2 files changed, 245 deletions(-) diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 5c85888f..e69de29b 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -1,230 +0,0 @@ -# Terraform Project Factory - -This module handles opinionated Google Cloud Platform project creation and configuration with Shared VPC, IAM, APIs, etc. - -## Requirements -### Terraform plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.10.x -- [terraform-provider-google](https://github.com/terraform-providers/terraform-provider-google) plugin v1.8.0 -- [terraform-provider-gsuite](https://github.com/DeviaVir/terraform-provider-gsuite) plugin - -### Configure a Service Account -In order to execute this module you must have a Service Account with the following: -#### Roles -The service account with the following roles: -- roles/resourcemanager.folderViewer on the folder that you want to create the project in -- roles/resourcemanager.organizationViewer on the organization -- roles/resourcemanager.projectCreator on the organization -- roles/billing.user on the organization -- roles/compute.xpnAdmin on the organization (if using a shared_vpc) -- roles/compute.networkAdmin on the organization -- roles/iam.serviceAccountAdmin on the organization -- roles/browser on the host project (if using a shared_vpc) -- roles/resourcemanager.projectIamAdmin on the host project (if using a shared_vpc) -- roles/storage.admin on bucket_project - -#### GSuite domain delegation -- Enable the G Suite Domain-wide Delegation on your service account - -#### Usage report -- Give enough permissions to Service Account on the bucket for reading and writing objects. - -### Enable API's -In order to operate with the Service Account you must activate the following API's on the base project where the Service Account was created: - -- Cloud Resource Manager API - cloudresourcemanager.googleapis.com -- Cloud Billing API - cloudbilling.googleapis.com -- Identity and Access Management API - iam.googleapis.com -- Admin SDK - admin.googleapis.com -- Google App Engine Admin API - appengine.googleapis.com - -### GSuite -#### Admin -- You will need an admin on your Google Admin site. - -#### API Client access -Give API client access to Service Account on your Google Admin site -- Go to Security > Settings > Advanced Settings > Manage API client access -- Add the client ID of your Service Account and the following scopes: - - https://www.googleapis.com/auth/admin.directory.group - - https://www.googleapis.com/auth/admin.directory.group.member - -## Install - -### Terraform -Be sure you have the correct Terraform version (0.10.x), you can choose the binary here: -- https://releases.hashicorp.com/terraform/ - -### Terraform plugins -Be sure you have the compiled plugins on $HOME/.terraform.d/plugins/ - -- [terraform-provider-gsuite](https://github.com/DeviaVir/terraform-provider-gsuite) plugin 0.1.0 (there are not compatible releases, you have to compile it from master branch) - -See each plugin page for more information about how to compile and use them - -## Fast install (optional) -For a fast install, please configure the variables on init_centos.sh or init_debian.sh script in the helpers directory and then launch it. - -The script will do: -- Environment variables setting -- Installation of base packages like wget, curl, unzip, gcloud, etc. -- Installation of go 1.9.0 -- Installation of Terraform 0.10.x -- Download the terraform-provider-gsuite plugin -- Compile the terraform-provider-gsuite plugin -- Move the terraform-provider-gsuite to the right location - -## Scripted Setting of Permissions on Host Project Service Account (optional) -This will grant the service account in your host project the needed permissions. - -To use the script, run the following with the appropriate values: -- `helpers/set-host-sa-permissions.sh ''` - -## Usage -You can go to the examples folder, however the usage of the module could be like this in your own main.tf file: - -*Configure the provider here before the module invocation, see the examples folder* - - locals { - credentials_file_path = "" - } - - provider "google" { - credentials = "${file(local.credentials_file_path)}" - } - - provider "gsuite" { - credentials = "${file(local.credentials_file_path)}" - impersonated_user_email = "" - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.group.member" - ] - } - module "project-factory" { - source = "" - name = "pf-test-1" - random_project_id = "true" - org_id = "1234567890" - usage_bucket_name = "pf-test-1-usage-report-bucket" - billing_account = "ABCDEF-ABCDEF-ABCDEF" - group_role = "roles/editor" - shared_vpc = "shared_vpc_host_name" - sa_group = "test_sa_group@yourdomain.com" - credentials_path = "${local.credentials_file_path}" - shared_vpc_subnets = [ - "projects/base-project-196723/regions/us-east1/subnetworks/default", - "projects/base-project-196723/regions/us-central1/subnetworks/default", - "projects/base-project-196723/regions/us-central1/subnetworks/subnet-1", - ] - } - -Then perform the following commands on the root folder: - -- `terraform init` to get the plugins -- `terraform plan` to see the infrastructure plan -- `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure - -#### Variables -Please refer the /variables.tf file for the required and optional variables. - -#### Outputs -Please refer the /outputs.tf file for the outputs that you can get with the `terraform output` command - -## Infrastructure -The resources/services/activations/deletions that this module will create/trigger are: -- A Google project -- Specified API's activation on the project -- Shared VPC configuration (if provided) -- A default Service Account creation -- Make the Service Account member of your provided `sa_group` -- Deletion of the default compute service (via bash script) -- Deletion of the default network (via bash script) -- Creation of a GSuite group with your provided `group_name` if `created_group` is true -- Storage the usage reports to the specified bucket -- Bucket `bucket_name` creation on `bucket_project` - -The roles granted for the specific resources are: -- Service Account - - compute.networkUser on host project or specific subnets - - compute.instanceAdmin.v1 on project - - MEMBER on `sa_group` - - storage.admin on `bucket_name` - -- `group_name` - - compute.networkUser on host project or specific subnets - - `group_role` on project - - iam.serviceAccountUser on Service Account - - storage.admin on `bucket_name` - -- APIs Service Account - - compute.networkUser on host project or specific subnets - - MEMBER on `api_sa_group` - - storage.admin on `bucket_name` - -## File structure -The project has the following folders and files: - -- /: root folder -- /examples: examples for using this module -- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file) -- /test: Folders with files for testing the module (see Testing section on this file) -- /helpers: Optional helper scripts for ease of use -- /main.tf: main file for this module, contains all the resources to create -- /variables.tf: all the variables for the module -- /output.tf: the outputs of the module -- /readme.MD: this file - -## Testing - -### Requirements -- [bats](https://github.com/sstephenson/bats) 0.4.0 -- [jq](https://stedolan.github.io/jq/) 1.5 - -### Integration test -##### Terraform integration tests -The integration tests for this module are built with bats, basically the test checks the following: -- Perform `terraform init` command -- Perform `terraform get` command -- Perform `terraform plan` command and check that it'll create *n* resources, modify 0 resources and delete 0 resources -- Perform `terraform apply -auto-approve` command and check that it has created the *n* resources, modified 0 resources and deleted 0 resources -- Perform several `gcloud` commands and check the infrastructure is in the desired state -- Perform `terraform destroy -force` command and check that it has destroyed the *n* resources - -You can use the following command to run the integration test in the folder */test/integration/gcloud-test* - - `. launch.sh` - -### Linting -The makefile in this project will lint or sometimes just format any shell, -Python, golang, Terraform, or Dockerfiles. The linters will only be run if -the makefile finds files with the appropriate file extension. - -All of the linter checks are in the default make target, so you just have to -run - -``` -make -s -``` - -The -s is for 'silent'. Successful output looks like this - -``` -Running shellcheck -Running flake8 -Running gofmt -Running terraform validate -Running hadolint on Dockerfiles -Test passed - Verified all file Apache 2 headers -``` - -The linters -are as follows: -* Shell - shellcheck. Can be found in homebrew -* Python - flake8. Can be installed with 'pip install flake8' -* Golang - gofmt. gofmt comes with the standard golang installation. golang -is a compiled language so there is no standard linter. -* Terraform - terraform has a built-in linter in the 'terraform validate' -command. -* Dockerfiles - hadolint. Can be found in homebrew diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 808fc10b..a11353b5 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -52,21 +52,11 @@ variable "group_name" { default = "" } -variable "create_group" { - description = "Whether to create the group or not" - default = "false" -} - variable "group_role" { description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" default = "roles/editor" } -variable "sa_group" { - description = "A GSuite group to place the default Service Account for the project in" - default = "" -} - variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" @@ -109,11 +99,6 @@ variable "bucket_name" { default = "" } -variable "api_sa_group" { - description = "A GSuite group to place the Google APIs Service Account for the project in" - default = "" -} - variable "auto_create_network" { description = "Create the default network" default = "false" From 74c95f4fe6a23e87bb950c01bdd4c7e6deddca7f Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Wed, 5 Sep 2018 09:56:38 -0700 Subject: [PATCH 011/105] removing unecessary dependencies --- modules/gsuite_enabled/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index b20f07a9..e9c3986e 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -83,8 +83,6 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { group = "${var.api_sa_group}" email = "${local.api_s_account}" role = "MEMBER" - - depends_on = ["module.project-factory"] } module "project-factory" { From 183bf5f532149889e9371432ab5ff28bcd8b7920 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Thu, 6 Sep 2018 16:46:34 -0700 Subject: [PATCH 012/105] +using module outputs instead of re-adding resource +deleting commented code --- examples/group_project/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/group_project/main.tf b/examples/group_project/main.tf index 5da580cd..7cfb9b0a 100644 --- a/examples/group_project/main.tf +++ b/examples/group_project/main.tf @@ -15,7 +15,6 @@ */ locals { - // credentials_file_path = "${path.module}/sa-key.json" credentials_file_path = "${var.credentials_file_path}" } From b0fec4e1cb833152613a2c977c89a4acbc499c00 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Thu, 6 Sep 2018 16:46:34 -0700 Subject: [PATCH 013/105] +using module outputs instead of re-adding resource +deleting commented code --- modules/core_project_factory/outputs.tf | 10 ++++++++++ modules/gsuite_enabled/main.tf | 14 +++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 3428468a..43369882 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -71,3 +71,13 @@ output "app_engine_enabled" { value = "${local.app_engine_enabled}" description = "Whether app engine is enabled" } + +output "api_s_account" { + value = "${local.api_s_account}" + description = "API service account email" +} + +output "api_s_account_fmt" { + value = "${local.api_s_account_fmt}" + description = "API service account email formatted for terraform use" +} diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index e9c3986e..03f8ef33 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -20,16 +20,12 @@ locals { project_number = "${module.project-factory.project_number}" - api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" - api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" - domain = "${data.google_organization.org.domain}" -} + api_s_account = "${module.project-factory.api_s_account}" + api_s_account_fmt = "${module.project-factory.api_s_account_fmt}" + domain = "${module.project-factory.domain}" -/****************************************** - Organization info retrieval - *****************************************/ -data "google_organization" "org" { - organization = "${var.org_id}" + // default group_name to ${project_name}-editors + group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" } /****************************************** From d2e097d85e5d40b2a06a458cc785f98fd89ea25c Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Thu, 6 Sep 2018 18:10:15 -0700 Subject: [PATCH 014/105] removing gsuite from simple project example --- examples/simple_project/main.tf | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/examples/simple_project/main.tf b/examples/simple_project/main.tf index d99c2dac..69dbf73a 100755 --- a/examples/simple_project/main.tf +++ b/examples/simple_project/main.tf @@ -23,19 +23,7 @@ locals { *****************************************/ provider "google" { credentials = "${file(local.credentials_file_path)}" - version = "~> 1.19" -} - -provider "gsuite" { - credentials = "${file(local.credentials_file_path)}" - impersonated_user_email = "${var.admin_email}" - - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.user", - ] - - version = "~> 0.1.9" + version = "~> 1.19" } module "project-factory" { From 12a73854c58c3d50e9522261a7ad3562ff969d92 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Fri, 7 Sep 2018 09:47:09 -0700 Subject: [PATCH 015/105] getting up to date with master v2 --- examples/gke_shared_vpc/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/gke_shared_vpc/README.md b/examples/gke_shared_vpc/README.md index cb3a4772..0309ca6f 100644 --- a/examples/gke_shared_vpc/README.md +++ b/examples/gke_shared_vpc/README.md @@ -5,15 +5,18 @@ This illustrates how to create a project with a shared VPC from a host project t As shown in this exmaple, GKE shared VPC is only enabled if the "container.googleapis.com" API is in the "activate_apis" variable list. It will do the following: + - Create a project - Give appropriate iam permissions to the API and GKE service accounts on the host vpc project Expected variables: + - `org_id` - `billing_account` - `shared_vpc` To specify a subnet use the "shared_vpc_subnets" variable, and list subnets like the following: + - ["projects//regions//subnetworks/", "projects//regions//subnetworks/"] If no subnets are specified, all networks and subnets from the host project are shared. @@ -32,4 +35,4 @@ More information about GKE with Shared VPC can be found here: https://cloud.goog | shared\_vpc | The ID of the host project which hosts the shared VPC | string | - | yes | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$PROJECT_ID/regions/$REGION/subnetworks/$SUBNET_ID) | list | `` | no | -[^]: (autogen_docs_end) +[^]: (autogen_docs_end) \ No newline at end of file From e51e572bb7d11bd55c2a40df122f3fb2c0e08e86 Mon Sep 17 00:00:00 2001 From: Casey Pearing <5456037+CPearring@users.noreply.github.com> Date: Wed, 12 Sep 2018 08:35:12 -0700 Subject: [PATCH 016/105] Update main.tf --- modules/core_project_factory/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index da10881b..7972f966 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -173,7 +173,7 @@ data "google_compute_default_service_account" "default" { *****************************************/ resource "null_resource" "delete_default_compute_service_account" { provisioner "local-exec" { - command = "${path.module}/../../scripts/delete-service-account.sh ${local.project_id} ${var.credentials_path} ${data.google_compute_default_service_account.default.id}" + command = "${path.module}/scripts/delete-service-account.sh ${local.project_id} ${var.credentials_path} ${data.google_compute_default_service_account.default.id}" } triggers { @@ -208,7 +208,7 @@ resource "google_project_iam_member" "default_service_account_membership" { Gsuite Group Role Configuration *****************************************/ resource "google_project_iam_member" "gsuite_group_role" { - count = "${local.gsuite_group ? 1 : 0}" + count = "${local.gsuite_group ? 1 : 0}" project = "${local.project_id}" role = "${var.group_role}" @@ -289,7 +289,7 @@ resource "google_project_usage_export_bucket" "usage_report_export" { project = "${local.project_id}" bucket_name = "${var.usage_bucket_name}" - prefix = "usage-${local.project_id}" + prefix = "${var.usage_bucket_prefix != "" ? var.usage_bucket_prefix : "usage-${local.project_id}"}" depends_on = ["google_project_service.project_services"] } From 1862dad13f09df76c701a57eb8a99e1901184a94 Mon Sep 17 00:00:00 2001 From: Casey Pearing <5456037+CPearring@users.noreply.github.com> Date: Wed, 12 Sep 2018 08:40:18 -0700 Subject: [PATCH 017/105] Update variables.tf --- modules/core_project_factory/variables.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index a11353b5..04ce53cf 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -26,7 +26,13 @@ variable "random_project_id" { } variable "org_id" { - description = "The organization id for the associated services" + description = "The organization id (optional if `domain` is passed)" + default = "" +} + +variable "domain" { + description = "The domain name (optional if `org_id` is passed)" + default = "" } variable "name" { @@ -72,6 +78,11 @@ variable "usage_bucket_name" { description = "Name of a GCS bucket to store GCE usage reports in (optional)" default = "" } + +variable "usage_bucket_prefix" { + description = "Prefix in the GCS bucket to store GCE usage reports in (optional)" + default = "" +} variable "credentials_path" { description = "Path to a Service Account credentials file with permissions documented in the readme" From 5b451fb02e4fe19e70f1b650240687ae21573efa Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Fri, 14 Sep 2018 16:52:02 -0700 Subject: [PATCH 018/105] keeping up to date with master, adding in migrate.sh script as first pass on migrating from v1 project factory --- examples/group_project/variables.tf | 8 +++++++- helpers/migrate.sh | 25 +++++++++++++++++++++++ main.tf | 11 ++++++++++ modules/core_project_factory/variables.tf | 5 +++++ modules/gsuite_enabled/main.tf | 8 ++++++++ modules/gsuite_enabled/variables.tf | 11 ++++++++++ 6 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 helpers/migrate.sh diff --git a/examples/group_project/variables.tf b/examples/group_project/variables.tf index e67271d4..bfc62380 100755 --- a/examples/group_project/variables.tf +++ b/examples/group_project/variables.tf @@ -16,24 +16,30 @@ variable "admin_email" { description = "Admin user email on Gsuite" + default = "casey@phoogle.net" } variable "credentials_file_path" { description = "Service account json auth path" + default = "~/testing/serviceAccountKeys/orgserviceaccount.json" } variable "organization_id" { description = "The organization id for the associated services" + default = "826592752744" } variable "billing_account" { description = "The ID of the billing account to associate this project with" + default = "01E8A0-35F760-5CF02A" } variable "api_sa_group" { description = "An existing GSuite group email to place the Google APIs Service Account for the project in" + default = "api_sa_group@phoogle.net" } variable "project_group_name" { description = "The name of a GSuite group to create for controlling the project" -} + default = "cpearring-test" +} \ No newline at end of file diff --git a/helpers/migrate.sh b/helpers/migrate.sh new file mode 100644 index 00000000..3c86d3a8 --- /dev/null +++ b/helpers/migrate.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +MODULE="module.project-factory" +NEW_MODULE="module.project-factory.module.project-factory" + +terraform state mv $MODULE.random_id.random_project_id_suffix $NEW_MODULE.random_id.random_project_id_suffix +terraform state mv $MODULE.google_project.project $NEW_MODULE.google_project.project +terraform state mv $MODULE.google_project_service.project_services $NEW_MODULE.google_project_service.project_services +terraform state mv $MODULE.google_compute_shared_vpc_service_project.shared_vpc_attachment $MODULE.google_compute_shared_vpc_service_project.shared_vpc_attachment +terraform state mv $MODULE.null_resource.delete_default_compute_service_account $NEW_MODULE.null_resource.delete_default_compute_service_account +terraform state mv $MODULE.google_service_account.default_service_account $NEW_MODULE.google_service_account.default_service_account +terraform state mv $MODULE.google_project_iam_member.default_service_account_membership $NEW_MODULE.google_project_iam_member.default_service_account_membership +terraform state mv $MODULE.google_project_iam_member.gsuite_group_role $NEW_MODULE.google_project_iam_member.gsuite_group_role +terraform state mv $MODULE.google_service_account_iam_member.service_account_grant_to_group $NEW_MODULE.google_service_account_iam_member.service_account_grant_to_group +terraform state mv $MODULE.google_project_iam_member.controlling_group_vpc_membership $NEW_MODULE.$MODULE.google_project_iam_member.controlling_group_vpc_membership +terraform state mv $MODULE.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets +terraform state mv $MODULE.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets +terraform state mv $MODULE.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets +terraform state mv $MODULE.google_project_usage_export_bucket.usage_report_export $NEW_MODULE.google_project_usage_export_bucket.usage_report_export +terraform state mv $MODULE.google_storage_bucket.project_bucket $NEW_MODULE.google_storage_bucket.project_bucket +terraform state mv $MODULE.google_storage_bucket_iam_member.group_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.group_storage_admin_on_project_bucket +terraform state mv $MODULE.google_storage_bucket_iam_member.s_account_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.s_account_storage_admin_on_project_bucket +terraform state mv $MODULE.google_storage_bucket_iam_member.api_s_account_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.api_s_account_storage_admin_on_project_bucket +terraform state mv $MODULE.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets +terraform state mv $MODULE.google_project_iam_member.gke_host_agent $NEW_MODULE.google_project_iam_member.gke_host_agent diff --git a/main.tf b/main.tf index 1b6d9226..07a3a3ac 100755 --- a/main.tf +++ b/main.tf @@ -14,12 +14,22 @@ * limitations under the License. */ +locals { + args_missing = "${(var.group_name != "" && var.org_id == "" && var.domain == "") ? 1 : 0}" +} + +resource "null_resource" "args_missing" { + count = "${local.args_missing}" + "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true +} + module "project-factory" { source = "modules/core_project_factory" lien = "${var.lien}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" + domain = "${var.domain}" name = "${var.name}" shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" @@ -29,6 +39,7 @@ module "project-factory" { sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" + usage_bucket_prefix = "${var.usage_bucket_prefix}" credentials_path = "${var.credentials_path}" shared_vpc_subnets = "${var.shared_vpc_subnets}" labels = "${var.labels}" diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 04ce53cf..61bbdc2f 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -84,6 +84,11 @@ variable "usage_bucket_prefix" { default = "" } +variable "usage_bucket_prefix" { + description = "Prefix in the GCS bucket to store GCE usage reports in (optional)" + default = "" +} + variable "credentials_path" { description = "Path to a Service Account credentials file with permissions documented in the readme" } diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 03f8ef33..11c58700 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -23,11 +23,17 @@ locals { api_s_account = "${module.project-factory.api_s_account}" api_s_account_fmt = "${module.project-factory.api_s_account_fmt}" domain = "${module.project-factory.domain}" + args_missing = "${(var.group_name != "" && var.org_id == "" && var.domain == "") ? 1 : 0}" // default group_name to ${project_name}-editors group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" } +resource "null_resource" "args_missing" { + count = "${local.args_missing}" + "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true +} + /****************************************** Group email construction when group already exists *****************************************/ @@ -87,6 +93,7 @@ module "project-factory" { lien = "${var.lien}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" + domain = "${var.domain}" name = "${var.name}" shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" @@ -96,6 +103,7 @@ module "project-factory" { sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" + usage_bucket_prefix = "${var.usage_bucket_prefix}" credentials_path = "${var.credentials_path}" shared_vpc_subnets = "${var.shared_vpc_subnets}" labels = "${var.labels}" diff --git a/modules/gsuite_enabled/variables.tf b/modules/gsuite_enabled/variables.tf index 808fc10b..abd33e39 100644 --- a/modules/gsuite_enabled/variables.tf +++ b/modules/gsuite_enabled/variables.tf @@ -27,6 +27,12 @@ variable "random_project_id" { variable "org_id" { description = "The organization id for the associated services" + default = "" +} + +variable "domain" { + description = "The domain name (optional if `org_id` is passed)" + default = "" } variable "name" { @@ -83,6 +89,11 @@ variable "usage_bucket_name" { default = "" } +variable "usage_bucket_prefix" { + description = "Prefix in the GCS bucket to store GCE usage reports in (optional)" + default = "" +} + variable "credentials_path" { description = "Path to a Service Account credentials file with permissions documented in the readme" } From 488c9e78cfcb38cadcb58380156e6f202bb7a8a8 Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Wed, 19 Sep 2018 08:30:10 -0700 Subject: [PATCH 019/105] removing set variables --- examples/group_project/variables.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/group_project/variables.tf b/examples/group_project/variables.tf index bfc62380..9a6e6108 100755 --- a/examples/group_project/variables.tf +++ b/examples/group_project/variables.tf @@ -16,30 +16,24 @@ variable "admin_email" { description = "Admin user email on Gsuite" - default = "casey@phoogle.net" } variable "credentials_file_path" { description = "Service account json auth path" - default = "~/testing/serviceAccountKeys/orgserviceaccount.json" } variable "organization_id" { description = "The organization id for the associated services" - default = "826592752744" } variable "billing_account" { description = "The ID of the billing account to associate this project with" - default = "01E8A0-35F760-5CF02A" } variable "api_sa_group" { description = "An existing GSuite group email to place the Google APIs Service Account for the project in" - default = "api_sa_group@phoogle.net" } variable "project_group_name" { description = "The name of a GSuite group to create for controlling the project" - default = "cpearring-test" } \ No newline at end of file From 6c413cb2bb99cf96101d14dad2fab98c15c918e1 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 21 Sep 2018 19:28:54 -0400 Subject: [PATCH 020/105] Fix duplicate variable --- modules/core_project_factory/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 61bbdc2f..04ce53cf 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -84,11 +84,6 @@ variable "usage_bucket_prefix" { default = "" } -variable "usage_bucket_prefix" { - description = "Prefix in the GCS bucket to store GCE usage reports in (optional)" - default = "" -} - variable "credentials_path" { description = "Path to a Service Account credentials file with permissions documented in the readme" } From e65e55a387e57e96c5f3eb2975bf13ae90340505 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 24 Sep 2018 16:05:22 -0700 Subject: [PATCH 021/105] Fixups for moved modules and deleted variables --- modules/core_project_factory/main.tf | 9 ++++----- modules/core_project_factory/outputs.tf | 2 +- .../scripts}/delete-service-account.sh | 0 3 files changed, 5 insertions(+), 6 deletions(-) rename {scripts => modules/core_project_factory/scripts}/delete-service-account.sh (100%) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 7972f966..321e4c35 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -40,7 +40,6 @@ locals { gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" create_bucket = "${var.bucket_project != "" ? "true" : "false"}" - gsuite_group = "${var.group_name != "" || var.create_group}" app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" shared_vpc_users = "${compact(list(local.s_account_fmt, data.null_data_source.data_group_email_format.outputs["group_fmt"], local.api_s_account_fmt, local.gke_s_account_fmt))}" @@ -208,7 +207,7 @@ resource "google_project_iam_member" "default_service_account_membership" { Gsuite Group Role Configuration *****************************************/ resource "google_project_iam_member" "gsuite_group_role" { - count = "${local.gsuite_group ? 1 : 0}" + count = "${var.group_name != "" ? 1 : 0}" project = "${local.project_id}" role = "${var.group_role}" @@ -219,7 +218,7 @@ resource "google_project_iam_member" "gsuite_group_role" { Granting serviceAccountUser to group *****************************************/ resource "google_service_account_iam_member" "service_account_grant_to_group" { - count = "${local.gsuite_group ? 1 : 0}" + count = "${var.group_name != "" ? 1 : 0}" service_account_id = "projects/${local.project_id}/serviceAccounts/${google_service_account.default_service_account.email}" role = "roles/iam.serviceAccountUser" @@ -257,7 +256,7 @@ resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_sub compute.networkUser role granted to GSuite group on vpc subnets *************************************************************************************/ resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && local.gsuite_group ? length(var.shared_vpc_subnets) : 0 }" + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && var.group_name != "" ? length(var.shared_vpc_subnets) : 0 }" subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" role = "roles/compute.networkUser" @@ -308,7 +307,7 @@ resource "google_storage_bucket" "project_bucket" { Project's bucket storage.admin granting to group ***********************************************/ resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { - count = "${local.create_bucket && local.gsuite_group ? 1 : 0}" + count = "${local.create_bucket && var.group_name != "" ? 1 : 0}" bucket = "${google_storage_bucket.project_bucket.name}" role = "roles/storage.admin" diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 43369882..6212fec6 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${local.gsuite_group ? data.null_data_source.data_final_group_email.outputs["final_group_email"] : ""}" + value = "${var.group_name != "" ? data.null_data_source.data_final_group_email.outputs["final_group_email"] : ""}" description = "The email of the created GSuite group with group_name" } diff --git a/scripts/delete-service-account.sh b/modules/core_project_factory/scripts/delete-service-account.sh similarity index 100% rename from scripts/delete-service-account.sh rename to modules/core_project_factory/scripts/delete-service-account.sh From 7aad8a2d97e44db1b1bde61c7b040ff7a176cfda Mon Sep 17 00:00:00 2001 From: Casey Pearring Date: Thu, 4 Oct 2018 16:55:36 -0700 Subject: [PATCH 022/105] terraform fmt and moving group email to locals --- examples/group_project/variables.tf | 2 +- modules/core_project_factory/main.tf | 65 +++++++++-------------- modules/core_project_factory/variables.tf | 7 ++- modules/gsuite_enabled/main.tf | 25 ++------- modules/gsuite_enabled/variables.tf | 2 +- 5 files changed, 37 insertions(+), 64 deletions(-) diff --git a/examples/group_project/variables.tf b/examples/group_project/variables.tf index 9a6e6108..e67271d4 100755 --- a/examples/group_project/variables.tf +++ b/examples/group_project/variables.tf @@ -36,4 +36,4 @@ variable "api_sa_group" { variable "project_group_name" { description = "The name of a GSuite group to create for controlling the project" -} \ No newline at end of file +} diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 321e4c35..f07470aa 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -25,25 +25,26 @@ resource "random_id" "random_project_id_suffix" { Locals configuration *****************************************/ locals { - project_id = "${google_project.project.project_id}" - project_number = "${google_project.project.number}" - project_org_id = "${var.folder_id != "" ? "" : var.org_id}" - project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" - temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" - domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" - s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" - api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" - api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" - gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" - gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" - gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" - project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" - create_bucket = "${var.bucket_project != "" ? "true" : "false"}" - app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - - shared_vpc_users = "${compact(list(local.s_account_fmt, data.null_data_source.data_group_email_format.outputs["group_fmt"], local.api_s_account_fmt, local.gke_s_account_fmt))}" - shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 + project_id = "${google_project.project.project_id}" + project_number = "${google_project.project.number}" + project_org_id = "${var.folder_id != "" ? "" : var.org_id}" + project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" + temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" + domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" + args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" + s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" + api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" + api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" + gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" + gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" + gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" + project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" + create_bucket = "${var.bucket_project != "" ? "true" : "false"}" + app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" + shared_vpc_users = "${compact(list(local.s_account_fmt, local.group_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" + shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 + final_group_email = "${var.group_email != "" ? var.group_email : (var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : "")}" + group_fmt = "${local.final_group_email != "" ? format("group:%s", local.final_group_email) : ""}" app_engine_config = { enabled = "${list(var.app_engine)}" @@ -56,24 +57,6 @@ resource "null_resource" "args_missing" { "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true } -/****************************************** - Group email to be used on resources - *****************************************/ -data "null_data_source" "data_final_group_email" { - inputs { - final_group_email = "${var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : ""}" - } -} - -/****************************************** - Group email formatting - *****************************************/ -data "null_data_source" "data_group_email_format" { - inputs { - group_fmt = "${data.null_data_source.data_final_group_email.outputs["final_group_email"] != "" ? format("group:%s", data.null_data_source.data_final_group_email.outputs["final_group_email"]) : ""}" - } -} - /****************************************** Organization info retrieval *****************************************/ @@ -211,7 +194,7 @@ resource "google_project_iam_member" "gsuite_group_role" { project = "${local.project_id}" role = "${var.group_role}" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" + member = "${local.group_fmt}" } /****************************************** @@ -223,7 +206,7 @@ resource "google_service_account_iam_member" "service_account_grant_to_group" { service_account_id = "projects/${local.project_id}/serviceAccounts/${google_service_account.default_service_account.email}" role = "roles/iam.serviceAccountUser" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" + member = "${local.group_fmt}" } /************************************************************************************* @@ -262,7 +245,7 @@ resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { role = "roles/compute.networkUser" region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" project = "${var.shared_vpc}" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" + member = "${local.group_fmt}" } /************************************************************************************* @@ -311,7 +294,7 @@ resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_buck bucket = "${google_storage_bucket.project_bucket.name}" role = "roles/storage.admin" - member = "${data.null_data_source.data_group_email_format.outputs["group_fmt"]}" + member = "${local.group_fmt}" } /*********************************************** diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 04ce53cf..698bd584 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -58,6 +58,11 @@ variable "group_name" { default = "" } +variable "group_email" { + description = "The email used for the group, this is automatically created" + default = "" +} + variable "group_role" { description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" default = "roles/editor" @@ -78,7 +83,7 @@ variable "usage_bucket_name" { description = "Name of a GCS bucket to store GCE usage reports in (optional)" default = "" } - + variable "usage_bucket_prefix" { description = "Prefix in the GCS bucket to store GCE usage reports in (optional)" default = "" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 11c58700..83fc02ff 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -26,7 +26,9 @@ locals { args_missing = "${(var.group_name != "" && var.org_id == "" && var.domain == "") ? 1 : 0}" // default group_name to ${project_name}-editors - group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" + group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" + given_group_email = "${var.create_group == "false" ? format("%s@%s", var.group_name, local.domain) : ""}" + final_group_email = "${var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : local.given_group_email}" } resource "null_resource" "args_missing" { @@ -34,24 +36,6 @@ resource "null_resource" "args_missing" { "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true } -/****************************************** - Group email construction when group already exists - *****************************************/ -data "null_data_source" "data_given_group_email" { - inputs { - given_group_email = "${var.create_group == "false" ? format("%s@%s", var.group_name, local.domain) : ""}" - } -} - -/****************************************** - Group email to be used on resources - *****************************************/ -data "null_data_source" "data_final_group_email" { - inputs { - final_group_email = "${var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : data.null_data_source.data_given_group_email.outputs["given_group_email"]}" - } -} - /*********************************************** Make service account member of sa_group group ***********************************************/ @@ -98,7 +82,8 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_name = "${var.create_group ? "${gsuite_group.group.name}" : var.group_name}" + group_name = "${var.create_group ? gsuite_group.group.name : local.group_name}" + group_email = "${local.final_group_email}" group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" diff --git a/modules/gsuite_enabled/variables.tf b/modules/gsuite_enabled/variables.tf index abd33e39..682e026d 100644 --- a/modules/gsuite_enabled/variables.tf +++ b/modules/gsuite_enabled/variables.tf @@ -27,7 +27,7 @@ variable "random_project_id" { variable "org_id" { description = "The organization id for the associated services" - default = "" + default = "" } variable "domain" { From c200547639ab4e30570ed7ea7222d7be2222006d Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Fri, 5 Oct 2018 11:10:30 -0700 Subject: [PATCH 023/105] Fix reference to removed null_data_source with local A refactor that removed null_data_sources in favor of local variables didn't update all of the locations where the old value was used; this commit fixes up the omission. --- modules/core_project_factory/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 6212fec6..c8c9c530 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${var.group_name != "" ? data.null_data_source.data_final_group_email.outputs["final_group_email"] : ""}" + value = "${var.group_name != "" ? local.final_group_email : ""}" description = "The email of the created GSuite group with group_name" } From c7a560710373e87b3c6fcf955323ced743d201e8 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 29 Nov 2018 21:07:26 +0000 Subject: [PATCH 024/105] Synchronize args_missing logic --- modules/gsuite_enabled/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 83fc02ff..8787914c 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -23,7 +23,7 @@ locals { api_s_account = "${module.project-factory.api_s_account}" api_s_account_fmt = "${module.project-factory.api_s_account_fmt}" domain = "${module.project-factory.domain}" - args_missing = "${(var.group_name != "" && var.org_id == "" && var.domain == "") ? 1 : 0}" + args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" // default group_name to ${project_name}-editors group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" From c017a0a89ddbf564500a535d0e006e1e01eb6446 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 29 Nov 2018 21:08:07 +0000 Subject: [PATCH 025/105] Use gsuite_enabled module in full fixture --- test/fixtures/full/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/full/main.tf b/test/fixtures/full/main.tf index faf39d6f..592ed695 100644 --- a/test/fixtures/full/main.tf +++ b/test/fixtures/full/main.tf @@ -57,7 +57,7 @@ module "vpc" { } module "project-factory" { - source = "../../../" + source = "../../../modules/gsuite_enabled" name = "${var.name}" random_project_id = true org_id = "${var.org_id}" From 98e6aabb947643726445c35c855af00e9b4c2779 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 16:19:48 +0000 Subject: [PATCH 026/105] Remove argument check from root, gsuite_enabled root and gsuite_enabled do not utilize var.org_id and var.domain in a manner which requires them to both exist. --- main.tf | 9 --------- modules/gsuite_enabled/main.tf | 6 ------ 2 files changed, 15 deletions(-) diff --git a/main.tf b/main.tf index 07a3a3ac..06b29408 100755 --- a/main.tf +++ b/main.tf @@ -14,15 +14,6 @@ * limitations under the License. */ -locals { - args_missing = "${(var.group_name != "" && var.org_id == "" && var.domain == "") ? 1 : 0}" -} - -resource "null_resource" "args_missing" { - count = "${local.args_missing}" - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true -} - module "project-factory" { source = "modules/core_project_factory" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 8787914c..71ce3472 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -23,7 +23,6 @@ locals { api_s_account = "${module.project-factory.api_s_account}" api_s_account_fmt = "${module.project-factory.api_s_account_fmt}" domain = "${module.project-factory.domain}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" // default group_name to ${project_name}-editors group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" @@ -31,11 +30,6 @@ locals { final_group_email = "${var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : local.given_group_email}" } -resource "null_resource" "args_missing" { - count = "${local.args_missing}" - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true -} - /*********************************************** Make service account member of sa_group group ***********************************************/ From 3f92372b2c824ee17ca5498dcb598fb4b2dffbba Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 17:08:14 +0000 Subject: [PATCH 027/105] Remove unused locals from gsuite_enabled --- modules/gsuite_enabled/main.tf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 71ce3472..60616325 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -19,10 +19,8 @@ *****************************************/ locals { - project_number = "${module.project-factory.project_number}" - api_s_account = "${module.project-factory.api_s_account}" - api_s_account_fmt = "${module.project-factory.api_s_account_fmt}" - domain = "${module.project-factory.domain}" + api_s_account = "${module.project-factory.api_s_account}" + domain = "${module.project-factory.domain}" // default group_name to ${project_name}-editors group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" From b4f013af815a6aeb94e8b378dfbc03ed2db98827 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 17:26:50 +0000 Subject: [PATCH 028/105] Extract domain calculation logic to module This change removes a cyclic dependency between gsuite_enabled and core_project_factory --- modules/core_project_factory/main.tf | 49 +++++++++++++----------- modules/core_project_factory/outputs.tf | 2 +- modules/google_organization/main.tf | 27 +++++++++++++ modules/google_organization/outputs.tf | 20 ++++++++++ modules/google_organization/variables.tf | 25 ++++++++++++ modules/gsuite_enabled/main.tf | 24 ++++++++++-- 6 files changed, 119 insertions(+), 28 deletions(-) create mode 100644 modules/google_organization/main.tf create mode 100644 modules/google_organization/outputs.tf create mode 100644 modules/google_organization/variables.tf diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index f07470aa..5d0d9c41 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -25,25 +25,26 @@ resource "random_id" "random_project_id_suffix" { Locals configuration *****************************************/ locals { - project_id = "${google_project.project.project_id}" - project_number = "${google_project.project.number}" - project_org_id = "${var.folder_id != "" ? "" : var.org_id}" - project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" - temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" - domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" - s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" - api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" - api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" - gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" - gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" - gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" - project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" - create_bucket = "${var.bucket_project != "" ? "true" : "false"}" - app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - shared_vpc_users = "${compact(list(local.s_account_fmt, local.group_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" - shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 - final_group_email = "${var.group_email != "" ? var.group_email : (var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : "")}" + project_id = "${google_project.project.project_id}" + project_number = "${google_project.project.number}" + project_org_id = "${var.folder_id != "" ? "" : var.org_id}" + project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" + temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" + args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" + s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" + api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" + api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" + gke_shared_vpc_enabled = "${var.shared_vpc != "" && contains(var.activate_apis, "container.googleapis.com") ? "true" : "false"}" + gke_s_account = "${format("service-%s@container-engine-robot.iam.gserviceaccount.com", local.project_number)}" + gke_s_account_fmt = "${local.gke_shared_vpc_enabled ? format("serviceAccount:%s", local.gke_s_account) : ""}" + project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" + create_bucket = "${var.bucket_project != "" ? "true" : "false"}" + app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" + shared_vpc_users = "${compact(list(local.s_account_fmt, local.group_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" + + # Workaround for https://github.com/hashicorp/terraform/issues/10857 + shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" + final_group_email = "${var.group_email != "" ? var.group_email : (var.group_name != "" ? format("%s@%s", var.group_name, module.google_organization.domain) : "")}" group_fmt = "${local.final_group_email != "" ? format("group:%s", local.final_group_email) : ""}" app_engine_config = { @@ -57,12 +58,14 @@ resource "null_resource" "args_missing" { "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true } -/****************************************** +/***************************************** Organization info retrieval *****************************************/ -data "google_organization" "org" { - count = "${var.org_id == "" ? 0 : 1}" - organization = "${var.org_id}" +module "google_organization" { + source = "../google_organization" + + domain = "${var.domain}" + org_id = "${var.org_id}" } resource "null_resource" "preconditions" { diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index c8c9c530..80583687 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -23,7 +23,7 @@ output "project_number" { } output "domain" { - value = "${local.domain}" + value = "${module.google_organization.domain}" description = "The organization's domain" } diff --git a/modules/google_organization/main.tf b/modules/google_organization/main.tf new file mode 100644 index 00000000..6165a661 --- /dev/null +++ b/modules/google_organization/main.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" +} + +/***************************************** + Organization info retrieval + *****************************************/ +data "google_organization" "org" { + count = "${var.org_id == "" ? 0 : 1}" + organization = "${var.org_id}" +} diff --git a/modules/google_organization/outputs.tf b/modules/google_organization/outputs.tf new file mode 100644 index 00000000..09e6ffbb --- /dev/null +++ b/modules/google_organization/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "domain" { + value = "${local.domain}" + description = "The organization's domain" +} diff --git a/modules/google_organization/variables.tf b/modules/google_organization/variables.tf new file mode 100644 index 00000000..6dac45d1 --- /dev/null +++ b/modules/google_organization/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "domain" { + description = "The domain name (optional if `org_id` is passed)" + default = "" +} + +variable "org_id" { + description = "The organization id (optional if `domain` is passed)" + default = "" +} diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 60616325..107ff896 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -20,11 +20,14 @@ locals { api_s_account = "${module.project-factory.api_s_account}" - domain = "${module.project-factory.domain}" // default group_name to ${project_name}-editors - group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" - given_group_email = "${var.create_group == "false" ? format("%s@%s", var.group_name, local.domain) : ""}" + group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" + + given_group_email = "${ + var.create_group == "false" ? format("%s@%s", var.group_name, module.google_organization.domain) : "" + }" + final_group_email = "${var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : local.given_group_email}" } @@ -41,13 +44,26 @@ resource "gsuite_group_member" "service_account_sa_group_member" { depends_on = ["module.project-factory"] } +/***************************************** + Organization info retrieval + *****************************************/ +module "google_organization" { + source = "../google_organization" + + domain = "${var.domain}" + org_id = "${var.org_id}" +} + /****************************************** Gsuite Group Configuration *****************************************/ resource "gsuite_group" "group" { count = "${var.create_group ? 1 : 0}" - email = "${var.group_name != "" ? format("%s@%s", var.group_name, local.domain) : format("%s-editors@%s", var.name, local.domain)}" + email = "${var.group_name != "" ? + format("%s@%s", var.group_name, module.google_organization.domain) : + format("%s-editors@%s", var.name, module.google_organization.domain)}" + name = "${var.group_name != "" ? var.group_name : format("%s-editors",var.name)}" description = "${var.name} project group" } From cc93cfcc0124ddaa7c2b8783575a5ce321990069 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 17:35:45 +0000 Subject: [PATCH 029/105] Pass full var.domain to gsuite_enabled --- test/fixtures/full/main.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/fixtures/full/main.tf b/test/fixtures/full/main.tf index 592ed695..3f989e68 100644 --- a/test/fixtures/full/main.tf +++ b/test/fixtures/full/main.tf @@ -16,7 +16,7 @@ provider "google" { credentials = "${file(var.credentials_path)}" - version = "~> 1.19" + version = "~> 1.19" } provider "gsuite" { @@ -57,7 +57,9 @@ module "vpc" { } module "project-factory" { - source = "../../../modules/gsuite_enabled" + source = "../../../modules/gsuite_enabled" + + domain = "${var.domain}" name = "${var.name}" random_project_id = true org_id = "${var.org_id}" From 7cbde2c389ff10694fa6b625b2b2a89b6155c7d6 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 18:02:38 +0000 Subject: [PATCH 030/105] Remove argument check from core_project_factory --- modules/core_project_factory/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 5d0d9c41..84ede6a8 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -30,7 +30,6 @@ locals { project_org_id = "${var.folder_id != "" ? "" : var.org_id}" project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" @@ -53,9 +52,6 @@ locals { } } -resource "null_resource" "args_missing" { - count = "${local.args_missing}" - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = true } /***************************************** From cdaff2f9a0e281f9d1aa34546e4f8f0d46336832 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 30 Nov 2018 18:02:56 +0000 Subject: [PATCH 031/105] Extract group identity logic to module --- modules/core_project_factory/main.tf | 21 ++++++++++------ modules/core_project_factory/outputs.tf | 2 +- modules/google_group/main.tf | 20 +++++++++++++++ modules/google_group/outputs.tf | 25 +++++++++++++++++++ modules/google_group/variables.tf | 33 +++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 modules/google_group/main.tf create mode 100644 modules/google_group/outputs.tf create mode 100644 modules/google_group/variables.tf diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 84ede6a8..483cd954 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -39,12 +39,13 @@ locals { project_bucket_name = "${var.bucket_name != "" ? var.bucket_name : format("%s-state", var.name)}" create_bucket = "${var.bucket_project != "" ? "true" : "false"}" app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - shared_vpc_users = "${compact(list(local.s_account_fmt, local.group_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" + + shared_vpc_users = "${ + compact(list(local.s_account_fmt, module.google_group.id, local.api_s_account_fmt, local.gke_s_account_fmt)) + }" # Workaround for https://github.com/hashicorp/terraform/issues/10857 shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" - final_group_email = "${var.group_email != "" ? var.group_email : (var.group_name != "" ? format("%s@%s", var.group_name, module.google_organization.domain) : "")}" - group_fmt = "${local.final_group_email != "" ? format("group:%s", local.final_group_email) : ""}" app_engine_config = { enabled = "${list(var.app_engine)}" @@ -52,6 +53,12 @@ locals { } } +module "google_group" { + source = "../google_group" + + domain = "${module.google_organization.domain}" + email = "${var.group_email}" + name = "${var.group_name}" } /***************************************** @@ -193,7 +200,7 @@ resource "google_project_iam_member" "gsuite_group_role" { project = "${local.project_id}" role = "${var.group_role}" - member = "${local.group_fmt}" + member = "${module.google_group.id}" } /****************************************** @@ -205,7 +212,7 @@ resource "google_service_account_iam_member" "service_account_grant_to_group" { service_account_id = "projects/${local.project_id}/serviceAccounts/${google_service_account.default_service_account.email}" role = "roles/iam.serviceAccountUser" - member = "${local.group_fmt}" + member = "${module.google_group.id}" } /************************************************************************************* @@ -244,7 +251,7 @@ resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { role = "roles/compute.networkUser" region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" project = "${var.shared_vpc}" - member = "${local.group_fmt}" + member = "${module.google_group.id}" } /************************************************************************************* @@ -293,7 +300,7 @@ resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_buck bucket = "${google_storage_bucket.project_bucket.name}" role = "roles/storage.admin" - member = "${local.group_fmt}" + member = "${module.google_group.id}" } /*********************************************** diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 80583687..1d28565a 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${var.group_name != "" ? local.final_group_email : ""}" + value = "${var.group_name != "" ? module.google_group.email : ""}" description = "The email of the created GSuite group with group_name" } diff --git a/modules/google_group/main.tf b/modules/google_group/main.tf new file mode 100644 index 00000000..0e9c5d72 --- /dev/null +++ b/modules/google_group/main.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + email = "${var.email != "" ? var.email : (var.name != "" ? format("%s@%s", var.name, var.domain) : "")}" + id = "${local.email != "" ? format("group:%s", local.email) : ""}" +} diff --git a/modules/google_group/outputs.tf b/modules/google_group/outputs.tf new file mode 100644 index 00000000..2664bda5 --- /dev/null +++ b/modules/google_group/outputs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "email" { + description = "The email of the group" + value = "${local.email}" +} + +output "id" { + description = "The identity of the group, in the format 'group:{email ID}'" + value = "${local.id}" +} diff --git a/modules/google_group/variables.tf b/modules/google_group/variables.tf new file mode 100644 index 00000000..85ca1fd8 --- /dev/null +++ b/modules/google_group/variables.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "domain" { + description = "The domain name (optional if `email` is passed)" + default = "" +} + +variable "email" { + description = "The email used for the group, this is automatically created" + default = "" +} + +variable "name" { + description = < Date: Mon, 3 Dec 2018 13:12:08 -0500 Subject: [PATCH 032/105] Refactor gsuite_enabled to use google_group --- modules/gsuite_enabled/main.tf | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 107ff896..5c43accb 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -24,11 +24,17 @@ locals { // default group_name to ${project_name}-editors group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" - given_group_email = "${ - var.create_group == "false" ? format("%s@%s", var.group_name, module.google_organization.domain) : "" + group_email = "${ + var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : + module.google_group.email }" +} + +module "google_group" { + source = "../google_group" - final_group_email = "${var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : local.given_group_email}" + domain = "${module.google_organization.domain}" + name = "${var.group_name}" } /*********************************************** @@ -91,7 +97,7 @@ module "project-factory" { billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" group_name = "${var.create_group ? gsuite_group.group.name : local.group_name}" - group_email = "${local.final_group_email}" + group_email = "${local.group_email}" group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" From d38be7d5d8e834566815323ad0eaf919189ead5c Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 13:28:48 -0500 Subject: [PATCH 033/105] Ignore VS Code directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7ba995d8..2ed756d0 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ crash.log terraform.tfstate.d/ *.auto.tfvars credentials.json +.vscode/ From cd4385dac23cf6bb47684adbd0b0f8fbbd7b912f Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 13:29:44 -0500 Subject: [PATCH 034/105] Generate documentation --- README.md | 3 -- examples/group_project/README.md | 2 +- examples/simple_project/README.md | 5 ++- modules/core_project_factory/README.md | 50 ++++++++++++++++++++++++++ modules/google_group/README.md | 20 +++++++++++ modules/google_organization/README.md | 18 ++++++++++ modules/gsuite_enabled/README.md | 50 ++++++++++++++++++++++++++ test/fixtures/full/README.md | 40 +++++++++++++++++++++ test/fixtures/minimal/README.md | 27 ++++++++++++++ 9 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 modules/google_group/README.md create mode 100644 modules/google_organization/README.md create mode 100644 test/fixtures/full/README.md create mode 100644 test/fixtures/minimal/README.md diff --git a/README.md b/README.md index 5e6d64c7..5f6c70ca 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,11 @@ The roles granted are specifically: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | activate\_apis | The list of apis to activate within the project | list | `` | no | -| api\_sa\_group | A GSuite group to place the Google APIs Service Account for the project in | string | `` | no | | app\_engine | A map for app engine configuration | map | `` | no | | auto\_create\_network | Create the default network | string | `false` | no | | billing\_account | The ID of the billing account to associate this project with | string | - | yes | | bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | -| create\_group | Whether to create the group or not | string | `false` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | | domain | The domain name (optional if `org_id` is passed) | string | `` | no | | folder\_id | The ID of a folder to host this project | string | `` | no | @@ -109,7 +107,6 @@ The roles granted are specifically: | name | The name for the project | string | - | yes | | org\_id | The organization id (optional if `domain` is passed) | string | `` | no | | random\_project\_id | Enables project random id generation | string | `false` | no | -| sa\_group | A GSuite group to place the default Service Account for the project in | string | `` | no | | sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | | shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | diff --git a/examples/group_project/README.md b/examples/group_project/README.md index 645cc00c..9a096b51 100644 --- a/examples/group_project/README.md +++ b/examples/group_project/README.md @@ -25,7 +25,7 @@ Expected variables: | billing\_account | The ID of the billing account to associate this project with | string | - | yes | | credentials\_file\_path | Service account json auth path | string | - | yes | | organization\_id | The organization id for the associated services | string | - | yes | -| project\_group\_name | The name of a GSuite group to create for controlling the project | string | `group-sample-project-owners` | no | +| project\_group\_name | The name of a GSuite group to create for controlling the project | string | - | yes | ## Outputs diff --git a/examples/simple_project/README.md b/examples/simple_project/README.md index 5c254bc9..32cbaec4 100755 --- a/examples/simple_project/README.md +++ b/examples/simple_project/README.md @@ -14,10 +14,9 @@ Expected variables: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| admin\_email | Admin user email on Gsuite | string | - | yes | -| billing\_account | - | string | - | yes | +| billing\_account | The ID of the billing account to associate this project with | string | - | yes | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| organization\_id | - | string | - | yes | +| organization\_id | The organization id for the associated services | string | - | yes | ## Outputs diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index e69de29b..b106452f 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -0,0 +1,50 @@ +# core_project_factory + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| activate\_apis | The list of apis to activate within the project | list | `` | no | +| app\_engine | A map for app engine configuration | map | `` | no | +| auto\_create\_network | Create the default network | string | `false` | no | +| billing\_account | The ID of the billing account to associate this project with | string | - | yes | +| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | +| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | +| domain | The domain name (optional if `org_id` is passed) | string | `` | no | +| folder\_id | The ID of a folder to host this project | string | `` | no | +| group\_email | The email used for the group, this is automatically created | string | `` | no | +| group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | +| labels | Map of labels for project | map | `` | no | +| name | The name for the project | string | - | yes | +| org\_id | The organization id (optional if `domain` is passed) | string | `` | no | +| random\_project\_id | Enables project random id generation | string | `false` | no | +| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | +| shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | +| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `` | no | +| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| api\_s\_account | API service account email | +| api\_s\_account\_fmt | API service account email formatted for terraform use | +| app\_engine\_enabled | Whether app engine is enabled | +| domain | The organization's domain | +| group\_email | The email of the created GSuite group with group_name | +| project\_bucket\_self\_link | Project's bucket selfLink | +| project\_bucket\_url | Project's bucket url | +| project\_id | - | +| project\_number | - | +| service\_account\_display\_name | The display name of the default service account | +| service\_account\_email | The email of the default service account | +| service\_account\_id | The id of the default service account | +| service\_account\_name | The fully-qualified name of the default service account | +| service\_account\_unique\_id | The unique id of the default service account | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/google_group/README.md b/modules/google_group/README.md new file mode 100644 index 00000000..bcb51528 --- /dev/null +++ b/modules/google_group/README.md @@ -0,0 +1,20 @@ +# google_group + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| domain | The domain name (optional if `email` is passed) | string | `` | no | +| email | The email used for the group, this is automatically created | string | `` | no | +| name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| email | The email of the group | +| id | The identity of the group, in the format 'group:{email ID}' | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/google_organization/README.md b/modules/google_organization/README.md new file mode 100644 index 00000000..7ecd548c --- /dev/null +++ b/modules/google_organization/README.md @@ -0,0 +1,18 @@ +# google_organization + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| domain | The domain name (optional if `org_id` is passed) | string | `` | no | +| org\_id | The organization id (optional if `domain` is passed) | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| domain | The organization's domain | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index e69de29b..91f7e46b 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -0,0 +1,50 @@ +# gsuite_enabled + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| activate\_apis | The list of apis to activate within the project | list | `` | no | +| api\_sa\_group | A GSuite group to place the Google APIs Service Account for the project in | string | `` | no | +| app\_engine | A map for app engine configuration | map | `` | no | +| auto\_create\_network | Create the default network | string | `false` | no | +| billing\_account | The ID of the billing account to associate this project with | string | - | yes | +| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | +| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | +| create\_group | Whether to create the group or not | string | `false` | no | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | +| domain | The domain name (optional if `org_id` is passed) | string | `` | no | +| folder\_id | The ID of a folder to host this project | string | `` | no | +| group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | +| labels | Map of labels for project | map | `` | no | +| name | The name for the project | string | - | yes | +| org\_id | The organization id for the associated services | string | `` | no | +| random\_project\_id | Enables project random id generation | string | `false` | no | +| sa\_group | A GSuite group to place the default Service Account for the project in | string | `` | no | +| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | +| shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | +| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `` | no | +| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| app\_engine\_enabled | Whether app engine is enabled | +| domain | The organization's domain | +| group\_email | The email of the created GSuite group with group_name | +| project\_bucket\_self\_link | Project's bucket selfLink | +| project\_bucket\_url | Project's bucket url | +| project\_id | - | +| project\_number | - | +| service\_account\_display\_name | The display name of the default service account | +| service\_account\_email | The email of the default service account | +| service\_account\_id | The id of the default service account | +| service\_account\_name | The fully-qualified name of the default service account | +| service\_account\_unique\_id | The unique id of the default service account | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md new file mode 100644 index 00000000..68f41d5d --- /dev/null +++ b/test/fixtures/full/README.md @@ -0,0 +1,40 @@ +# full + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| billing\_account | - | string | - | yes | +| create\_group | - | string | `false` | no | +| credentials\_path | - | string | - | yes | +| domain | - | string | - | yes | +| folder\_id | - | string | `` | no | +| group\_name | - | string | `` | no | +| group\_role | - | string | `roles/viewer` | no | +| gsuite\_admin\_account | - | string | - | yes | +| name | - | string | `pf-test-int-full` | no | +| org\_id | - | string | - | yes | +| region | - | string | `us-east4` | no | +| sa\_group | - | string | `` | no | +| sa\_role | - | string | `roles/editor` | no | +| shared\_vpc | - | string | `` | no | +| usage\_bucket\_name | - | string | `` | no | +| usage\_bucket\_prefix | - | string | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | +| domain | - | +| extra\_service\_account\_email | - | +| group\_email | - | +| gsuite\_admin\_account | - | +| project\_id | - | +| project\_number | - | +| region | - | +| service\_account\_email | - | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/test/fixtures/minimal/README.md b/test/fixtures/minimal/README.md new file mode 100644 index 00000000..9300b200 --- /dev/null +++ b/test/fixtures/minimal/README.md @@ -0,0 +1,27 @@ +# minimal + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| billing\_account | - | string | - | yes | +| credentials\_path | - | string | - | yes | +| domain | - | string | `` | no | +| folder\_id | - | string | `` | no | +| gsuite\_admin\_account | - | string | `` | no | +| name | - | string | `pf-test-int-minimal` | no | +| org\_id | - | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | +| domain | - | +| project\_id | - | +| project\_number | - | +| service\_account\_email | - | + +[^]: (autogen_docs_end) \ No newline at end of file From b66ecb86fc566af6b924e38c99bc27c0ae01050a Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 13:30:08 -0500 Subject: [PATCH 035/105] Ignore pyenv directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2ed756d0..6f615fcd 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ terraform.tfstate.d/ *.auto.tfvars credentials.json .vscode/ +env/ From 966a5d0797dbf369928e236aa35a21ad6de404f0 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 13:30:24 -0500 Subject: [PATCH 036/105] Print name of file in combine_docfiles.py --- helpers/combine_docfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py index b01af7c5..156ca0da 100755 --- a/helpers/combine_docfiles.py +++ b/helpers/combine_docfiles.py @@ -33,7 +33,7 @@ if len(sys.argv) != 3: sys.exit(1) - +print(sys.argv[1]) input = open(sys.argv[1], "r").read() replace_content = open(sys.argv[2], "r").read() From bb6fc1d58045c9aee124860c55867bcd1ffec5b2 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:10:14 -0500 Subject: [PATCH 037/105] Add name generation to google_group --- modules/core_project_factory/main.tf | 7 ++++--- modules/google_group/README.md | 2 ++ modules/google_group/main.tf | 5 +++-- modules/google_group/outputs.tf | 5 +++++ modules/google_group/variables.tf | 4 ++++ modules/gsuite_enabled/main.tf | 12 +++++------- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 483cd954..29a0e499 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -56,9 +56,10 @@ locals { module "google_group" { source = "../google_group" - domain = "${module.google_organization.domain}" - email = "${var.group_email}" - name = "${var.group_name}" + domain = "${module.google_organization.domain}" + email = "${var.group_email}" + name = "${var.group_name}" + project_name = "${var.name}" } /***************************************** diff --git a/modules/google_group/README.md b/modules/google_group/README.md index bcb51528..aec4c6ff 100644 --- a/modules/google_group/README.md +++ b/modules/google_group/README.md @@ -9,6 +9,7 @@ | domain | The domain name (optional if `email` is passed) | string | `` | no | | email | The email used for the group, this is automatically created | string | `` | no | | name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | +| project\_name | The name of the project in which the group will exist | string | - | yes | ## Outputs @@ -16,5 +17,6 @@ |------|-------------| | email | The email of the group | | id | The identity of the group, in the format 'group:{email ID}' | +| name | The name of the group | [^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/google_group/main.tf b/modules/google_group/main.tf index 0e9c5d72..93ea9fa5 100644 --- a/modules/google_group/main.tf +++ b/modules/google_group/main.tf @@ -15,6 +15,7 @@ */ locals { - email = "${var.email != "" ? var.email : (var.name != "" ? format("%s@%s", var.name, var.domain) : "")}" - id = "${local.email != "" ? format("group:%s", local.email) : ""}" + email = "${var.email != "" ? var.email : format("%s@%s", local.name, var.domain)}" + id = "${format("group:%s", local.email)}" + name = "${var.name != "" ? var.name : format("%s-editors", var.project_name)}" } diff --git a/modules/google_group/outputs.tf b/modules/google_group/outputs.tf index 2664bda5..9971f8a8 100644 --- a/modules/google_group/outputs.tf +++ b/modules/google_group/outputs.tf @@ -23,3 +23,8 @@ output "id" { description = "The identity of the group, in the format 'group:{email ID}'" value = "${local.id}" } + +output "name" { + description = "The name of the group" + value = "${local.name}" +} diff --git a/modules/google_group/variables.tf b/modules/google_group/variables.tf index 85ca1fd8..9786fa2f 100644 --- a/modules/google_group/variables.tf +++ b/modules/google_group/variables.tf @@ -31,3 +31,7 @@ variable "name" { default = "" } + +variable "project_name" { + description = "The name of the project in which the group will exist" +} diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 5c43accb..ec1473c5 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -21,9 +21,6 @@ locals { api_s_account = "${module.project-factory.api_s_account}" - // default group_name to ${project_name}-editors - group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" - group_email = "${ var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : module.google_group.email @@ -33,8 +30,9 @@ locals { module "google_group" { source = "../google_group" - domain = "${module.google_organization.domain}" - name = "${var.group_name}" + domain = "${module.google_organization.domain}" + name = "${var.group_name}" + project_name = "${var.name}" } /*********************************************** @@ -70,8 +68,8 @@ resource "gsuite_group" "group" { format("%s@%s", var.group_name, module.google_organization.domain) : format("%s-editors@%s", var.name, module.google_organization.domain)}" - name = "${var.group_name != "" ? var.group_name : format("%s-editors",var.name)}" description = "${var.name} project group" + name = "${module.google_group.name}" } /*********************************************** @@ -96,8 +94,8 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_name = "${var.create_group ? gsuite_group.group.name : local.group_name}" group_email = "${local.group_email}" + group_name = "${module.google_group.name}" group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" From 7f5cf8ce8a2abafb7efbbf7d52dde4827f564a44 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:10:49 -0500 Subject: [PATCH 038/105] Move G Suite group role to gsuite_enabled --- modules/core_project_factory/main.tf | 11 ----------- modules/gsuite_enabled/main.tf | 9 +++++++++ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 29a0e499..d74f1503 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -193,17 +193,6 @@ resource "google_project_iam_member" "default_service_account_membership" { member = "${local.s_account_fmt}" } -/****************************************** - Gsuite Group Role Configuration - *****************************************/ -resource "google_project_iam_member" "gsuite_group_role" { - count = "${var.group_name != "" ? 1 : 0}" - - project = "${local.project_id}" - role = "${var.group_role}" - member = "${module.google_group.id}" -} - /****************************************** Granting serviceAccountUser to group *****************************************/ diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index ec1473c5..27654cd8 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -109,3 +109,12 @@ module "project-factory" { auto_create_network = "${var.auto_create_network}" app_engine = "${var.app_engine}" } + +/****************************************** + Gsuite Group Role Configuration + *****************************************/ +resource "google_project_iam_member" "gsuite_group_role" { + member = "${module.google_group.id}" + project = "${module.project-factory.project_id}" + role = "${var.group_role}" +} From 124bff35d6cae7abfb67165886c8111bbd68ce6c Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:12:24 -0500 Subject: [PATCH 039/105] Use google_group.email in gsuite_enabled --- modules/gsuite_enabled/main.tf | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 27654cd8..5e22239f 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -20,11 +20,6 @@ locals { api_s_account = "${module.project-factory.api_s_account}" - - group_email = "${ - var.create_group == "true" ? element(coalescelist(gsuite_group.group.*.email, list("")), 0) : - module.google_group.email - }" } module "google_group" { @@ -64,11 +59,8 @@ module "google_organization" { resource "gsuite_group" "group" { count = "${var.create_group ? 1 : 0}" - email = "${var.group_name != "" ? - format("%s@%s", var.group_name, module.google_organization.domain) : - format("%s-editors@%s", var.name, module.google_organization.domain)}" - description = "${var.name} project group" + email = "${module.google_group.email}" name = "${module.google_group.name}" } @@ -94,8 +86,8 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_email = "${local.group_email}" group_name = "${module.google_group.name}" + group_email = "${module.google_group.email}" group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" From 213a2c612d495c1f2a500a4a63e007c63d2fe7fd Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:13:24 -0500 Subject: [PATCH 040/105] Refactor unnecessary local in gsuite_enabled --- modules/gsuite_enabled/main.tf | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 5e22239f..c4eb5502 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -14,14 +14,6 @@ * limitations under the License. */ -/****************************************** - Locals configuration - *****************************************/ - -locals { - api_s_account = "${module.project-factory.api_s_account}" -} - module "google_group" { source = "../google_group" @@ -71,7 +63,7 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { count = "${var.api_sa_group != "" ? 1 : 0}" group = "${var.api_sa_group}" - email = "${local.api_s_account}" + email = "${module.project-factory.api_s_account}" role = "MEMBER" } From 6b1e404be5bc7050e4293249f71e849318f29fc9 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:13:51 -0500 Subject: [PATCH 041/105] Remove G Suite from minimal fixture --- test/fixtures/minimal/README.md | 1 - test/fixtures/minimal/main.tf | 14 +------------- test/fixtures/minimal/terraform.tfvars.example | 5 ----- test/fixtures/minimal/variables.tf | 4 ---- 4 files changed, 1 insertion(+), 23 deletions(-) diff --git a/test/fixtures/minimal/README.md b/test/fixtures/minimal/README.md index 9300b200..52a51c6e 100644 --- a/test/fixtures/minimal/README.md +++ b/test/fixtures/minimal/README.md @@ -10,7 +10,6 @@ | credentials\_path | - | string | - | yes | | domain | - | string | `` | no | | folder\_id | - | string | `` | no | -| gsuite\_admin\_account | - | string | `` | no | | name | - | string | `pf-test-int-minimal` | no | | org\_id | - | string | - | yes | diff --git a/test/fixtures/minimal/main.tf b/test/fixtures/minimal/main.tf index 7dab5600..47374c3d 100644 --- a/test/fixtures/minimal/main.tf +++ b/test/fixtures/minimal/main.tf @@ -16,19 +16,7 @@ provider "google" { credentials = "${file(var.credentials_path)}" - version = "~> 1.19" -} - -provider "gsuite" { - credentials = "${file(var.credentials_path)}" - impersonated_user_email = "${var.gsuite_admin_account}" - - oauth_scopes = [ - "https://www.googleapis.com/auth/admin.directory.group", - "https://www.googleapis.com/auth/admin.directory.group.member", - ] - - version = "~> 0.1.9" + version = "~> 1.19" } module "project-factory" { diff --git a/test/fixtures/minimal/terraform.tfvars.example b/test/fixtures/minimal/terraform.tfvars.example index 7650b25d..bd977a5c 100644 --- a/test/fixtures/minimal/terraform.tfvars.example +++ b/test/fixtures/minimal/terraform.tfvars.example @@ -24,8 +24,3 @@ billing_account="000000-000000-000000" # The service account credentials to use when running Terraform. credentials_path="/cftk/workdir/credentials.json" - -# A G Suite admin account to impersonate. This isn't used within the minimal -# test suite, but must be specified until the gsuite dependency becomes -# optional. -gsuite_admin_account="admin@example.com" diff --git a/test/fixtures/minimal/variables.tf b/test/fixtures/minimal/variables.tf index 7f0160a6..755adc30 100644 --- a/test/fixtures/minimal/variables.tf +++ b/test/fixtures/minimal/variables.tf @@ -30,8 +30,4 @@ variable "domain" { default = "" } -variable "gsuite_admin_account" { - default = "" -} - variable "billing_account" {} From 67cf72c20793dc90ffda0ec28ce75602914f35a5 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:36:49 -0500 Subject: [PATCH 042/105] Remove group_role from core_project_factory This variable is only necessary on gsuite_enabled. --- main.tf | 1 - modules/core_project_factory/README.md | 1 - modules/core_project_factory/variables.tf | 5 ----- modules/gsuite_enabled/main.tf | 1 - test/fixtures/full/README.md | 5 +++++ 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 06b29408..26fc4882 100755 --- a/main.tf +++ b/main.tf @@ -26,7 +26,6 @@ module "project-factory" { billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" group_name = "${var.group_name}" - group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index b106452f..455e09ae 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -17,7 +17,6 @@ | folder\_id | The ID of a folder to host this project | string | `` | no | | group\_email | The email used for the group, this is automatically created | string | `` | no | | group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | -| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | | labels | Map of labels for project | map | `` | no | | name | The name for the project | string | - | yes | | org\_id | The organization id (optional if `domain` is passed) | string | `` | no | diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 698bd584..a1669924 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -63,11 +63,6 @@ variable "group_email" { default = "" } -variable "group_role" { - description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" - default = "roles/editor" -} - variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index c4eb5502..b44b9f22 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -80,7 +80,6 @@ module "project-factory" { folder_id = "${var.folder_id}" group_name = "${module.google_group.name}" group_email = "${module.google_group.email}" - group_role = "${var.group_role}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md index 68f41d5d..3c5eedb9 100644 --- a/test/fixtures/full/README.md +++ b/test/fixtures/full/README.md @@ -31,10 +31,15 @@ | domain | - | | extra\_service\_account\_email | - | | group\_email | - | +| group\_role | - | | gsuite\_admin\_account | - | | project\_id | - | | project\_number | - | | region | - | +| sa\_role | - | | service\_account\_email | - | +| shared\_vpc | - | +| usage\_bucket\_name | - | +| usage\_bucket\_prefix | - | [^]: (autogen_docs_end) \ No newline at end of file From e96a53fbae176eafd4c7d2f0c719d879acf6efb5 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 16:48:43 -0500 Subject: [PATCH 043/105] Grant group serviceAccountUser in gsuite_enabled --- modules/core_project_factory/main.tf | 12 ------------ modules/gsuite_enabled/main.tf | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index d74f1503..ceb9173a 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -193,18 +193,6 @@ resource "google_project_iam_member" "default_service_account_membership" { member = "${local.s_account_fmt}" } -/****************************************** - Granting serviceAccountUser to group - *****************************************/ -resource "google_service_account_iam_member" "service_account_grant_to_group" { - count = "${var.group_name != "" ? 1 : 0}" - - service_account_id = "projects/${local.project_id}/serviceAccounts/${google_service_account.default_service_account.email}" - role = "roles/iam.serviceAccountUser" - - member = "${module.google_group.id}" -} - /************************************************************************************* compute.networkUser role granted to GSuite group, APIs Service account, Project Service Account, and GKE Service Account on shared VPC *************************************************************************************/ diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index b44b9f22..d2bc0643 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -101,3 +101,15 @@ resource "google_project_iam_member" "gsuite_group_role" { project = "${module.project-factory.project_id}" role = "${var.group_role}" } + +/****************************************** + Granting serviceAccountUser to group + *****************************************/ +resource "google_service_account_iam_member" "service_account_grant_to_group" { + member = "${module.google_group.id}" + role = "roles/iam.serviceAccountUser" + + service_account_id = < Date: Mon, 3 Dec 2018 17:12:33 -0500 Subject: [PATCH 044/105] Move G Suite networkUser role to gsuite_enabled --- modules/core_project_factory/main.tf | 7 +++---- modules/gsuite_enabled/main.tf | 11 +++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index ceb9173a..c23710e0 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -40,9 +40,7 @@ locals { create_bucket = "${var.bucket_project != "" ? "true" : "false"}" app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - shared_vpc_users = "${ - compact(list(local.s_account_fmt, module.google_group.id, local.api_s_account_fmt, local.gke_s_account_fmt)) - }" + shared_vpc_users = "${compact(list(local.s_account_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" @@ -194,7 +192,8 @@ resource "google_project_iam_member" "default_service_account_membership" { } /************************************************************************************* - compute.networkUser role granted to GSuite group, APIs Service account, Project Service Account, and GKE Service Account on shared VPC + compute.networkUser role granted to APIs Service account, Project Service Account, and GKE Service Account on shared + VPC *************************************************************************************/ resource "google_project_iam_member" "controlling_group_vpc_membership" { count = "${(var.shared_vpc != "" && (length(compact(var.shared_vpc_subnets)) > 0)) ? local.shared_vpc_users_length : 0}" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index d2bc0643..e906e1bb 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -113,3 +113,14 @@ resource "google_service_account_iam_member" "service_account_grant_to_group" { projects/${module.project-factory.project_id}/serviceAccounts/${module.project-factory.service_account_email} EOS } + +/************************************************************************************* + compute.networkUser role granted to GSuite group on shared VPC + *************************************************************************************/ +resource "google_project_iam_member" "controlling_group_vpc_membership" { + count = "${(var.shared_vpc != "" && (length(compact(var.shared_vpc_subnets)) > 0)) ? 1 : 0}" + + member = "${module.google_group.id}" + project = "${var.shared_vpc}" + role = "roles/compute.networkUser" +} From ecbbc22b68d28a52debb9bc882a3afad2b9850d5 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 17:14:17 -0500 Subject: [PATCH 045/105] Move networkUser for VPC subnets to gsuite_enabled --- modules/core_project_factory/main.tf | 13 ------------- modules/gsuite_enabled/main.tf | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index c23710e0..1cb9196b 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -218,19 +218,6 @@ resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_sub member = "${local.s_account_fmt}" } -/************************************************************************************* - compute.networkUser role granted to GSuite group on vpc subnets - *************************************************************************************/ -resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && var.group_name != "" ? length(var.shared_vpc_subnets) : 0 }" - - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" - role = "roles/compute.networkUser" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - project = "${var.shared_vpc}" - member = "${module.google_group.id}" -} - /************************************************************************************* compute.networkUser role granted to APIs Service Account on vpc subnets *************************************************************************************/ diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index e906e1bb..bd2d6bbd 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -124,3 +124,16 @@ resource "google_project_iam_member" "controlling_group_vpc_membership" { project = "${var.shared_vpc}" role = "roles/compute.networkUser" } + +/************************************************************************************* + compute.networkUser role granted to GSuite group on vpc subnets + *************************************************************************************/ +resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" + + member = "${module.google_group.id}" + project = "${var.shared_vpc}" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + role = "roles/compute.networkUser" + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" +} From 60e1ad7ac00b4511d1394b970470e88469728e5a Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 17:15:12 -0500 Subject: [PATCH 046/105] Move bucket admin for group to gsuite_enabled --- modules/core_project_factory/main.tf | 11 ----------- modules/core_project_factory/outputs.tf | 5 +++++ modules/gsuite_enabled/main.tf | 11 +++++++++++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 1cb9196b..3b1bf5af 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -256,17 +256,6 @@ resource "google_storage_bucket" "project_bucket" { project = "${var.bucket_project}" } -/*********************************************** - Project's bucket storage.admin granting to group - ***********************************************/ -resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { - count = "${local.create_bucket && var.group_name != "" ? 1 : 0}" - - bucket = "${google_storage_bucket.project_bucket.name}" - role = "roles/storage.admin" - member = "${module.google_group.id}" -} - /*********************************************** Project's bucket storage.admin granting to default compute service account ***********************************************/ diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 1d28565a..eefab023 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -57,6 +57,11 @@ output "service_account_unique_id" { description = "The unique id of the default service account" } +output "project_bucket_name" { + description = "The name of the projec's bucket" + value = "${google_storage_bucket.project_bucket.*.name}" +} + output "project_bucket_self_link" { value = "${google_storage_bucket.project_bucket.*.self_link}" description = "Project's bucket selfLink" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index bd2d6bbd..eb2f6bee 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -137,3 +137,14 @@ resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { role = "roles/compute.networkUser" subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" } + +/*********************************************** + Project's bucket storage.admin granting to group + ***********************************************/ +resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { + count = "${var.bucket_project != "" ? 1 : 0}" + + bucket = "${module.project-factory.project_bucket_name}" + member = "${module.google_group.id}" + role = "roles/storage.admin" +} From 4325f64c48bf95ed22cc776ebc7d675b8d6540fd Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 17:16:02 -0500 Subject: [PATCH 047/105] Remove obsolete code from core_project_factory --- modules/core_project_factory/main.tf | 9 --------- modules/core_project_factory/outputs.tf | 5 ----- modules/core_project_factory/variables.tf | 10 ---------- modules/gsuite_enabled/outputs.tf | 2 +- 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 3b1bf5af..29734a6c 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -51,15 +51,6 @@ locals { } } -module "google_group" { - source = "../google_group" - - domain = "${module.google_organization.domain}" - email = "${var.group_email}" - name = "${var.group_name}" - project_name = "${var.name}" -} - /***************************************** Organization info retrieval *****************************************/ diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index eefab023..9d20246b 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -27,11 +27,6 @@ output "domain" { description = "The organization's domain" } -output "group_email" { - value = "${var.group_name != "" ? module.google_group.email : ""}" - description = "The email of the created GSuite group with group_name" -} - output "service_account_id" { value = "${google_service_account.default_service_account.account_id}" description = "The id of the default service account" diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index a1669924..46b30199 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -53,16 +53,6 @@ variable "folder_id" { default = "" } -variable "group_name" { - description = "A group to control the project by being assigned group_role - defaults to ${project_name}-editors" - default = "" -} - -variable "group_email" { - description = "The email used for the group, this is automatically created" - default = "" -} - variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" diff --git a/modules/gsuite_enabled/outputs.tf b/modules/gsuite_enabled/outputs.tf index 11d0904f..c490ed43 100644 --- a/modules/gsuite_enabled/outputs.tf +++ b/modules/gsuite_enabled/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${module.project-factory.group_email}" + value = "${module.google_group.email}" description = "The email of the created GSuite group with group_name" } From b92126ca599fefe260bb1f7dda515f17efb3dfbb Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 3 Dec 2018 17:16:19 -0500 Subject: [PATCH 048/105] Update generated documentation --- modules/core_project_factory/README.md | 4 +--- test/fixtures/full/README.md | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 455e09ae..219ff363 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -15,8 +15,6 @@ | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | | domain | The domain name (optional if `org_id` is passed) | string | `` | no | | folder\_id | The ID of a folder to host this project | string | `` | no | -| group\_email | The email used for the group, this is automatically created | string | `` | no | -| group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | | labels | Map of labels for project | map | `` | no | | name | The name for the project | string | - | yes | | org\_id | The organization id (optional if `domain` is passed) | string | `` | no | @@ -35,7 +33,7 @@ | api\_s\_account\_fmt | API service account email formatted for terraform use | | app\_engine\_enabled | Whether app engine is enabled | | domain | The organization's domain | -| group\_email | The email of the created GSuite group with group_name | +| project\_bucket\_name | The name of the projec's bucket | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | | project\_id | - | diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md index 3c5eedb9..68f41d5d 100644 --- a/test/fixtures/full/README.md +++ b/test/fixtures/full/README.md @@ -31,15 +31,10 @@ | domain | - | | extra\_service\_account\_email | - | | group\_email | - | -| group\_role | - | | gsuite\_admin\_account | - | | project\_id | - | | project\_number | - | | region | - | -| sa\_role | - | | service\_account\_email | - | -| shared\_vpc | - | -| usage\_bucket\_name | - | -| usage\_bucket\_prefix | - | [^]: (autogen_docs_end) \ No newline at end of file From d1c1c5c1ff98e4aea2bcdecc61005ca0389fe91b Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 4 Dec 2018 11:39:31 -0500 Subject: [PATCH 049/105] Remove obsolete attributes use in gsuite_enabled --- modules/gsuite_enabled/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index eb2f6bee..b26e2d67 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -78,8 +78,6 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_name = "${module.google_group.name}" - group_email = "${module.google_group.email}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" From f7bc912e94097dbbc0da933d2545a0d079c59e77 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 4 Dec 2018 11:40:10 -0500 Subject: [PATCH 050/105] Fix multiline split of service_account_id --- modules/gsuite_enabled/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index b26e2d67..62394eec 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -107,9 +107,9 @@ resource "google_service_account_iam_member" "service_account_grant_to_group" { member = "${module.google_group.id}" role = "roles/iam.serviceAccountUser" - service_account_id = < Date: Tue, 4 Dec 2018 13:58:36 -0500 Subject: [PATCH 051/105] Remove obsolete variables from root module --- main.tf | 1 - variables.tf | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/main.tf b/main.tf index 26fc4882..5238c4fe 100755 --- a/main.tf +++ b/main.tf @@ -25,7 +25,6 @@ module "project-factory" { shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" folder_id = "${var.folder_id}" - group_name = "${var.group_name}" sa_role = "${var.sa_role}" activate_apis = "${var.activate_apis}" usage_bucket_name = "${var.usage_bucket_name}" diff --git a/variables.tf b/variables.tf index 6db45303..a8c994dd 100755 --- a/variables.tf +++ b/variables.tf @@ -47,16 +47,6 @@ variable "folder_id" { default = "" } -variable "group_name" { - description = "A group to control the project by being assigned group_role (defaults to project editor)" - default = "" -} - -variable "group_role" { - description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" - default = "roles/editor" -} - variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" From 7e047c8a163f07b68d3f879867504402c8632d21 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 7 Dec 2018 13:40:31 -0500 Subject: [PATCH 052/105] Update Read Mes of root, gsuite_enabled --- README.md | 62 ++++++++++++---------------- modules/gsuite_enabled/README.md | 69 +++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 5f6c70ca..fbede636 100644 --- a/README.md +++ b/README.md @@ -9,81 +9,72 @@ access, Service Accounts, and API enablement to follow best practices. ## Usage -There are multiple examples included in the [examples](./examples/) folder but -simple usage is as follows: +There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: ```hcl module "project-factory" { - source = "terraform-google-modules/project-factory/google" - version = "v0.3.0" + source = "terraform-google-modules/project-factory/google" + version = "v0.3.0" - name = "pf-test-1" - random_project_id = "true" - org_id = "1234567890" - usage_bucket_name = "pf-test-1-usage-report-bucket" - usage_bucket_prefix = "pf/test/1/integration" - billing_account = "ABCDEF-ABCDEF-ABCDEF" - group_role = "roles/editor" - shared_vpc = "shared_vpc_host_name" - sa_group = "test_sa_group@yourdomain.com" - credentials_path = "${local.credentials_file_path}" + billing_account = "ABCDEF-ABCDEF-ABCDEF" + credentials_path = "${local.credentials_file_path}" + name = "pf-test-1" + org_id = "1234567890" + random_project_id = "true" + shared_vpc = "shared_vpc_host_name" shared_vpc_subnets = [ "projects/base-project-196723/regions/us-east1/subnetworks/default", "projects/base-project-196723/regions/us-central1/subnetworks/default", "projects/base-project-196723/regions/us-central1/subnetworks/subnet-1", ] + + usage_bucket_name = "pf-test-1-usage-report-bucket" + usage_bucket_prefix = "pf/test/1/integration" } ``` -### Features +## Features The Project Factory module will take the following actions: 1. Create a new GCP project using the `project_name`. -1. If a shared VPC is specified, attach the new project to the `shared_vpc`. +1. If a shared VPC is specified, attach the new project to the + `shared_vpc`. It will also give the following users network access on the specified subnets: - - The prroject's new default service account (see step 4) + - The project's new default service account (see step 4) - The Google API service account for the project - - The project controlling group specified in `group_name` 1. Delete the default compute service account. 1. Create a new default service account for the project. - 1. Give it access to the shared VPC (to be able to launch instances). - 1. Add it to the `sa_group` in Google Groups, if specified. + 1. Give it access to the shared VPC + (to be able to launch instances). 1. Attach the billing account (`billing_account`) to the project. -1. Create a new Google Group for the project (`group_name`) if `create_group` is - `true`. 1. Give the controlling group access to the project, with the `group_role`. 1. Enable the required and specified APIs (`activate_apis`). 1. Delete the default network. 1. Enable usage report for GCE into central project bucket (`target_usage_bucket`), if provided. -1. If specified, create the GCS bucket `bucket_name` and give the following - groups Storage Admin on it: +1. If specified, create the GCS bucket `bucket_name` and give the + following accounts Storage Admin on it: 1. The controlling group (`group_name`) - 1. The new default compute service account created for the project - 1. The Google APIs service account for the project -1. Add the Google APIs service account to the `api_sa_group` (if specified) + 1. The new default compute service account created for the project. + 1. The Google APIs service account for the project. The roles granted are specifically: - New Default Service Account - `compute.networkUser` on host project or specified subnets - `storage.admin` on `bucket_name` GCS bucket - - MEMBER of the specified `sa_group` -- `group_name` is the new controlling group - - `compute.networkUser` on host project or specific subnets - - Specified `group_role` on project - - `iam.serviceAccountUser` on the default Service Account - - `storage.admin` on `bucket_name` GCS bucket - Google APIs Service Account - `compute.networkUser` on host project or specified subnets - `storage.admin` on `bucket_name` GCS bucket - - MEMBER of the specified `api_sa_group` + +To include G Suite integration, use the +[gsuite_enabled module][gsuite-enabled-module]. [^]: (autogen_docs_start) @@ -100,8 +91,6 @@ The roles granted are specifically: | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | | domain | The domain name (optional if `org_id` is passed) | string | `` | no | | folder\_id | The ID of a folder to host this project | string | `` | no | -| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `` | no | -| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | | name | The name for the project | string | - | yes | @@ -399,6 +388,7 @@ target to work. See the Terraform documentation for more info on [releasing new versions][release-new-version]. +[gsuite-enabled-module]: modules/gsuite_enabled/README.md [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google [terraform-provider-gsuite]: https://github.com/DeviaVir/terraform-provider-gsuite [glossary]: /docs/GLOSSARY.md diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index 91f7e46b..245eb0ad 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -1,4 +1,66 @@ -# gsuite_enabled +# Google Cloud Project Factory with G Suite Terraform Module + +This module performs the same functions as the +[root module][root-module] with the addition of integrating G Suite. + +## Usage + +There are multiple examples included in the [examples] folder but simple usage is as follows: + +```hcl +module "project-factory" { + source = "terraform-google-modules/project-factory/google//modules/gsuite_enabled" + version = "0.2.1" + + billing_account = "ABCDEF-ABCDEF-ABCDEF" + create_group = "true" + credentials_path = "${local.credentials_file_path}" + group_name = "test_sa_group" + group_role = "roles/editor" + name = "pf-test-1" + org_id = "1234567890" + random_project_id = "true" + sa_group = "test_sa_group@yourdomain.com" + shared_vpc = "shared_vpc_host_name" + + shared_vpc_subnets = [ + "projects/base-project-196723/regions/us-east1/subnetworks/default", + "projects/base-project-196723/regions/us-central1/subnetworks/default", + "projects/base-project-196723/regions/us-central1/subnetworks/subnet-1", + ] + + usage_bucket_name = "pf-test-1-usage-report-bucket" + usage_bucket_prefix = "pf/test/1/integration" +} +``` + +## Features + +The G Suite Enabled module will perform the following actions in +addition to those of the root module: + +1. Create a new Google group for the project using `group_name` if + `create_group` is `"true"`. +1. Give the group access to the project with the `group_role`. +1. Give the project controlling group specified in `group_name` network + access on the specified subnets if `shared_vpc` is specified. +1. Add the new default service account for the project to the + `sa_group` in Google Groups, if specified. +1. Give the group Storage Admin on `bucket_name`, if specified. +1. Add the Google APIs service account to the `api_sa_group`, + if specified. + +The roles granted are specifically: + +- New Default Service Account + - MEMBER of the specified `sa_group` +- `group_name` is the new controlling group + - `compute.networkUser` on host project or specific subnets + - Specified `group_role` on project + - `iam.serviceAccountUser` on the default Service Account + - `storage.admin` on `bucket_name` GCS bucket +- Google APIs Service Account + - MEMBER of the specified `api_sa_group` [^]: (autogen_docs_start) @@ -47,4 +109,7 @@ | service\_account\_name | The fully-qualified name of the default service account | | service\_account\_unique\_id | The unique id of the default service account | -[^]: (autogen_docs_end) \ No newline at end of file +[^]: (autogen_docs_end) + +[examples]: ../../examples/ +[root-module]: ../../README.md From cde56e04e3da32fe88b9821f6f7759773a4a4ef4 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 12 Dec 2018 10:14:20 -0500 Subject: [PATCH 053/105] Remove obsolete reference to group_email --- outputs.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/outputs.tf b/outputs.tf index 11d0904f..0341ca62 100755 --- a/outputs.tf +++ b/outputs.tf @@ -27,11 +27,6 @@ output "domain" { description = "The organization's domain" } -output "group_email" { - value = "${module.project-factory.group_email}" - description = "The email of the created GSuite group with group_name" -} - output "service_account_id" { value = "${module.project-factory.service_account_id}" description = "The id of the default service account" From 5a133b3940eafdc576179fc5e968ee76d7f491e7 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 12 Dec 2018 10:28:11 -0500 Subject: [PATCH 054/105] Update generated documentation --- README.md | 1 - modules/core_project_factory/README.md | 1 + modules/gsuite_enabled/README.md | 3 ++- test/fixtures/full/README.md | 5 +++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fbede636..ed5418fe 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,6 @@ To include G Suite integration, use the |------|-------------| | app\_engine\_enabled | Whether app engine is enabled | | domain | The organization's domain | -| group\_email | The email of the created GSuite group with group_name | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | | project\_id | - | diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 219ff363..61adedb5 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -16,6 +16,7 @@ | domain | The domain name (optional if `org_id` is passed) | string | `` | no | | folder\_id | The ID of a folder to host this project | string | `` | no | | labels | Map of labels for project | map | `` | no | +| lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | | name | The name for the project | string | - | yes | | org\_id | The organization id (optional if `domain` is passed) | string | `` | no | | random\_project\_id | Enables project random id generation | string | `false` | no | diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index 245eb0ad..c721e40f 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -82,6 +82,7 @@ The roles granted are specifically: | group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | | group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | | labels | Map of labels for project | map | `` | no | +| lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | | name | The name for the project | string | - | yes | | org\_id | The organization id for the associated services | string | `` | no | | random\_project\_id | Enables project random id generation | string | `false` | no | @@ -112,4 +113,4 @@ The roles granted are specifically: [^]: (autogen_docs_end) [examples]: ../../examples/ -[root-module]: ../../README.md +[root-module]: ../../README.md \ No newline at end of file diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md index 68f41d5d..3c5eedb9 100644 --- a/test/fixtures/full/README.md +++ b/test/fixtures/full/README.md @@ -31,10 +31,15 @@ | domain | - | | extra\_service\_account\_email | - | | group\_email | - | +| group\_role | - | | gsuite\_admin\_account | - | | project\_id | - | | project\_number | - | | region | - | +| sa\_role | - | | service\_account\_email | - | +| shared\_vpc | - | +| usage\_bucket\_name | - | +| usage\_bucket\_prefix | - | [^]: (autogen_docs_end) \ No newline at end of file From 26ffa955dfa7b3f7812ffb9431a9fc83b7a2e45b Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 13 Dec 2018 16:25:13 -0500 Subject: [PATCH 055/105] Inline google_group to gsuite_enabled This module is no longer necessary since all group logic has been removed from core_project_factory. --- modules/google_group/README.md | 22 ------------------ modules/google_group/main.tf | 21 ------------------ modules/google_group/outputs.tf | 30 ------------------------- modules/google_group/variables.tf | 37 ------------------------------- modules/gsuite_enabled/main.tf | 24 +++++++++----------- modules/gsuite_enabled/outputs.tf | 2 +- 6 files changed, 12 insertions(+), 124 deletions(-) delete mode 100644 modules/google_group/README.md delete mode 100644 modules/google_group/main.tf delete mode 100644 modules/google_group/outputs.tf delete mode 100644 modules/google_group/variables.tf diff --git a/modules/google_group/README.md b/modules/google_group/README.md deleted file mode 100644 index aec4c6ff..00000000 --- a/modules/google_group/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# google_group - -[^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| domain | The domain name (optional if `email` is passed) | string | `` | no | -| email | The email used for the group, this is automatically created | string | `` | no | -| name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | -| project\_name | The name of the project in which the group will exist | string | - | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| email | The email of the group | -| id | The identity of the group, in the format 'group:{email ID}' | -| name | The name of the group | - -[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/google_group/main.tf b/modules/google_group/main.tf deleted file mode 100644 index 93ea9fa5..00000000 --- a/modules/google_group/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - email = "${var.email != "" ? var.email : format("%s@%s", local.name, var.domain)}" - id = "${format("group:%s", local.email)}" - name = "${var.name != "" ? var.name : format("%s-editors", var.project_name)}" -} diff --git a/modules/google_group/outputs.tf b/modules/google_group/outputs.tf deleted file mode 100644 index 9971f8a8..00000000 --- a/modules/google_group/outputs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "email" { - description = "The email of the group" - value = "${local.email}" -} - -output "id" { - description = "The identity of the group, in the format 'group:{email ID}'" - value = "${local.id}" -} - -output "name" { - description = "The name of the group" - value = "${local.name}" -} diff --git a/modules/google_group/variables.tf b/modules/google_group/variables.tf deleted file mode 100644 index 9786fa2f..00000000 --- a/modules/google_group/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "domain" { - description = "The domain name (optional if `email` is passed)" - default = "" -} - -variable "email" { - description = "The email used for the group, this is automatically created" - default = "" -} - -variable "name" { - description = < 0)) ? 1 : 0}" - member = "${module.google_group.id}" + member = "${local.group_id}" project = "${var.shared_vpc}" role = "roles/compute.networkUser" } @@ -129,7 +127,7 @@ resource "google_project_iam_member" "controlling_group_vpc_membership" { resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" - member = "${module.google_group.id}" + member = "${local.group_id}" project = "${var.shared_vpc}" region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" role = "roles/compute.networkUser" @@ -143,6 +141,6 @@ resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_buck count = "${var.bucket_project != "" ? 1 : 0}" bucket = "${module.project-factory.project_bucket_name}" - member = "${module.google_group.id}" + member = "${local.group_id}" role = "roles/storage.admin" } diff --git a/modules/gsuite_enabled/outputs.tf b/modules/gsuite_enabled/outputs.tf index c490ed43..40a4a2db 100644 --- a/modules/gsuite_enabled/outputs.tf +++ b/modules/gsuite_enabled/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${module.google_group.email}" + value = "${local.group_email}" description = "The email of the created GSuite group with group_name" } From 40879dfd1044a57262f7a329db09dc2b5beb2257 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Fri, 28 Sep 2018 14:09:10 -0700 Subject: [PATCH 056/105] Add scaffolding for python migration script --- helpers/migrate.py | 145 +++++++++++++++++++++++++++++++++++ test/helpers/test_migrate.py | 38 +++++++++ 2 files changed, 183 insertions(+) create mode 100755 helpers/migrate.py create mode 100755 test/helpers/test_migrate.py diff --git a/helpers/migrate.py b/helpers/migrate.py new file mode 100755 index 00000000..5a7e0d5b --- /dev/null +++ b/helpers/migrate.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +"""usage: migrate.py """ + +import argparse +import pprint +import subprocess +import sys + + +class TerraformModule: + """ + A Terraform module with associated resources. + """ + + def __init__(self, name, resources): + """ + Create a new module and associate it with a list of resources. + """ + self.name = name + self.resources = resources + + def has_resource(self, resource_type, resource_name): + """ + Does this module contain a resource with the matching type and name? + """ + for resource in self.resources: + if resource.resource_type == resource_type \ + and resource.name == resource_name: + return True + return False + + def __repr__(self): + return "{}({!r}, {!r})".format( + self.__class__.__name__, + self.name, + [repr(resource) for resource in self.resources]) + + +class TerraformResource: + """ + A Terraform resource, defined by the the identifier of that resource. + + >>> path = 'module.project-factory.google_project.project' + >>> resource = TerraformResource.from_path(path) + >>> assert resource.module == 'module.project-factory' + >>> assert resource.resource_type == 'google_project' + >>> assert resource.name == 'project' + """ + + @classmethod + def from_path(cls, path): + """ + Generate a new Terraform resource, based on the fully qualified + Terraform resource path. + """ + parts = path.split(".") + name = parts.pop() + resource_type = parts.pop() + module = ".".join(parts) + return cls(module, resource_type, name) + + def __init__(self, module, resource_type, name): + """ + Create a new TerraformResource from a pre-parsed path. + """ + self.module = module + self.resource_type = resource_type + self.name = name + + def path(self): + """ + Return the fully qualified resource path. + """ + parts = [self.module, self.resource_type, self.name] + if parts[0] == '': + del parts[0] + return ".".join(parts) + + def __repr__(self): + return "{}({!r}, {!r}, {!r})".format( + self.__class__.__name__, + self.module, + self.resource_type, + self.name) + + +def group(resources): + """ + Group a set of resources according to their containing module. + """ + + groups = {} + for resource in resources: + if resource.module in groups: + groups[resource.module].append(resource) + else: + groups[resource.module] = [resource] + + return [ + TerraformModule(name, contained) + for name, contained in groups.items() + ] + + +def read_state(statefile): + """ + Read the terraform state at the given path. + """ + argv = ["terraform", "state", "list", "-state", statefile] + result = subprocess.run(argv, + capture_output=True, + check=True, + encoding='utf-8') + elements = result.stdout.split("\n") + elements.pop() + return elements + + +def main(argv): + parser = argparser() + args = parser.parse_args(argv[1:]) + + resources = [ + TerraformResource.from_path(path) + for path in read_state(args.oldstate) + ] + pprint.pprint(resources) + modules = group(resources) + pprint.pprint(modules) + print("done") + + +def argparser(): + parser = argparse.ArgumentParser(description='Migrate Terraform state') + parser.add_argument('oldstate', metavar='oldstate.json', + help='The current Terraform state (will not be ' + 'modified)') + parser.add_argument('newstate', metavar='newstate.json', + help='The path to the new state file') + return parser + + +if __name__ == "__main__": + main(sys.argv) diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py new file mode 100755 index 00000000..1987c4a9 --- /dev/null +++ b/test/helpers/test_migrate.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +import os +import sys +sys.path.append( + os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '../../helpers/'))) + +import unittest # noqa: E402 +import migrate # noqa: E402 + + +class TestTerraformResource(unittest.TestCase): + + def test_resource_init(self): + resource = migrate.TerraformResource('', 'google_project', 'project') + assert resource.module == '' + assert resource.resource_type == 'google_project' + assert resource.name == 'project' + + def test_resource_path_no_module(self): + resource = migrate.TerraformResource('', 'google_project', 'project') + assert resource.path() == 'google_project.project' + + def test_resource_path_with_module(self): + resource = migrate.TerraformResource( + 'module.project-factory', + 'google_project', + 'project') + expected = 'module.project-factory.google_project.project' + actual = resource.path() + assert expected == actual + + +if __name__ == "__main__": + unittest.main() From b9705d776869cc19bbd77a1a878463e6b39874af Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Fri, 28 Sep 2018 17:28:04 -0700 Subject: [PATCH 057/105] Implement GSuiteMigration class This commit adds a class that will generate `terraform state mv` commands to update a Terraform statefile to match the changes made in the G Suite refactor. There are some outstanding questions about how to handle Terraform configurations that use the G Suite resources since it's non-trivial to determine when this change happened, so this commit will need follow up work to fully implement it. --- helpers/migrate.py | 161 +++++++++++++++++++++++++++++++---- test/helpers/test_migrate.py | 71 +++++++++++++-- 2 files changed, 205 insertions(+), 27 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 5a7e0d5b..156d448b 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -3,11 +3,96 @@ """usage: migrate.py """ import argparse -import pprint +import copy import subprocess import sys +class GSuiteMigration: + """ + Migrate the resources from a flat project factory to match the new + module structure created by the G Suite refactor. + """ + + GSUITE_ENABLED_SELECTORS = [ + ("gsuite_group", "group"), + ("gsuite_group_member", "api_s_account_api_sa_group_member"), + ("gsuite_group_member", "service_account_sa_group_member"), + ("null_data_source", "data_given_group_email"), + ("null_data_source", "data_final_group_email"), + ] + + CORE_PROJECT_FACTORY_SELECTORS = [ + ("google_compute_shared_vpc_service_project", None), + ("google_compute_subnetwork_iam_member", None), + ("google_project", None), + ("google_project_iam_member", None), + ("google_project_service", None), + ("google_project_usage_export_bucket", None), + ("google_service_account", None), + ("google_service_account_iam_member", None), + ("google_storage_bucket", None), + ("google_storage_bucket_iam_member", None), + ("null_resource", None), + ("random_id", None), + ] + + def __init__(self, project_factory): + self.project_factory = project_factory + + def commands(self): + return self.mv_gsuite_resources() + \ + self.mv_core_project_factory_resources() + + def gsuite_enabled_resources(self): + """ + A list of resources that will be moved to the `gsuite_enabled` module. + """ + selectors = self.__class__.GSUITE_ENABLED_SELECTORS + return self._select_resources(selectors) + + def mv_gsuite_resources(self): + return self._mv_resources( + self.gsuite_enabled_resources(), + ".module.gsuite-enabled") + + def core_project_factory_resources(self): + """ + A list of resources that will be moved to the `core_project_factory` + module. + """ + selectors = self.__class__.CORE_PROJECT_FACTORY_SELECTORS + return self._select_resources(selectors) + + def mv_core_project_factory_resources(self): + return self._mv_resources( + self.core_project_factory_resources(), + ".module.project-factory") + + def _mv_resources(self, resources, path): + mv_commands = [] + for old in resources: + new = copy.deepcopy(old) + new.module += path + + cmd = "terraform state mv {} {}".format(old.path(), new.path()) + mv_commands.append(cmd) + + return mv_commands + + def _select_resources(self, selectors): + to_move = [] + for selector in selectors: + resource_type = selector[0] + resource_name = selector[1] + matching_resources = self.project_factory.get_resources( + resource_type, + resource_name) + to_move += matching_resources + + return to_move + + class TerraformModule: """ A Terraform module with associated resources. @@ -20,21 +105,45 @@ def __init__(self, name, resources): self.name = name self.resources = resources - def has_resource(self, resource_type, resource_name): + def get_resources(self, resource_type=None, resource_name=None): + """ + Return a list of resources matching the given resource type and name. + """ + + ret = [] + for resource in self.resources: + matches_type = (resource_type is None or + resource_type == resource.resource_type) + + matches_name = (resource_name is None or + resource_name == resource.name) + + if matches_type and matches_name: + ret.append(resource) + + return ret + + def has_resource(self, resource_type=None, resource_name=None): """ Does this module contain a resource with the matching type and name? """ for resource in self.resources: - if resource.resource_type == resource_type \ - and resource.name == resource_name: + matches_type = (resource_type is None or + resource_type == resource.resource_type) + + matches_name = (resource_name is None or + resource_name == resource.name) + + if matches_type and matches_name: return True + return False def __repr__(self): return "{}({!r}, {!r})".format( - self.__class__.__name__, - self.name, - [repr(resource) for resource in self.resources]) + self.__class__.__name__, + self.name, + [repr(resource) for resource in self.resources]) class TerraformResource: @@ -79,10 +188,10 @@ def path(self): def __repr__(self): return "{}({!r}, {!r}, {!r})".format( - self.__class__.__name__, - self.module, - self.resource_type, - self.name) + self.__class__.__name__, + self.module, + self.resource_type, + self.name) def group(resources): @@ -98,8 +207,8 @@ def group(resources): groups[resource.module] = [resource] return [ - TerraformModule(name, contained) - for name, contained in groups.items() + TerraformModule(name, contained) + for name, contained in groups.items() ] @@ -122,13 +231,29 @@ def main(argv): args = parser.parse_args(argv[1:]) resources = [ - TerraformResource.from_path(path) - for path in read_state(args.oldstate) + TerraformResource.from_path(path) + for path in read_state(args.oldstate) ] - pprint.pprint(resources) + modules = group(resources) - pprint.pprint(modules) - print("done") + factories = [ + module for module in modules + if module.has_resource("random_id", "random_project_id_suffix") + ] + + print("---- Candidate modules:") + for factory in factories: + print("-- " + factory.name) + + print("---- Plan:") + to_move = [] + for factory in factories: + migration = GSuiteMigration(factory) + + to_move += migration.commands() + + for cmd in to_move: + print(cmd) def argparser(): diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index 1987c4a9..7b9635b2 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -1,16 +1,72 @@ #!/usr/bin/env python3 +import unittest import os import sys sys.path.append( - os.path.abspath( - os.path.join( - os.path.dirname(__file__), - '../../helpers/'))) + os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '../../helpers/'))) -import unittest # noqa: E402 import migrate # noqa: E402 +TERRAFORM_STATE_LIST = """google_project_iam_member.additive_sa_role +google_project_iam_member.additive_shared_vpc_role +google_service_account.extra_service_account +google_service_account_iam_member.additive_service_account_grant_to_group +module.project-factory.google_compute_default_service_account.default +module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment +module.project-factory.google_project.project +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3] +module.project-factory.google_project_iam_member.default_service_account_membership +module.project-factory.google_project_iam_member.gke_host_agent +module.project-factory.google_project_iam_member.gsuite_group_role +module.project-factory.google_project_service.project_services[0] +module.project-factory.google_project_service.project_services[1] +module.project-factory.google_project_usage_export_bucket.usage_report_export +module.project-factory.google_service_account.default_service_account +module.project-factory.google_service_account_iam_member.service_account_grant_to_group +module.project-factory.null_data_source.data_final_group_email +module.project-factory.null_data_source.data_given_group_email +module.project-factory.null_data_source.data_group_email_format +module.project-factory.null_resource.delete_default_compute_service_account +module.project-factory.random_id.random_project_id_suffix""" + + +class TestTerraformModule(unittest.TestCase): + def setUp(self): + self.resources = [ + migrate.TerraformResource.from_path(path) + for path in TERRAFORM_STATE_LIST.split("\n") + ] + + self.module = migrate.TerraformModule( + 'module.project-factory', + self.resources) + + def test_has_resource(self): + assert self.module.has_resource('google_project', 'project') + assert self.module.has_resource(None, 'project') + assert self.module.has_resource('google_project', None) + assert self.module.has_resource(None, None) + + def test_has_resource_empty(self): + assert self.module.has_resource('google_compute_instance', None) is False + + def test_get_resources(self): + expected = [resource for resource in self.resources + if resource.resource_type == 'google_project_iam_member'] + actual = self.module.get_resources('google_project_iam_member', None) + assert actual == expected + + def test_get_resources_empty(self): + actual = self.module.get_resources('google_compute_instance', None) + assert actual == [] + class TestTerraformResource(unittest.TestCase): @@ -25,10 +81,7 @@ def test_resource_path_no_module(self): assert resource.path() == 'google_project.project' def test_resource_path_with_module(self): - resource = migrate.TerraformResource( - 'module.project-factory', - 'google_project', - 'project') + resource = migrate.TerraformResource('module.project-factory', 'google_project', 'project') expected = 'module.project-factory.google_project.project' actual = resource.path() assert expected == actual From 97f4fe2c0d41fd72bb27b9e797a342da8b3c3ea3 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 1 Oct 2018 11:52:17 -0700 Subject: [PATCH 058/105] Only migrate resources into core_project_factory Resources formerly defined in `gsuite_override.tf` don't need to be migrated, since they're in the top level module. All other resources can be uniformly migrated into the nested `project-factory` module. --- helpers/migrate.py | 110 ++++++++++++++++++----------------- test/helpers/test_migrate.py | 46 ++++++++++++++- 2 files changed, 102 insertions(+), 54 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 156d448b..1de57020 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -"""usage: migrate.py """ - import argparse import copy import subprocess import sys +import shutil class GSuiteMigration: @@ -14,15 +13,11 @@ class GSuiteMigration: module structure created by the G Suite refactor. """ - GSUITE_ENABLED_SELECTORS = [ - ("gsuite_group", "group"), - ("gsuite_group_member", "api_s_account_api_sa_group_member"), - ("gsuite_group_member", "service_account_sa_group_member"), - ("null_data_source", "data_given_group_email"), - ("null_data_source", "data_final_group_email"), - ] - CORE_PROJECT_FACTORY_SELECTORS = [ + ("google_organization", "org"), + ("google_compute_default_service_account", "default"), + ("null_data_source", "data_final_group_email"), + ("null_data_source", "data_group_email_format"), ("google_compute_shared_vpc_service_project", None), ("google_compute_subnetwork_iam_member", None), ("google_project", None), @@ -40,49 +35,30 @@ class GSuiteMigration: def __init__(self, project_factory): self.project_factory = project_factory - def commands(self): - return self.mv_gsuite_resources() + \ - self.mv_core_project_factory_resources() - - def gsuite_enabled_resources(self): + def moves(self): """ - A list of resources that will be moved to the `gsuite_enabled` module. + Generate the set of old/new resource pairs that will be migrated + to the `core_project_factory` module. """ - selectors = self.__class__.GSUITE_ENABLED_SELECTORS - return self._select_resources(selectors) + resources = self.targets() + moves = [] + for old in resources: + new = copy.deepcopy(old) + new.module += ".module.project-factory" - def mv_gsuite_resources(self): - return self._mv_resources( - self.gsuite_enabled_resources(), - ".module.gsuite-enabled") + pair = (old.path(), new.path()) + moves.append(pair) - def core_project_factory_resources(self): + return moves + + def targets(self): """ A list of resources that will be moved to the `core_project_factory` module. """ - selectors = self.__class__.CORE_PROJECT_FACTORY_SELECTORS - return self._select_resources(selectors) - - def mv_core_project_factory_resources(self): - return self._mv_resources( - self.core_project_factory_resources(), - ".module.project-factory") - - def _mv_resources(self, resources, path): - mv_commands = [] - for old in resources: - new = copy.deepcopy(old) - new.module += path - - cmd = "terraform state mv {} {}".format(old.path(), new.path()) - mv_commands.append(cmd) - - return mv_commands - - def _select_resources(self, selectors): to_move = [] - for selector in selectors: + + for selector in self.__class__.CORE_PROJECT_FACTORY_SELECTORS: resource_type = selector[0] resource_name = selector[1] matching_resources = self.project_factory.get_resources( @@ -226,34 +202,59 @@ def read_state(statefile): return elements -def main(argv): - parser = argparser() - args = parser.parse_args(argv[1:]) +def migrate(statefile, dryrun=False): + """ + Migrate the terraform state in `statefile` to match the post-refactor + resource structure. + """ + # Generate a list of Terraform resource states from the output of + # `terraform state list` resources = [ TerraformResource.from_path(path) - for path in read_state(args.oldstate) + for path in read_state(statefile) ] + # Group resources based on the module where they're defined. modules = group(resources) + + # Filter our list of Terraform modules down to anything that lookst like a + # project-factory module. We key this off the presence off of + # `random_id.random_project_id_suffix` since that should almost always be + # unique to a project-factory module. factories = [ module for module in modules if module.has_resource("random_id", "random_project_id_suffix") ] - print("---- Candidate modules:") + print("---- Migrating the following project-factory modules:") for factory in factories: print("-- " + factory.name) - print("---- Plan:") + # Collect a list of resources for each project factory that need to be + # migrated. to_move = [] for factory in factories: migration = GSuiteMigration(factory) + to_move += migration.moves() + + for (old, new) in to_move: + argv = ["terraform", "state", "mv", "-state", statefile, old, new] + if dryrun: + print(" ".join(argv)) + else: + subprocess.run(argv, check=True, encoding='utf-8') + + +def main(argv): + parser = argparser() + args = parser.parse_args(argv[1:]) - to_move += migration.commands() + print("cp {} {}".format(args.oldstate, args.newstate)) + shutil.copy(args.oldstate, args.newstate) - for cmd in to_move: - print(cmd) + migrate(args.newstate, dryrun=args.dryrun) + print("State migration complete, verify migration with `terraform plan`") def argparser(): @@ -263,6 +264,9 @@ def argparser(): 'modified)') parser.add_argument('newstate', metavar='newstate.json', help='The path to the new state file') + parser.add_argument('--dryrun', action='store_true', + help='Print the `terraform state mv` commands instead ' + 'of running the commands.') return parser diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index 7b9635b2..7effa04b 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -import unittest +import copy import os import sys +import unittest sys.path.append( os.path.abspath( os.path.join( @@ -36,12 +37,55 @@ module.project-factory.null_resource.delete_default_compute_service_account module.project-factory.random_id.random_project_id_suffix""" +TERRAFORM_MIGRATED_RESOURCES = """module.project-factory.google_compute_default_service_account.default +module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment +module.project-factory.google_project.project +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] +module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3] +module.project-factory.google_project_iam_member.default_service_account_membership +module.project-factory.google_project_iam_member.gke_host_agent +module.project-factory.google_project_iam_member.gsuite_group_role +module.project-factory.google_project_service.project_services[0] +module.project-factory.google_project_service.project_services[1] +module.project-factory.google_project_usage_export_bucket.usage_report_export +module.project-factory.google_service_account.default_service_account +module.project-factory.google_service_account_iam_member.service_account_grant_to_group +module.project-factory.null_data_source.data_final_group_email +module.project-factory.null_data_source.data_group_email_format +module.project-factory.null_resource.delete_default_compute_service_account +module.project-factory.random_id.random_project_id_suffix""" + + +class TestGSuiteMigration(unittest.TestCase): + def setUp(self): + self.resources = [ + migrate.TerraformResource.from_path(path) + for path in TERRAFORM_MIGRATED_RESOURCES.split("\n") + ] + + module = migrate.TerraformModule( + 'module.project-factory', + self.resources) + self.migration = migrate.GSuiteMigration(module) + + def test_moves(self): + moves = self.migration.moves() + + for old in self.resources: + new = copy.deepcopy(old) + new.module += ".module.project-factory" + move = (old.path(), new.path()) + assert move in moves + class TestTerraformModule(unittest.TestCase): def setUp(self): self.resources = [ migrate.TerraformResource.from_path(path) for path in TERRAFORM_STATE_LIST.split("\n") + if path.startswith("module.project-factory") ] self.module = migrate.TerraformModule( From 405d3c8c0cc697798b0094f2eb9c000edff0f2af Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 1 Oct 2018 12:26:43 -0700 Subject: [PATCH 059/105] Add boilerplate to migrate.py and tests --- helpers/migrate.py | 14 ++++++++++++++ test/helpers/test_migrate.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index 1de57020..8189e82a 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -1,5 +1,19 @@ #!/usr/bin/env python3 +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import copy import subprocess diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index 7effa04b..e5df393c 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -1,5 +1,19 @@ #!/usr/bin/env python3 +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import copy import os import sys From 59f70ac4a7ccbf7d09a5691f1e2d23cce3704973 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 6 Dec 2018 10:44:49 -0800 Subject: [PATCH 060/105] Prefer arrays over multiline strings for test fixtures --- test/helpers/test_migrate.py | 96 +++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index e5df393c..762508a0 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -26,57 +26,61 @@ import migrate # noqa: E402 -TERRAFORM_STATE_LIST = """google_project_iam_member.additive_sa_role -google_project_iam_member.additive_shared_vpc_role -google_service_account.extra_service_account -google_service_account_iam_member.additive_service_account_grant_to_group -module.project-factory.google_compute_default_service_account.default -module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment -module.project-factory.google_project.project -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3] -module.project-factory.google_project_iam_member.default_service_account_membership -module.project-factory.google_project_iam_member.gke_host_agent -module.project-factory.google_project_iam_member.gsuite_group_role -module.project-factory.google_project_service.project_services[0] -module.project-factory.google_project_service.project_services[1] -module.project-factory.google_project_usage_export_bucket.usage_report_export -module.project-factory.google_service_account.default_service_account -module.project-factory.google_service_account_iam_member.service_account_grant_to_group -module.project-factory.null_data_source.data_final_group_email -module.project-factory.null_data_source.data_given_group_email -module.project-factory.null_data_source.data_group_email_format -module.project-factory.null_resource.delete_default_compute_service_account -module.project-factory.random_id.random_project_id_suffix""" - -TERRAFORM_MIGRATED_RESOURCES = """module.project-factory.google_compute_default_service_account.default -module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment -module.project-factory.google_project.project -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2] -module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3] -module.project-factory.google_project_iam_member.default_service_account_membership -module.project-factory.google_project_iam_member.gke_host_agent -module.project-factory.google_project_iam_member.gsuite_group_role -module.project-factory.google_project_service.project_services[0] -module.project-factory.google_project_service.project_services[1] -module.project-factory.google_project_usage_export_bucket.usage_report_export -module.project-factory.google_service_account.default_service_account -module.project-factory.google_service_account_iam_member.service_account_grant_to_group -module.project-factory.null_data_source.data_final_group_email -module.project-factory.null_data_source.data_group_email_format -module.project-factory.null_resource.delete_default_compute_service_account -module.project-factory.random_id.random_project_id_suffix""" +TERRAFORM_STATE_LIST = [ + "google_project_iam_member.additive_sa_role", + "google_project_iam_member.additive_shared_vpc_role", + "google_service_account.extra_service_account", + "google_service_account_iam_member.additive_service_account_grant_to_group", + "module.project-factory.google_compute_default_service_account.default", + "module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment", + "module.project-factory.google_project.project", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3]", + "module.project-factory.google_project_iam_member.default_service_account_membership", + "module.project-factory.google_project_iam_member.gke_host_agent", + "module.project-factory.google_project_iam_member.gsuite_group_role", + "module.project-factory.google_project_service.project_services[0]", + "module.project-factory.google_project_service.project_services[1]", + "module.project-factory.google_project_usage_export_bucket.usage_report_export", + "module.project-factory.google_service_account.default_service_account", + "module.project-factory.google_service_account_iam_member.service_account_grant_to_group", + "module.project-factory.null_data_source.data_final_group_email", + "module.project-factory.null_data_source.data_given_group_email", + "module.project-factory.null_data_source.data_group_email_format", + "module.project-factory.null_resource.delete_default_compute_service_account", + "module.project-factory.random_id.random_project_id_suffix" +] + +TERRAFORM_MIGRATED_RESOURCES = [ + "module.project-factory.google_compute_default_service_account.default", + "module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment", + "module.project-factory.google_project.project", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2]", + "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3]", + "module.project-factory.google_project_iam_member.default_service_account_membership", + "module.project-factory.google_project_iam_member.gke_host_agent", + "module.project-factory.google_project_iam_member.gsuite_group_role", + "module.project-factory.google_project_service.project_services[0]", + "module.project-factory.google_project_service.project_services[1]", + "module.project-factory.google_project_usage_export_bucket.usage_report_export", + "module.project-factory.google_service_account.default_service_account", + "module.project-factory.google_service_account_iam_member.service_account_grant_to_group", + "module.project-factory.null_data_source.data_final_group_email", + "module.project-factory.null_data_source.data_group_email_format", + "module.project-factory.null_resource.delete_default_compute_service_account", + "module.project-factory.random_id.random_project_id_suffix", +] class TestGSuiteMigration(unittest.TestCase): def setUp(self): self.resources = [ migrate.TerraformResource.from_path(path) - for path in TERRAFORM_MIGRATED_RESOURCES.split("\n") + for path in TERRAFORM_MIGRATED_RESOURCES ] module = migrate.TerraformModule( @@ -98,7 +102,7 @@ class TestTerraformModule(unittest.TestCase): def setUp(self): self.resources = [ migrate.TerraformResource.from_path(path) - for path in TERRAFORM_STATE_LIST.split("\n") + for path in TERRAFORM_STATE_LIST if path.startswith("module.project-factory") ] From 248bb6960cdc2a1ddbc92edd0e6f900e6a98ca21 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 6 Dec 2018 10:53:54 -0800 Subject: [PATCH 061/105] Add test coverage for `Resource.from_path` --- test/helpers/test_migrate.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index 762508a0..f46c4e46 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -132,6 +132,18 @@ def test_get_resources_empty(self): class TestTerraformResource(unittest.TestCase): + def test_root_resource_from_path(self): + resource = migrate.TerraformResource.from_path("google_project.project") + assert resource.module == '' + assert resource.resource_type == 'google_project' + assert resource.name == 'project' + + def test_module_resource_from_path(self): + resource = migrate.TerraformResource.from_path("module.project-factory.google_project.project") + assert resource.module == 'module.project-factory' + assert resource.resource_type == 'google_project' + assert resource.name == 'project' + def test_resource_init(self): resource = migrate.TerraformResource('', 'google_project', 'project') assert resource.module == '' From 056f47bbd30e8623d678a771998a00703e32b5b4 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 6 Dec 2018 11:16:18 -0800 Subject: [PATCH 062/105] Add validation and negative test case for `.from_path` --- helpers/migrate.py | 4 ++++ test/helpers/test_migrate.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index 8189e82a..35f9ffe6 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -19,6 +19,7 @@ import subprocess import sys import shutil +import re class GSuiteMigration: @@ -153,6 +154,9 @@ def from_path(cls, path): Generate a new Terraform resource, based on the fully qualified Terraform resource path. """ + if re.match(r'\A[\w.\[\]-]+\Z', path) is None: + raise ValueError("Invalid Terraform resource path {!r}".format(path)) + parts = path.split(".") name = parts.pop() resource_type = parts.pop() diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index f46c4e46..b2e4dfec 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -144,6 +144,11 @@ def test_module_resource_from_path(self): assert resource.resource_type == 'google_project' assert resource.name == 'project' + def test_invalid_resource_from_path(self): + self.assertRaises( + Exception, + lambda: migrate.TerraformResource.from_path("not a resource path!")) + def test_resource_init(self): resource = migrate.TerraformResource('', 'google_project', 'project') assert resource.module == '' From dbf879dfce1a94acc54523973cc7415e3199fb5e Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 6 Dec 2018 11:16:44 -0800 Subject: [PATCH 063/105] Remove extraneous resources and list comprehension filter --- test/helpers/test_migrate.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index b2e4dfec..d1b8fe78 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -27,10 +27,6 @@ import migrate # noqa: E402 TERRAFORM_STATE_LIST = [ - "google_project_iam_member.additive_sa_role", - "google_project_iam_member.additive_shared_vpc_role", - "google_service_account.extra_service_account", - "google_service_account_iam_member.additive_service_account_grant_to_group", "module.project-factory.google_compute_default_service_account.default", "module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment", "module.project-factory.google_project.project", @@ -103,7 +99,6 @@ def setUp(self): self.resources = [ migrate.TerraformResource.from_path(path) for path in TERRAFORM_STATE_LIST - if path.startswith("module.project-factory") ] self.module = migrate.TerraformModule( From 6ce03c7c868be245e0f3c3a69a41eb476976056e Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Wed, 12 Dec 2018 16:39:16 -0800 Subject: [PATCH 064/105] Improve command hint to use new terraform state --- helpers/migrate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 35f9ffe6..5a17df8e 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -272,7 +272,8 @@ def main(argv): shutil.copy(args.oldstate, args.newstate) migrate(args.newstate, dryrun=args.dryrun) - print("State migration complete, verify migration with `terraform plan`") + print("State migration complete, verify migration with " + "`terraform plan -state '{}'`".format(args.newstate)) def argparser(): From 227e4a5e699c328ce43e2d6314cde5850016ae79 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Wed, 12 Dec 2018 17:02:44 -0800 Subject: [PATCH 065/105] Improve name of `group_by_module` function --- helpers/migrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 5a17df8e..2b5b3fba 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -188,7 +188,7 @@ def __repr__(self): self.name) -def group(resources): +def group_by_module(resources): """ Group a set of resources according to their containing module. """ @@ -234,7 +234,7 @@ def migrate(statefile, dryrun=False): ] # Group resources based on the module where they're defined. - modules = group(resources) + modules = group_by_module(resources) # Filter our list of Terraform modules down to anything that lookst like a # project-factory module. We key this off the presence off of From 174fef21d4b9481f69ba4519e6ed6ca6535410c2 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Wed, 12 Dec 2018 17:03:00 -0800 Subject: [PATCH 066/105] Extract function for computing per-module migration commands --- helpers/migrate.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 2b5b3fba..96e36ab9 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -220,6 +220,22 @@ def read_state(statefile): return elements +def state_changes_for_module(module, statefile): + """ + Compute the Terraform state changes (deletions and moves) for a single + project-factory module. + """ + commands = [] + + migration = GSuiteMigration(module) + + for (old, new) in migration.moves(): + argv = ["terraform", "state", "mv", "-state", statefile, old, new] + commands.append(argv) + + return commands + + def migrate(statefile, dryrun=False): """ Migrate the terraform state in `statefile` to match the post-refactor @@ -251,13 +267,11 @@ def migrate(statefile, dryrun=False): # Collect a list of resources for each project factory that need to be # migrated. - to_move = [] + commands = [] for factory in factories: - migration = GSuiteMigration(factory) - to_move += migration.moves() + commands += state_changes_for_module(factory, statefile) - for (old, new) in to_move: - argv = ["terraform", "state", "mv", "-state", statefile, old, new] + for argv in commands: if dryrun: print(" ".join(argv)) else: From 2ecb092f9ade766243e7d15010b2088b7af05eaf Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Wed, 12 Dec 2018 17:58:49 -0800 Subject: [PATCH 067/105] Refactor migrations to better handle multiple destination modules --- helpers/migrate.py | 114 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 25 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 96e36ab9..c2d014c3 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -21,6 +21,89 @@ import shutil import re +MIGRATIONS = [ + { + "resource_type": "random_id", + "name": "random_project_id_suffix", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project", + "name": "project", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project_service", + "name": "project_services", + "module": ".module.project-factory" + }, + { + "resource_type": "google_compute_shared_vpc_service_project", + "name": "shared_vpc_attachment", + "module": ".module.project-factory" + }, + { + "resource_type": "null_resource", + "name": "delete_default_compute_service_account", + "module": ".module.project-factory" + }, + { + "resource_type": "google_service_account", + "name": "default_service_account", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project_iam_member", + "name": "default_service_account_membership", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project_iam_member", + "name": "controlling_group_vpc_membership", + "module": ".module.project-factory" + }, + { + "resource_type": "google_compute_subnetwork_iam_member", + "name": "service_account_role_to_vpc_subnets", + "module": ".module.project-factory" + }, + { + "resource_type": "google_compute_subnetwork_iam_member", + "name": "apis_service_account_role_to_vpc_subnets", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project_usage_export_bucket", + "name": "usage_report_export", + "module": ".module.project-factory" + }, + { + "resource_type": "google_storage_bucket", + "name": "project_bucket", + "module": ".module.project-factory" + }, + { + "resource_type": "google_storage_bucket_iam_member", + "name": "s_account_storage_admin_on_project_bucket", + "module": ".module.project-factory" + }, + { + "resource_type": "google_storage_bucket_iam_member", + "name": "api_s_account_storage_admin_on_project_bucket", + "module": ".module.project-factory" + }, + { + "resource_type": "google_compute_subnetwork_iam_member", + "name": "gke_shared_vpc_subnets", + "module": ".module.project-factory" + }, + { + "resource_type": "google_project_iam_member", + "name": "gke_host_agent", + "module": ".module.project-factory" + }, +] + class GSuiteMigration: """ @@ -28,25 +111,6 @@ class GSuiteMigration: module structure created by the G Suite refactor. """ - CORE_PROJECT_FACTORY_SELECTORS = [ - ("google_organization", "org"), - ("google_compute_default_service_account", "default"), - ("null_data_source", "data_final_group_email"), - ("null_data_source", "data_group_email_format"), - ("google_compute_shared_vpc_service_project", None), - ("google_compute_subnetwork_iam_member", None), - ("google_project", None), - ("google_project_iam_member", None), - ("google_project_service", None), - ("google_project_usage_export_bucket", None), - ("google_service_account", None), - ("google_service_account_iam_member", None), - ("google_storage_bucket", None), - ("google_storage_bucket_iam_member", None), - ("null_resource", None), - ("random_id", None), - ] - def __init__(self, project_factory): self.project_factory = project_factory @@ -57,9 +121,9 @@ def moves(self): """ resources = self.targets() moves = [] - for old in resources: + for (old, module) in resources: new = copy.deepcopy(old) - new.module += ".module.project-factory" + new.module += module pair = (old.path(), new.path()) moves.append(pair) @@ -73,13 +137,13 @@ def targets(self): """ to_move = [] - for selector in self.__class__.CORE_PROJECT_FACTORY_SELECTORS: - resource_type = selector[0] - resource_name = selector[1] + for selector in MIGRATIONS: + resource_type = selector["resource_type"] + resource_name = selector["name"] matching_resources = self.project_factory.get_resources( resource_type, resource_name) - to_move += matching_resources + to_move += [(r, selector["module"]) for r in matching_resources] return to_move From e7bbd27e73e6f9f8017188b3747b0c8a345494be Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 13 Dec 2018 16:20:28 -0800 Subject: [PATCH 068/105] Migrate resources with a count > 1 --- helpers/migrate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index c2d014c3..d63ae3ed 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -170,8 +170,9 @@ def get_resources(self, resource_type=None, resource_name=None): matches_type = (resource_type is None or resource_type == resource.resource_type) + name_pattern = re.compile(r'%s(\[\d+\])?' % resource_name) matches_name = (resource_name is None or - resource_name == resource.name) + name_pattern.match(resource.name)) if matches_type and matches_name: ret.append(resource) From a2ce7ef237c5f110acce9def4c489d6bd216f00f Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 17 Dec 2018 22:17:08 +0000 Subject: [PATCH 069/105] Remove obsolete migration script This script is replaced by #35. --- helpers/migrate.sh | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 helpers/migrate.sh diff --git a/helpers/migrate.sh b/helpers/migrate.sh deleted file mode 100644 index 3c86d3a8..00000000 --- a/helpers/migrate.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -MODULE="module.project-factory" -NEW_MODULE="module.project-factory.module.project-factory" - -terraform state mv $MODULE.random_id.random_project_id_suffix $NEW_MODULE.random_id.random_project_id_suffix -terraform state mv $MODULE.google_project.project $NEW_MODULE.google_project.project -terraform state mv $MODULE.google_project_service.project_services $NEW_MODULE.google_project_service.project_services -terraform state mv $MODULE.google_compute_shared_vpc_service_project.shared_vpc_attachment $MODULE.google_compute_shared_vpc_service_project.shared_vpc_attachment -terraform state mv $MODULE.null_resource.delete_default_compute_service_account $NEW_MODULE.null_resource.delete_default_compute_service_account -terraform state mv $MODULE.google_service_account.default_service_account $NEW_MODULE.google_service_account.default_service_account -terraform state mv $MODULE.google_project_iam_member.default_service_account_membership $NEW_MODULE.google_project_iam_member.default_service_account_membership -terraform state mv $MODULE.google_project_iam_member.gsuite_group_role $NEW_MODULE.google_project_iam_member.gsuite_group_role -terraform state mv $MODULE.google_service_account_iam_member.service_account_grant_to_group $NEW_MODULE.google_service_account_iam_member.service_account_grant_to_group -terraform state mv $MODULE.google_project_iam_member.controlling_group_vpc_membership $NEW_MODULE.$MODULE.google_project_iam_member.controlling_group_vpc_membership -terraform state mv $MODULE.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets -terraform state mv $MODULE.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets -terraform state mv $MODULE.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets -terraform state mv $MODULE.google_project_usage_export_bucket.usage_report_export $NEW_MODULE.google_project_usage_export_bucket.usage_report_export -terraform state mv $MODULE.google_storage_bucket.project_bucket $NEW_MODULE.google_storage_bucket.project_bucket -terraform state mv $MODULE.google_storage_bucket_iam_member.group_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.group_storage_admin_on_project_bucket -terraform state mv $MODULE.google_storage_bucket_iam_member.s_account_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.s_account_storage_admin_on_project_bucket -terraform state mv $MODULE.google_storage_bucket_iam_member.api_s_account_storage_admin_on_project_bucket $NEW_MODULE.google_storage_bucket_iam_member.api_s_account_storage_admin_on_project_bucket -terraform state mv $MODULE.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets $NEW_MODULE.google_compute_subnetwork_iam_member.gke_shared_vpc_subnets -terraform state mv $MODULE.google_project_iam_member.gke_host_agent $NEW_MODULE.google_project_iam_member.gke_host_agent From bd01441633041a6187b1bbc3eb21e9bff8c06a96 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Mon, 17 Dec 2018 14:27:23 -0800 Subject: [PATCH 070/105] Update tests to reflect new migrations, clean up assertions --- test/helpers/test_migrate.py | 109 ++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/test/helpers/test_migrate.py b/test/helpers/test_migrate.py index d1b8fe78..d4f8ef16 100755 --- a/test/helpers/test_migrate.py +++ b/test/helpers/test_migrate.py @@ -27,47 +27,34 @@ import migrate # noqa: E402 TERRAFORM_STATE_LIST = [ + "google_compute_instance.test", + "google_project_iam_member.user-editor", "module.project-factory.google_compute_default_service_account.default", "module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment", + "module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[0]", + "module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[1]", + "module.project-factory.google_compute_subnetwork_iam_member.apis_service_account_role_to_vpc_subnets[2]", + "module.project-factory.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[0]", + "module.project-factory.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[1]", + "module.project-factory.google_compute_subnetwork_iam_member.group_role_to_vpc_subnets[2]", + "module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[0]", + "module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[1]", + "module.project-factory.google_compute_subnetwork_iam_member.service_account_role_to_vpc_subnets[2]", + "module.project-factory.google_organization.org", "module.project-factory.google_project.project", "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0]", "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1]", "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2]", - "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3]", - "module.project-factory.google_project_iam_member.default_service_account_membership", - "module.project-factory.google_project_iam_member.gke_host_agent", "module.project-factory.google_project_iam_member.gsuite_group_role", - "module.project-factory.google_project_service.project_services[0]", - "module.project-factory.google_project_service.project_services[1]", - "module.project-factory.google_project_usage_export_bucket.usage_report_export", + "module.project-factory.google_project_service.project_services", "module.project-factory.google_service_account.default_service_account", "module.project-factory.google_service_account_iam_member.service_account_grant_to_group", + "module.project-factory.gsuite_group.group", + "module.project-factory.gsuite_group_member.api_s_account_api_sa_group_member", "module.project-factory.null_data_source.data_final_group_email", "module.project-factory.null_data_source.data_given_group_email", "module.project-factory.null_data_source.data_group_email_format", "module.project-factory.null_resource.delete_default_compute_service_account", - "module.project-factory.random_id.random_project_id_suffix" -] - -TERRAFORM_MIGRATED_RESOURCES = [ - "module.project-factory.google_compute_default_service_account.default", - "module.project-factory.google_compute_shared_vpc_service_project.shared_vpc_attachment", - "module.project-factory.google_project.project", - "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[0]", - "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[1]", - "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[2]", - "module.project-factory.google_project_iam_member.controlling_group_vpc_membership[3]", - "module.project-factory.google_project_iam_member.default_service_account_membership", - "module.project-factory.google_project_iam_member.gke_host_agent", - "module.project-factory.google_project_iam_member.gsuite_group_role", - "module.project-factory.google_project_service.project_services[0]", - "module.project-factory.google_project_service.project_services[1]", - "module.project-factory.google_project_usage_export_bucket.usage_report_export", - "module.project-factory.google_service_account.default_service_account", - "module.project-factory.google_service_account_iam_member.service_account_grant_to_group", - "module.project-factory.null_data_source.data_final_group_email", - "module.project-factory.null_data_source.data_group_email_format", - "module.project-factory.null_resource.delete_default_compute_service_account", "module.project-factory.random_id.random_project_id_suffix", ] @@ -76,22 +63,36 @@ class TestGSuiteMigration(unittest.TestCase): def setUp(self): self.resources = [ migrate.TerraformResource.from_path(path) - for path in TERRAFORM_MIGRATED_RESOURCES + for path in TERRAFORM_STATE_LIST ] - module = migrate.TerraformModule( + self.module = migrate.TerraformModule( 'module.project-factory', self.resources) - self.migration = migrate.GSuiteMigration(module) + self.migration = migrate.GSuiteMigration(self.module) def test_moves(self): - moves = self.migration.moves() + resources_to_move = [] + + for selector in migrate.MIGRATIONS: + resources_to_move += self.module.get_resources( + resource_type=selector["resource_type"], + resource_name=selector["name"]) + + computed_moves = self.migration.moves() - for old in self.resources: + for old in resources_to_move: new = copy.deepcopy(old) new.module += ".module.project-factory" - move = (old.path(), new.path()) - assert move in moves + expected_move = (old.path(), new.path()) + self.assertIn(expected_move, computed_moves) + + def test_no_moves_outside_of_module(self): + computed_moves = self.migration.moves() + + old_resources = [move[0] for move in computed_moves] + self.assertFalse("google_compute_instance.test" in old_resources) + self.assertFalse("google_project_iam_member.user-editor" in old_resources) class TestTerraformModule(unittest.TestCase): @@ -106,38 +107,38 @@ def setUp(self): self.resources) def test_has_resource(self): - assert self.module.has_resource('google_project', 'project') - assert self.module.has_resource(None, 'project') - assert self.module.has_resource('google_project', None) - assert self.module.has_resource(None, None) + self.assertTrue(self.module.has_resource('google_project', 'project')) + self.assertTrue(self.module.has_resource(None, 'project')) + self.assertTrue(self.module.has_resource('google_project', None)) + self.assertTrue(self.module.has_resource(None, None)) def test_has_resource_empty(self): - assert self.module.has_resource('google_compute_instance', None) is False + self.assertFalse(self.module.has_resource('google_cloudiot_registry', None)) def test_get_resources(self): expected = [resource for resource in self.resources if resource.resource_type == 'google_project_iam_member'] actual = self.module.get_resources('google_project_iam_member', None) - assert actual == expected + self.assertEqual(actual, expected) def test_get_resources_empty(self): - actual = self.module.get_resources('google_compute_instance', None) - assert actual == [] + actual = self.module.get_resources('google_cloudiot_registry', None) + self.assertTrue(len(actual) == 0) class TestTerraformResource(unittest.TestCase): def test_root_resource_from_path(self): resource = migrate.TerraformResource.from_path("google_project.project") - assert resource.module == '' - assert resource.resource_type == 'google_project' - assert resource.name == 'project' + self.assertEqual(resource.module, '') + self.assertEqual(resource.resource_type, 'google_project') + self.assertEqual(resource.name, 'project') def test_module_resource_from_path(self): resource = migrate.TerraformResource.from_path("module.project-factory.google_project.project") - assert resource.module == 'module.project-factory' - assert resource.resource_type == 'google_project' - assert resource.name == 'project' + self.assertEqual(resource.module, 'module.project-factory') + self.assertEqual(resource.resource_type, 'google_project') + self.assertEqual(resource.name, 'project') def test_invalid_resource_from_path(self): self.assertRaises( @@ -146,19 +147,19 @@ def test_invalid_resource_from_path(self): def test_resource_init(self): resource = migrate.TerraformResource('', 'google_project', 'project') - assert resource.module == '' - assert resource.resource_type == 'google_project' - assert resource.name == 'project' + self.assertEqual(resource.module, '') + self.assertEqual(resource.resource_type, 'google_project') + self.assertEqual(resource.name, 'project') def test_resource_path_no_module(self): resource = migrate.TerraformResource('', 'google_project', 'project') - assert resource.path() == 'google_project.project' + self.assertEqual(resource.path(), 'google_project.project') def test_resource_path_with_module(self): resource = migrate.TerraformResource('module.project-factory', 'google_project', 'project') expected = 'module.project-factory.google_project.project' actual = resource.path() - assert expected == actual + self.assertEqual(expected, actual) if __name__ == "__main__": From 6bff8eea6ed940dfb5fefeb112d92681ee2192ae Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 4 Jan 2019 11:05:13 -0500 Subject: [PATCH 071/105] Revert order of Read Me sample module arguments --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ed5418fe..6fa6fb77 100644 --- a/README.md +++ b/README.md @@ -16,21 +16,20 @@ module "project-factory" { source = "terraform-google-modules/project-factory/google" version = "v0.3.0" - billing_account = "ABCDEF-ABCDEF-ABCDEF" - credentials_path = "${local.credentials_file_path}" - name = "pf-test-1" - org_id = "1234567890" - random_project_id = "true" - shared_vpc = "shared_vpc_host_name" + name = "pf-test-1" + random_project_id = "true" + org_id = "1234567890" + usage_bucket_name = "pf-test-1-usage-report-bucket" + usage_bucket_prefix = "pf/test/1/integration" + billing_account = "ABCDEF-ABCDEF-ABCDEF" + shared_vpc = "shared_vpc_host_name" + credentials_path = "${local.credentials_file_path}" shared_vpc_subnets = [ "projects/base-project-196723/regions/us-east1/subnetworks/default", "projects/base-project-196723/regions/us-central1/subnetworks/default", "projects/base-project-196723/regions/us-central1/subnetworks/subnet-1", ] - - usage_bucket_name = "pf-test-1-usage-report-bucket" - usage_bucket_prefix = "pf/test/1/integration" } ``` From af21297c281bcb4207253d8717f2dec265cabf68 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 4 Jan 2019 11:52:51 -0500 Subject: [PATCH 072/105] Highlight how to include G Sutie in Read Me --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6fa6fb77..76ee6c3c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ This module allows you to create opinionated Google Cloud Platform projects. It creates projects and configures aspects like Shared VPC connectivity, IAM access, Service Accounts, and API enablement to follow best practices. +To include G Suite integration, use the +[gsuite_enabled module][gsuite-enabled-module]. + ## Usage There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: @@ -72,9 +75,6 @@ The roles granted are specifically: - `compute.networkUser` on host project or specified subnets - `storage.admin` on `bucket_name` GCS bucket -To include G Suite integration, use the -[gsuite_enabled module][gsuite-enabled-module]. - [^]: (autogen_docs_start) ## Inputs From 5aebb40ad61fe8ec38b5e7a2656d1afc7f617415 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 4 Jan 2019 12:52:05 -0500 Subject: [PATCH 073/105] Add group_name, group_role back to root module --- README.md | 2 ++ variables.tf | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/README.md b/README.md index 76ee6c3c..31e58f97 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ The roles granted are specifically: | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | | domain | The domain name (optional if `org_id` is passed) | string | `` | no | | folder\_id | The ID of a folder to host this project | string | `` | no | +| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `` | no | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | | name | The name for the project | string | - | yes | diff --git a/variables.tf b/variables.tf index a8c994dd..6db45303 100755 --- a/variables.tf +++ b/variables.tf @@ -47,6 +47,16 @@ variable "folder_id" { default = "" } +variable "group_name" { + description = "A group to control the project by being assigned group_role (defaults to project editor)" + default = "" +} + +variable "group_role" { + description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" + default = "roles/editor" +} + variable "sa_role" { description = "A role to give the default Service Account for the project (defaults to none)" default = "" From c5f2043438e9179587189f031dc05a66d8af95d6 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 15:18:33 +0000 Subject: [PATCH 074/105] Manage all non G Suite resources in core This commit moves all `google` resources under the core submodule and reintroduces support for managing an existing G Suite group to the root module. --- main.tf | 2 + modules/core_project_factory/main.tf | 67 ++++++++++++++--- modules/core_project_factory/outputs.tf | 7 +- modules/core_project_factory/variables.tf | 8 +++ modules/gsuite_enabled/main.tf | 71 +++---------------- modules/gsuite_enabled/outputs.tf | 2 +- modules/gsuite_group/README.md | 20 ++++++ .../main.tf | 1 + .../outputs.tf | 7 +- .../variables.tf | 4 ++ outputs.tf | 5 ++ 11 files changed, 120 insertions(+), 74 deletions(-) create mode 100644 modules/gsuite_group/README.md rename modules/{google_organization => gsuite_group}/main.tf (94%) rename modules/{google_organization => gsuite_group}/outputs.tf (80%) rename modules/{google_organization => gsuite_group}/variables.tf (92%) diff --git a/main.tf b/main.tf index 5238c4fe..148086aa 100755 --- a/main.tf +++ b/main.tf @@ -17,6 +17,8 @@ module "project-factory" { source = "modules/core_project_factory" + group_name = "${var.group_name}" + group_role = "${var.group_role}" lien = "${var.lien}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 29734a6c..0b4d1d21 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -25,6 +25,7 @@ resource "random_id" "random_project_id_suffix" { Locals configuration *****************************************/ locals { + group_id = "${var.group_name != "" ? format("group:%s", module.gsuite_group.email) : ""}" project_id = "${google_project.project.project_id}" project_number = "${google_project.project.number}" project_org_id = "${var.folder_id != "" ? "" : var.org_id}" @@ -40,7 +41,7 @@ locals { create_bucket = "${var.bucket_project != "" ? "true" : "false"}" app_engine_enabled = "${length(keys(var.app_engine)) > 0 ? true : false}" - shared_vpc_users = "${compact(list(local.s_account_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" + shared_vpc_users = "${compact(list(local.group_id, local.s_account_fmt, local.api_s_account_fmt, local.gke_s_account_fmt))}" # Workaround for https://github.com/hashicorp/terraform/issues/10857 shared_vpc_users_length = "${local.gke_shared_vpc_enabled ? 4 : 3}" @@ -52,12 +53,13 @@ locals { } /***************************************** - Organization info retrieval + G Suite group information retrieval *****************************************/ -module "google_organization" { - source = "../google_organization" +module "gsuite_group" { + source = "../gsuite_group" domain = "${var.domain}" + name = "${var.group_name}" org_id = "${var.org_id}" } @@ -182,10 +184,35 @@ resource "google_project_iam_member" "default_service_account_membership" { member = "${local.s_account_fmt}" } -/************************************************************************************* - compute.networkUser role granted to APIs Service account, Project Service Account, and GKE Service Account on shared - VPC - *************************************************************************************/ +/****************************************** + Gsuite Group Role Configuration + *****************************************/ +resource "google_project_iam_member" "gsuite_group_role" { + count = "${local.group_id != "" ? 1 : 0}" + + member = "${local.group_id}" + project = "${local.project_id}" + role = "${var.group_role}" +} + +/****************************************** + Granting serviceAccountUser to group + *****************************************/ +resource "google_service_account_iam_member" "service_account_grant_to_group" { + count = "${local.group_id != "" ? 1 : 0}" + + member = "${local.group_id}" + role = "roles/iam.serviceAccountUser" + + service_account_id = "projects/${local.project_id}/serviceAccounts/${ + google_service_account.default_service_account.email + }" +} + +/****************************************************************************************************************** + compute.networkUser role granted to G Suite group, APIs Service account, Project Service Account, and GKE Service + Account on shared VPC + *****************************************************************************************************************/ resource "google_project_iam_member" "controlling_group_vpc_membership" { count = "${(var.shared_vpc != "" && (length(compact(var.shared_vpc_subnets)) > 0)) ? local.shared_vpc_users_length : 0}" @@ -209,6 +236,19 @@ resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_sub member = "${local.s_account_fmt}" } +/************************************************************************************* + compute.networkUser role granted to GSuite group on vpc subnets + *************************************************************************************/ +resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && local.group_id != "" ? length(var.shared_vpc_subnets) : 0 }" + + member = "${local.group_id}" + project = "${var.shared_vpc}" + region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" + role = "roles/compute.networkUser" + subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" +} + /************************************************************************************* compute.networkUser role granted to APIs Service Account on vpc subnets *************************************************************************************/ @@ -247,6 +287,17 @@ resource "google_storage_bucket" "project_bucket" { project = "${var.bucket_project}" } +/*********************************************** + Project's bucket storage.admin granting to group + ***********************************************/ +resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { + count = "${local.create_bucket && local.group_id != "" ? 1 : 0}" + + bucket = "${google_storage_bucket.project_bucket.name}" + member = "${local.group_id}" + role = "roles/storage.admin" +} + /*********************************************** Project's bucket storage.admin granting to default compute service account ***********************************************/ diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 9d20246b..f6037ea2 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -23,10 +23,15 @@ output "project_number" { } output "domain" { - value = "${module.google_organization.domain}" + value = "${module.gsuite_group.domain}" description = "The organization's domain" } +output "group_email" { + value = "${local.gsuite_group_id ? module.gsuite_group.email : ""}" + description = "The email of the created GSuite group with group_name" +} + output "service_account_id" { value = "${google_service_account.default_service_account.account_id}" description = "The id of the default service account" diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 46b30199..341c6113 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -14,6 +14,14 @@ * limitations under the License. */ +variable "group_name" { + description = "A group to control the project by being assigned group_role (defaults to project editor)" +} + +variable "group_role" { + description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" +} + variable "lien" { description = "Add a lien on the project to prevent accidental deletion" default = "false" diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index ed8ec2d5..c864af72 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -15,9 +15,7 @@ */ locals { - group_email = "${format("%s@%s", local.group_name, module.google_organization.domain)}" - group_id = "${format("group:%s", local.group_email)}" - group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" + group_name = "${var.group_name != "" ? var.group_name : format("%s-editors", var.name)}" } /*********************************************** @@ -34,12 +32,13 @@ resource "gsuite_group_member" "service_account_sa_group_member" { } /***************************************** - Organization info retrieval + G Suite group information retrieval *****************************************/ -module "google_organization" { - source = "../google_organization" +module "gsuite_group" { + source = "../gsuite_group" domain = "${var.domain}" + name = "${local.group_name}" org_id = "${var.org_id}" } @@ -50,7 +49,7 @@ resource "gsuite_group" "group" { count = "${var.create_group ? 1 : 0}" description = "${var.name} project group" - email = "${local.group_email}" + email = "${module.gsuite_group.email}" name = "${local.group_name}" } @@ -68,6 +67,8 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { module "project-factory" { source = "../core_project_factory/" + group_name = "${local.group_name}" + group_role = "${var.group_role}" lien = "${var.lien}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" @@ -88,59 +89,3 @@ module "project-factory" { auto_create_network = "${var.auto_create_network}" app_engine = "${var.app_engine}" } - -/****************************************** - Gsuite Group Role Configuration - *****************************************/ -resource "google_project_iam_member" "gsuite_group_role" { - member = "${local.group_id}" - project = "${module.project-factory.project_id}" - role = "${var.group_role}" -} - -/****************************************** - Granting serviceAccountUser to group - *****************************************/ -resource "google_service_account_iam_member" "service_account_grant_to_group" { - member = "${local.group_id}" - role = "roles/iam.serviceAccountUser" - - service_account_id = "projects/${module.project-factory.project_id}/serviceAccounts/${ - module.project-factory.service_account_email - }" -} - -/************************************************************************************* - compute.networkUser role granted to GSuite group on shared VPC - *************************************************************************************/ -resource "google_project_iam_member" "controlling_group_vpc_membership" { - count = "${(var.shared_vpc != "" && (length(compact(var.shared_vpc_subnets)) > 0)) ? 1 : 0}" - - member = "${local.group_id}" - project = "${var.shared_vpc}" - role = "roles/compute.networkUser" -} - -/************************************************************************************* - compute.networkUser role granted to GSuite group on vpc subnets - *************************************************************************************/ -resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 ? length(var.shared_vpc_subnets) : 0 }" - - member = "${local.group_id}" - project = "${var.shared_vpc}" - region = "${element(split("/", var.shared_vpc_subnets[count.index]), 3)}" - role = "roles/compute.networkUser" - subnetwork = "${element(split("/", var.shared_vpc_subnets[count.index]), 5)}" -} - -/*********************************************** - Project's bucket storage.admin granting to group - ***********************************************/ -resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { - count = "${var.bucket_project != "" ? 1 : 0}" - - bucket = "${module.project-factory.project_bucket_name}" - member = "${local.group_id}" - role = "roles/storage.admin" -} diff --git a/modules/gsuite_enabled/outputs.tf b/modules/gsuite_enabled/outputs.tf index 40a4a2db..ec2de95c 100644 --- a/modules/gsuite_enabled/outputs.tf +++ b/modules/gsuite_enabled/outputs.tf @@ -28,7 +28,7 @@ output "domain" { } output "group_email" { - value = "${local.group_email}" + value = "${module.gsuite_group.email}" description = "The email of the created GSuite group with group_name" } diff --git a/modules/gsuite_group/README.md b/modules/gsuite_group/README.md new file mode 100644 index 00000000..fb451586 --- /dev/null +++ b/modules/gsuite_group/README.md @@ -0,0 +1,20 @@ +# gsuite_group + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| name | The name of the group. | string | n/a | yes | +| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| domain | The domain of the group's organization. | +| email | The email address of the group. | + +[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/google_organization/main.tf b/modules/gsuite_group/main.tf similarity index 94% rename from modules/google_organization/main.tf rename to modules/gsuite_group/main.tf index 6165a661..ea679d84 100644 --- a/modules/google_organization/main.tf +++ b/modules/gsuite_group/main.tf @@ -16,6 +16,7 @@ locals { domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" + email = "${format("%s@%s", var.name, local.domain)}" } /***************************************** diff --git a/modules/google_organization/outputs.tf b/modules/gsuite_group/outputs.tf similarity index 80% rename from modules/google_organization/outputs.tf rename to modules/gsuite_group/outputs.tf index 09e6ffbb..fefc873a 100644 --- a/modules/google_organization/outputs.tf +++ b/modules/gsuite_group/outputs.tf @@ -16,5 +16,10 @@ output "domain" { value = "${local.domain}" - description = "The organization's domain" + description = "The domain of the group's organization." +} + +output "email" { + description = "The email address of the group." + value = "${local.email}" } diff --git a/modules/google_organization/variables.tf b/modules/gsuite_group/variables.tf similarity index 92% rename from modules/google_organization/variables.tf rename to modules/gsuite_group/variables.tf index 6dac45d1..a4c8cdb3 100644 --- a/modules/google_organization/variables.tf +++ b/modules/gsuite_group/variables.tf @@ -19,6 +19,10 @@ variable "domain" { default = "" } +variable "name" { + description = "The name of the group." +} + variable "org_id" { description = "The organization id (optional if `domain` is passed)" default = "" diff --git a/outputs.tf b/outputs.tf index 0341ca62..11d0904f 100755 --- a/outputs.tf +++ b/outputs.tf @@ -27,6 +27,11 @@ output "domain" { description = "The organization's domain" } +output "group_email" { + value = "${module.project-factory.group_email}" + description = "The email of the created GSuite group with group_name" +} + output "service_account_id" { value = "${module.project-factory.service_account_id}" description = "The id of the default service account" From 1f37c10aaff97545a0a8df2292d3f6301ece3927 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 15:37:57 +0000 Subject: [PATCH 075/105] Generate documentation --- README.md | 40 +++++++++--------- examples/app_engine/README.md | 6 +-- examples/gke_shared_vpc/README.md | 8 ++-- examples/group_project/README.md | 12 +++--- examples/project-hierarchy/README.md | 8 ++-- examples/simple_project/README.md | 6 +-- modules/core_project_factory/README.md | 37 ++++++++-------- modules/google_organization/README.md | 18 -------- modules/gsuite_enabled/README.md | 44 +++++++++---------- test/fixtures/full/README.md | 58 +++++++++++++------------- test/fixtures/minimal/README.md | 20 ++++----- 11 files changed, 122 insertions(+), 135 deletions(-) delete mode 100644 modules/google_organization/README.md diff --git a/README.md b/README.md index 31e58f97..fd2e2594 100644 --- a/README.md +++ b/README.md @@ -83,25 +83,25 @@ The roles granted are specifically: |------|-------------|:----:|:-----:|:-----:| | activate\_apis | The list of apis to activate within the project | list | `` | no | | app\_engine | A map for app engine configuration | map | `` | no | -| auto\_create\_network | Create the default network | string | `false` | no | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | -| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `` | no | -| folder\_id | The ID of a folder to host this project | string | `` | no | -| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `` | no | -| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | +| auto\_create\_network | Create the default network | string | `"false"` | no | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | +| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| folder\_id | The ID of a folder to host this project | string | `""` | no | +| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `""` | no | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `"roles/editor"` | no | | labels | Map of labels for project | map | `` | no | -| lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | -| name | The name for the project | string | - | yes | -| org\_id | The organization id (optional if `domain` is passed) | string | `` | no | -| random\_project\_id | Enables project random id generation | string | `false` | no | -| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | -| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | +| lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | +| name | The name for the project | string | n/a | yes | +| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | +| random\_project\_id | Enables project random id generation | string | `"false"` | no | +| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `""` | no | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | -| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `` | no | -| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `` | no | +| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `""` | no | +| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `""` | no | ## Outputs @@ -109,10 +109,11 @@ The roles granted are specifically: |------|-------------| | app\_engine\_enabled | Whether app engine is enabled | | domain | The organization's domain | +| group\_email | The email of the created GSuite group with group_name | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | -| project\_id | - | -| project\_number | - | +| project\_id | | +| project\_number | | | service\_account\_display\_name | The display name of the default service account | | service\_account\_email | The email of the default service account | | service\_account\_id | The id of the default service account | @@ -122,6 +123,7 @@ The roles granted are specifically: [^]: (autogen_docs_end) ## File structure + The project has the following folders and files: - /: root folder diff --git a/examples/app_engine/README.md b/examples/app_engine/README.md index c54dc293..e18cf63e 100755 --- a/examples/app_engine/README.md +++ b/examples/app_engine/README.md @@ -18,9 +18,9 @@ Expected variables: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| admin\_email | Admin user email on Gsuite | string | - | yes | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| organization\_id | The organization id for the associated services | string | - | yes | +| admin\_email | Admin user email on Gsuite | string | n/a | yes | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| organization\_id | The organization id for the associated services | string | n/a | yes | ## Outputs diff --git a/examples/gke_shared_vpc/README.md b/examples/gke_shared_vpc/README.md index 0309ca6f..40540ef8 100644 --- a/examples/gke_shared_vpc/README.md +++ b/examples/gke_shared_vpc/README.md @@ -29,10 +29,10 @@ More information about GKE with Shared VPC can be found here: https://cloud.goog | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| billing\_account | billing account | string | - | yes | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| org\_id | organization id | string | - | yes | -| shared\_vpc | The ID of the host project which hosts the shared VPC | string | - | yes | +| billing\_account | billing account | string | n/a | yes | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| org\_id | organization id | string | n/a | yes | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | n/a | yes | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$PROJECT_ID/regions/$REGION/subnetworks/$SUBNET_ID) | list | `` | no | [^]: (autogen_docs_end) \ No newline at end of file diff --git a/examples/group_project/README.md b/examples/group_project/README.md index 9a096b51..e48e58ca 100644 --- a/examples/group_project/README.md +++ b/examples/group_project/README.md @@ -20,12 +20,12 @@ Expected variables: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| admin\_email | Admin user email on Gsuite | string | - | yes | -| api\_sa\_group | An existing GSuite group email to place the Google APIs Service Account for the project in | string | - | yes | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| credentials\_file\_path | Service account json auth path | string | - | yes | -| organization\_id | The organization id for the associated services | string | - | yes | -| project\_group\_name | The name of a GSuite group to create for controlling the project | string | - | yes | +| admin\_email | Admin user email on Gsuite | string | n/a | yes | +| api\_sa\_group | An existing GSuite group email to place the Google APIs Service Account for the project in | string | n/a | yes | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| credentials\_file\_path | Service account json auth path | string | n/a | yes | +| organization\_id | The organization id for the associated services | string | n/a | yes | +| project\_group\_name | The name of a GSuite group to create for controlling the project | string | n/a | yes | ## Outputs diff --git a/examples/project-hierarchy/README.md b/examples/project-hierarchy/README.md index 1d18b2c1..5a296965 100644 --- a/examples/project-hierarchy/README.md +++ b/examples/project-hierarchy/README.md @@ -26,10 +26,10 @@ Expected variables: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| admin\_email | Admin user email on Gsuite | string | - | yes | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| organization\_id | The organization id for the associated services | string | - | yes | +| admin\_email | Admin user email on Gsuite | string | n/a | yes | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| organization\_id | The organization id for the associated services | string | n/a | yes | ## Outputs diff --git a/examples/simple_project/README.md b/examples/simple_project/README.md index 32cbaec4..d6b66f34 100755 --- a/examples/simple_project/README.md +++ b/examples/simple_project/README.md @@ -14,9 +14,9 @@ Expected variables: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| organization\_id | The organization id for the associated services | string | - | yes | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| organization\_id | The organization id for the associated services | string | n/a | yes | ## Outputs diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 61adedb5..75a2695e 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -8,23 +8,25 @@ |------|-------------|:----:|:-----:|:-----:| | activate\_apis | The list of apis to activate within the project | list | `` | no | | app\_engine | A map for app engine configuration | map | `` | no | -| auto\_create\_network | Create the default network | string | `false` | no | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | -| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `` | no | -| folder\_id | The ID of a folder to host this project | string | `` | no | +| auto\_create\_network | Create the default network | string | `"false"` | no | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | +| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| folder\_id | The ID of a folder to host this project | string | `""` | no | +| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | n/a | yes | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | n/a | yes | | labels | Map of labels for project | map | `` | no | -| lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | -| name | The name for the project | string | - | yes | -| org\_id | The organization id (optional if `domain` is passed) | string | `` | no | -| random\_project\_id | Enables project random id generation | string | `false` | no | -| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | -| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | +| lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | +| name | The name for the project | string | n/a | yes | +| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | +| random\_project\_id | Enables project random id generation | string | `"false"` | no | +| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `""` | no | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | -| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `` | no | -| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `` | no | +| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `""` | no | +| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `""` | no | ## Outputs @@ -34,11 +36,12 @@ | api\_s\_account\_fmt | API service account email formatted for terraform use | | app\_engine\_enabled | Whether app engine is enabled | | domain | The organization's domain | +| group\_email | The email of the created GSuite group with group_name | | project\_bucket\_name | The name of the projec's bucket | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | -| project\_id | - | -| project\_number | - | +| project\_id | | +| project\_number | | | service\_account\_display\_name | The display name of the default service account | | service\_account\_email | The email of the default service account | | service\_account\_id | The id of the default service account | diff --git a/modules/google_organization/README.md b/modules/google_organization/README.md deleted file mode 100644 index 7ecd548c..00000000 --- a/modules/google_organization/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# google_organization - -[^]: (autogen_docs_start) - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| domain | The domain name (optional if `org_id` is passed) | string | `` | no | -| org\_id | The organization id (optional if `domain` is passed) | string | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| domain | The organization's domain | - -[^]: (autogen_docs_end) \ No newline at end of file diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index c721e40f..499d3128 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -69,29 +69,29 @@ The roles granted are specifically: | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | activate\_apis | The list of apis to activate within the project | list | `` | no | -| api\_sa\_group | A GSuite group to place the Google APIs Service Account for the project in | string | `` | no | +| api\_sa\_group | A GSuite group to place the Google APIs Service Account for the project in | string | `""` | no | | app\_engine | A map for app engine configuration | map | `` | no | -| auto\_create\_network | Create the default network | string | `false` | no | -| billing\_account | The ID of the billing account to associate this project with | string | - | yes | -| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `` | no | -| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `` | no | -| create\_group | Whether to create the group or not | string | `false` | no | -| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | - | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `` | no | -| folder\_id | The ID of a folder to host this project | string | `` | no | -| group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `` | no | -| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `roles/editor` | no | +| auto\_create\_network | Create the default network | string | `"false"` | no | +| billing\_account | The ID of the billing account to associate this project with | string | n/a | yes | +| bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | +| bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | +| create\_group | Whether to create the group or not | string | `"false"` | no | +| credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| folder\_id | The ID of a folder to host this project | string | `""` | no | +| group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `""` | no | +| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `"roles/editor"` | no | | labels | Map of labels for project | map | `` | no | -| lien | Add a lien on the project to prevent accidental deletion | string | `false` | no | -| name | The name for the project | string | - | yes | -| org\_id | The organization id for the associated services | string | `` | no | -| random\_project\_id | Enables project random id generation | string | `false` | no | -| sa\_group | A GSuite group to place the default Service Account for the project in | string | `` | no | -| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `` | no | -| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `` | no | +| lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | +| name | The name for the project | string | n/a | yes | +| org\_id | The organization id for the associated services | string | `""` | no | +| random\_project\_id | Enables project random id generation | string | `"false"` | no | +| sa\_group | A GSuite group to place the default Service Account for the project in | string | `""` | no | +| sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | +| shared\_vpc | The ID of the host project which hosts the shared VPC | string | `""` | no | | shared\_vpc\_subnets | List of subnets fully qualified subnet IDs (ie. projects/$project_id/regions/$region/subnetworks/$subnet_id) | list | `` | no | -| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `` | no | -| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `` | no | +| usage\_bucket\_name | Name of a GCS bucket to store GCE usage reports in (optional) | string | `""` | no | +| usage\_bucket\_prefix | Prefix in the GCS bucket to store GCE usage reports in (optional) | string | `""` | no | ## Outputs @@ -102,8 +102,8 @@ The roles granted are specifically: | group\_email | The email of the created GSuite group with group_name | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | -| project\_id | - | -| project\_number | - | +| project\_id | | +| project\_number | | | service\_account\_display\_name | The display name of the default service account | | service\_account\_email | The email of the default service account | | service\_account\_id | The id of the default service account | diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md index 3c5eedb9..b9c27f55 100644 --- a/test/fixtures/full/README.md +++ b/test/fixtures/full/README.md @@ -6,40 +6,40 @@ | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| billing\_account | - | string | - | yes | -| create\_group | - | string | `false` | no | -| credentials\_path | - | string | - | yes | -| domain | - | string | - | yes | -| folder\_id | - | string | `` | no | -| group\_name | - | string | `` | no | -| group\_role | - | string | `roles/viewer` | no | -| gsuite\_admin\_account | - | string | - | yes | -| name | - | string | `pf-test-int-full` | no | -| org\_id | - | string | - | yes | -| region | - | string | `us-east4` | no | -| sa\_group | - | string | `` | no | -| sa\_role | - | string | `roles/editor` | no | -| shared\_vpc | - | string | `` | no | -| usage\_bucket\_name | - | string | `` | no | -| usage\_bucket\_prefix | - | string | `` | no | +| billing\_account | | string | n/a | yes | +| create\_group | | string | `"false"` | no | +| credentials\_path | | string | n/a | yes | +| domain | | string | n/a | yes | +| folder\_id | | string | `""` | no | +| group\_name | | string | `""` | no | +| group\_role | | string | `"roles/viewer"` | no | +| gsuite\_admin\_account | | string | n/a | yes | +| name | | string | `"pf-test-int-full"` | no | +| org\_id | | string | n/a | yes | +| region | | string | `"us-east4"` | no | +| sa\_group | | string | `""` | no | +| sa\_role | | string | `"roles/editor"` | no | +| shared\_vpc | | string | `""` | no | +| usage\_bucket\_name | | string | `""` | no | +| usage\_bucket\_prefix | | string | `""` | no | ## Outputs | Name | Description | |------|-------------| | credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | -| domain | - | -| extra\_service\_account\_email | - | -| group\_email | - | -| group\_role | - | -| gsuite\_admin\_account | - | -| project\_id | - | -| project\_number | - | -| region | - | -| sa\_role | - | -| service\_account\_email | - | -| shared\_vpc | - | -| usage\_bucket\_name | - | -| usage\_bucket\_prefix | - | +| domain | | +| extra\_service\_account\_email | | +| group\_email | | +| group\_role | | +| gsuite\_admin\_account | | +| project\_id | | +| project\_number | | +| region | | +| sa\_role | | +| service\_account\_email | | +| shared\_vpc | | +| usage\_bucket\_name | | +| usage\_bucket\_prefix | | [^]: (autogen_docs_end) \ No newline at end of file diff --git a/test/fixtures/minimal/README.md b/test/fixtures/minimal/README.md index 52a51c6e..159d992f 100644 --- a/test/fixtures/minimal/README.md +++ b/test/fixtures/minimal/README.md @@ -6,21 +6,21 @@ | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| billing\_account | - | string | - | yes | -| credentials\_path | - | string | - | yes | -| domain | - | string | `` | no | -| folder\_id | - | string | `` | no | -| name | - | string | `pf-test-int-minimal` | no | -| org\_id | - | string | - | yes | +| billing\_account | | string | n/a | yes | +| credentials\_path | | string | n/a | yes | +| domain | | string | `""` | no | +| folder\_id | | string | `""` | no | +| name | | string | `"pf-test-int-minimal"` | no | +| org\_id | | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| | credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | -| domain | - | -| project\_id | - | -| project\_number | - | -| service\_account\_email | - | +| domain | | +| project\_id | | +| project\_number | | +| service\_account\_email | | [^]: (autogen_docs_end) \ No newline at end of file From 7e64a82a85fe9e0cd8bbeee16e6c3acbfcf65ad2 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 16:09:40 +0000 Subject: [PATCH 076/105] Restore G Suite group documentation in root module --- README.md | 19 ++++++++++++------- modules/gsuite_enabled/README.md | 9 --------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index fd2e2594..635ebfcb 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ The Project Factory module will take the following actions: 1. If a shared VPC is specified, attach the new project to the `shared_vpc`. - It will also give the following users network access on the specified - subnets: + It will also give the following users network access on the specified subnets: - - The project's new default service account (see step 4) - - The Google API service account for the project + - The project's new default service account (see step 4) + - The Google API service account for the project + - The project controlling group specified in `group_name` 1. Delete the default compute service account. 1. Create a new default service account for the project. @@ -62,15 +62,20 @@ The Project Factory module will take the following actions: (`target_usage_bucket`), if provided. 1. If specified, create the GCS bucket `bucket_name` and give the following accounts Storage Admin on it: - 1. The controlling group (`group_name`) - 1. The new default compute service account created for the project. - 1. The Google APIs service account for the project. + 1. The controlling group (`group_name`). + 1. The new default compute service account created for the project. + 1. The Google APIs service account for the project. The roles granted are specifically: - New Default Service Account - `compute.networkUser` on host project or specified subnets - `storage.admin` on `bucket_name` GCS bucket +- `group_name` is the controlling group + - `compute.networkUser` on host project or specific subnets + - Specified `group_role` on project + - `iam.serviceAccountUser` on the default Service Account + - `storage.admin` on `bucket_name` GCS bucket - Google APIs Service Account - `compute.networkUser` on host project or specified subnets - `storage.admin` on `bucket_name` GCS bucket diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index 499d3128..b527dae4 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -41,12 +41,8 @@ addition to those of the root module: 1. Create a new Google group for the project using `group_name` if `create_group` is `"true"`. -1. Give the group access to the project with the `group_role`. -1. Give the project controlling group specified in `group_name` network - access on the specified subnets if `shared_vpc` is specified. 1. Add the new default service account for the project to the `sa_group` in Google Groups, if specified. -1. Give the group Storage Admin on `bucket_name`, if specified. 1. Add the Google APIs service account to the `api_sa_group`, if specified. @@ -54,11 +50,6 @@ The roles granted are specifically: - New Default Service Account - MEMBER of the specified `sa_group` -- `group_name` is the new controlling group - - `compute.networkUser` on host project or specific subnets - - Specified `group_role` on project - - `iam.serviceAccountUser` on the default Service Account - - `storage.admin` on `bucket_name` GCS bucket - Google APIs Service Account - MEMBER of the specified `api_sa_group` From 7566ad891825f06505193f716ae187fbecfb7e39 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 16:19:17 +0000 Subject: [PATCH 077/105] Reimplement arguments check --- modules/core_project_factory/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 0b4d1d21..c7c80e75 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -31,6 +31,7 @@ locals { project_org_id = "${var.folder_id != "" ? "" : var.org_id}" project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" + args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" @@ -52,6 +53,12 @@ locals { } } +resource "null_resource" "args_missinG" { + count = "${local.args_missing}" + + "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = "true" +} + /***************************************** G Suite group information retrieval *****************************************/ From c4081ca49773bea64694ade2c80886fe945f9202 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 16:43:20 +0000 Subject: [PATCH 078/105] Move missing arguments check to gsuite_group --- modules/core_project_factory/main.tf | 7 ------- modules/gsuite_group/main.tf | 11 +++++++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index c7c80e75..0b4d1d21 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -31,7 +31,6 @@ locals { project_org_id = "${var.folder_id != "" ? "" : var.org_id}" project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" - args_missing = "${var.group_name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" s_account_fmt = "${format("serviceAccount:%s", google_service_account.default_service_account.email)}" api_s_account = "${format("%s@cloudservices.gserviceaccount.com", local.project_number)}" api_s_account_fmt = "${format("serviceAccount:%s", local.api_s_account)}" @@ -53,12 +52,6 @@ locals { } } -resource "null_resource" "args_missinG" { - count = "${local.args_missing}" - - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = "true" -} - /***************************************** G Suite group information retrieval *****************************************/ diff --git a/modules/gsuite_group/main.tf b/modules/gsuite_group/main.tf index ea679d84..66ae73e5 100644 --- a/modules/gsuite_group/main.tf +++ b/modules/gsuite_group/main.tf @@ -15,8 +15,15 @@ */ locals { - domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" - email = "${format("%s@%s", var.name, local.domain)}" + args_missing = "${var.name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" + domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" + email = "${format("%s@%s", var.name, local.domain)}" +} + +resource "null_resource" "args_missinG" { + count = "${local.args_missing}" + + "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = "true" } /***************************************** From acfad021718db2a184972cdc594f418e255317f8 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 7 Jan 2019 18:48:25 +0000 Subject: [PATCH 079/105] Make var.org_id required The preconditions script requires org_id to be defined. --- README.md | 4 ++-- modules/core_project_factory/README.md | 4 ++-- modules/core_project_factory/variables.tf | 5 ++--- modules/gsuite_enabled/README.md | 4 ++-- modules/gsuite_enabled/variables.tf | 5 ++--- modules/gsuite_group/README.md | 4 ++-- modules/gsuite_group/main.tf | 12 ++---------- modules/gsuite_group/variables.tf | 5 ++--- test/fixtures/full/variables.tf | 3 +-- variables.tf | 5 ++--- 10 files changed, 19 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 635ebfcb..cd087e9a 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,14 @@ The roles granted are specifically: | bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| domain | The domain name (optional). | string | `""` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | | group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `""` | no | | group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `"roles/editor"` | no | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | | name | The name for the project | string | n/a | yes | -| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | +| org\_id | The organization ID. | string | n/a | yes | | random\_project\_id | Enables project random id generation | string | `"false"` | no | | sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | | shared\_vpc | The ID of the host project which hosts the shared VPC | string | `""` | no | diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 75a2695e..8eb681dd 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -13,14 +13,14 @@ | bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| domain | The domain name (optional). | string | `""` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | | group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | n/a | yes | | group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | n/a | yes | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | | name | The name for the project | string | n/a | yes | -| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | +| org\_id | The organization ID. | string | n/a | yes | | random\_project\_id | Enables project random id generation | string | `"false"` | no | | sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | | shared\_vpc | The ID of the host project which hosts the shared VPC | string | `""` | no | diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 341c6113..b6a952ac 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -34,12 +34,11 @@ variable "random_project_id" { } variable "org_id" { - description = "The organization id (optional if `domain` is passed)" - default = "" + description = "The organization ID." } variable "domain" { - description = "The domain name (optional if `org_id` is passed)" + description = "The domain name (optional)." default = "" } diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index b527dae4..9c3ba5a3 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -68,14 +68,14 @@ The roles granted are specifically: | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | create\_group | Whether to create the group or not | string | `"false"` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | -| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| domain | The domain name (optional). | string | `""` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | | group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `""` | no | | group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | `"roles/editor"` | no | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | | name | The name for the project | string | n/a | yes | -| org\_id | The organization id for the associated services | string | `""` | no | +| org\_id | The organization ID. | string | n/a | yes | | random\_project\_id | Enables project random id generation | string | `"false"` | no | | sa\_group | A GSuite group to place the default Service Account for the project in | string | `""` | no | | sa\_role | A role to give the default Service Account for the project (defaults to none) | string | `""` | no | diff --git a/modules/gsuite_enabled/variables.tf b/modules/gsuite_enabled/variables.tf index 682e026d..bf61bd68 100644 --- a/modules/gsuite_enabled/variables.tf +++ b/modules/gsuite_enabled/variables.tf @@ -26,12 +26,11 @@ variable "random_project_id" { } variable "org_id" { - description = "The organization id for the associated services" - default = "" + description = "The organization ID." } variable "domain" { - description = "The domain name (optional if `org_id` is passed)" + description = "The domain name (optional)." default = "" } diff --git a/modules/gsuite_group/README.md b/modules/gsuite_group/README.md index fb451586..967689bd 100644 --- a/modules/gsuite_group/README.md +++ b/modules/gsuite_group/README.md @@ -6,9 +6,9 @@ | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| domain | The domain name (optional if `org_id` is passed) | string | `""` | no | +| domain | The domain name | string | `""` | no | | name | The name of the group. | string | n/a | yes | -| org\_id | The organization id (optional if `domain` is passed) | string | `""` | no | +| org\_id | The organization ID. | string | n/a | yes | ## Outputs diff --git a/modules/gsuite_group/main.tf b/modules/gsuite_group/main.tf index 66ae73e5..e0eacde1 100644 --- a/modules/gsuite_group/main.tf +++ b/modules/gsuite_group/main.tf @@ -15,21 +15,13 @@ */ locals { - args_missing = "${var.name != "" && var.org_id == "" && var.domain == "" ? 1 : 0}" - domain = "${var.domain != "" ? var.domain : var.org_id != "" ? join("", data.google_organization.org.*.domain) : ""}" - email = "${format("%s@%s", var.name, local.domain)}" -} - -resource "null_resource" "args_missinG" { - count = "${local.args_missing}" - - "ERROR: Variable `group_name` was passed. Please provide either `org_id` or `domain` variables" = "true" + domain = "${var.domain != "" ? var.domain : data.google_organization.org.domain}" + email = "${format("%s@%s", var.name, local.domain)}" } /***************************************** Organization info retrieval *****************************************/ data "google_organization" "org" { - count = "${var.org_id == "" ? 0 : 1}" organization = "${var.org_id}" } diff --git a/modules/gsuite_group/variables.tf b/modules/gsuite_group/variables.tf index a4c8cdb3..a3d74b4b 100644 --- a/modules/gsuite_group/variables.tf +++ b/modules/gsuite_group/variables.tf @@ -15,7 +15,7 @@ */ variable "domain" { - description = "The domain name (optional if `org_id` is passed)" + description = "The domain name" default = "" } @@ -24,6 +24,5 @@ variable "name" { } variable "org_id" { - description = "The organization id (optional if `domain` is passed)" - default = "" + description = "The organization ID." } diff --git a/test/fixtures/full/variables.tf b/test/fixtures/full/variables.tf index bd0c0075..1c7bc75a 100644 --- a/test/fixtures/full/variables.tf +++ b/test/fixtures/full/variables.tf @@ -26,8 +26,7 @@ variable "folder_id" { default = "" } -variable "domain" { -} +variable "domain" {} variable "usage_bucket_name" { default = "" diff --git a/variables.tf b/variables.tf index 6db45303..e49f4fbd 100755 --- a/variables.tf +++ b/variables.tf @@ -20,12 +20,11 @@ variable "random_project_id" { } variable "org_id" { - description = "The organization id (optional if `domain` is passed)" - default = "" + description = "The organization ID." } variable "domain" { - description = "The domain name (optional if `org_id` is passed)" + description = "The domain name (optional)." default = "" } From 2bbac4d65547a888770195500e16aebf3aee4bbc Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 8 Jan 2019 13:22:14 -0500 Subject: [PATCH 080/105] Remove reference to obsolete local.gsuite_group_id --- modules/core_project_factory/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index f6037ea2..1e84cd5c 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -28,8 +28,8 @@ output "domain" { } output "group_email" { - value = "${local.gsuite_group_id ? module.gsuite_group.email : ""}" description = "The email of the created GSuite group with group_name" + value = "${module.gsuite_group.email}" } output "service_account_id" { From ddeb8f0b338e90b0b1dabeab8e2294f4b8f2dcd0 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 8 Jan 2019 13:22:41 -0500 Subject: [PATCH 081/105] Fix descriptions of group_email outputs The group is only created in the gsuite_enabled module. --- modules/core_project_factory/outputs.tf | 2 +- outputs.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index 1e84cd5c..e4930da7 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -28,8 +28,8 @@ output "domain" { } output "group_email" { - description = "The email of the created GSuite group with group_name" value = "${module.gsuite_group.email}" + description = "The email of the GSuite group with group_name" } output "service_account_id" { diff --git a/outputs.tf b/outputs.tf index 11d0904f..2ea85388 100755 --- a/outputs.tf +++ b/outputs.tf @@ -29,7 +29,7 @@ output "domain" { output "group_email" { value = "${module.project-factory.group_email}" - description = "The email of the created GSuite group with group_name" + description = "The email of the GSuite group with group_name" } output "service_account_id" { From e54554812ec4f78e5e9a4f55ba37d72ba486b095 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 9 Jan 2019 11:25:26 -0500 Subject: [PATCH 082/105] Add gsuite_group_role to migrations --- helpers/migrate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index d63ae3ed..6b5d96ec 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -102,6 +102,11 @@ "name": "gke_host_agent", "module": ".module.project-factory" }, + { + "resource_type": "google_project_iam_member", + "name": "gsuite_group_role", + "module": ".module.project-factory", + }, ] From 6f5ffb6ff7af005a3277a097b455e14d4b7a57a7 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 9 Jan 2019 11:25:58 -0500 Subject: [PATCH 083/105] Add service_account_grant_to_group to migrations --- helpers/migrate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index 6b5d96ec..38c45753 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -107,6 +107,11 @@ "name": "gsuite_group_role", "module": ".module.project-factory", }, + { + "resource_type": "google_service_account_iam_member", + "name": "service_account_grant_to_group", + "module": ".module.project-factory", + }, ] From 191d9a69919ea62c6b54844ce3c93807f5bcb29d Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 10 Jan 2019 11:02:54 -0500 Subject: [PATCH 084/105] Fix path to migrate.py in upgrade instructions --- docs/upgrading_to_project_factory_v1.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index b6167b2a..3d3af44b 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -123,7 +123,7 @@ terraform init ### Migrate the Terraform state to match the new Project Factory module structure ``` -./migrate.py terraform.tfstate terraform.tfstate.new +./helpers/migrate.py terraform.tfstate terraform.tfstate.new ``` Expected output: From 9f5daed189a86d58c731460fcee3cdb2c126d3de Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 10 Jan 2019 11:16:45 -0500 Subject: [PATCH 085/105] Restore migrate.py download instructions --- docs/upgrading_to_project_factory_v1.0.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 3d3af44b..10e1159c 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -89,6 +89,13 @@ index d876954..ebb3b1e 100755 org_id = "${var.org_id}" ``` +### Download the state migration script + + ``` +curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/1.0-rc1/helpers/migrate.py +chmod +x migrate.py +``` + ### Reinitialize Terraform ``` @@ -123,7 +130,7 @@ terraform init ### Migrate the Terraform state to match the new Project Factory module structure ``` -./helpers/migrate.py terraform.tfstate terraform.tfstate.new +./migrate.py terraform.tfstate terraform.tfstate.new ``` Expected output: From 097860efab92636414a515c468a62aaa5a8c865a Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 10 Jan 2019 11:49:16 -0500 Subject: [PATCH 086/105] Add group_role_to_vpc_subnets to migration --- helpers/migrate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index 38c45753..af6c236a 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -112,6 +112,11 @@ "name": "service_account_grant_to_group", "module": ".module.project-factory", }, + { + "resource_type": "google_compute_subnetwork_iam_member", + "name": "group_role_to_vpc_subnets", + "module": ".module.project-factory", + }, ] From 7acfc29403bea2e8719d7b995edae63c1f8ff762 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 10 Jan 2019 11:49:29 -0500 Subject: [PATCH 087/105] Add lien to migration --- helpers/migrate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/helpers/migrate.py b/helpers/migrate.py index af6c236a..002c0a87 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -117,6 +117,11 @@ "name": "group_role_to_vpc_subnets", "module": ".module.project-factory", }, + { + "resource_type": "google_resource_manager_lien", + "name": "lien", + "module": ".module.project-factory", + }, ] From 7542083dd17ca051bffe128190ad072a0974dc90 Mon Sep 17 00:00:00 2001 From: Adrien Thebo Date: Thu, 10 Jan 2019 08:58:53 -0800 Subject: [PATCH 088/105] Make migration script idempotent This commit renames the `google_project.project` resource to distinguish between Terraform v0.x and v1.0 and updates the migration script to distinguish between the two migration levels. --- helpers/migrate.py | 18 ++++++++++++------ modules/core_project_factory/main.tf | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/helpers/migrate.py b/helpers/migrate.py index 38c45753..d277be81 100755 --- a/helpers/migrate.py +++ b/helpers/migrate.py @@ -30,6 +30,7 @@ { "resource_type": "google_project", "name": "project", + "rename": "main", "module": ".module.project-factory" }, { @@ -131,9 +132,13 @@ def moves(self): """ resources = self.targets() moves = [] - for (old, module) in resources: + for (old, migration) in resources: new = copy.deepcopy(old) - new.module += module + new.module += migration["module"] + + # If the "rename" value is set, update the copied resource with the new name + if "rename" in migration: + new.name = migration["rename"] pair = (old.path(), new.path()) moves.append(pair) @@ -147,13 +152,13 @@ def targets(self): """ to_move = [] - for selector in MIGRATIONS: - resource_type = selector["resource_type"] - resource_name = selector["name"] + for migration in MIGRATIONS: + resource_type = migration["resource_type"] + resource_name = migration["name"] matching_resources = self.project_factory.get_resources( resource_type, resource_name) - to_move += [(r, selector["module"]) for r in matching_resources] + to_move += [(r, migration) for r in matching_resources] return to_move @@ -334,6 +339,7 @@ def migrate(statefile, dryrun=False): factories = [ module for module in modules if module.has_resource("random_id", "random_project_id_suffix") + and module.has_resource("google_project", "project") ] print("---- Migrating the following project-factory modules:") diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 0b4d1d21..d80863e6 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -26,8 +26,8 @@ resource "random_id" "random_project_id_suffix" { *****************************************/ locals { group_id = "${var.group_name != "" ? format("group:%s", module.gsuite_group.email) : ""}" - project_id = "${google_project.project.project_id}" - project_number = "${google_project.project.number}" + project_id = "${google_project.main.project_id}" + project_number = "${google_project.main.number}" project_org_id = "${var.folder_id != "" ? "" : var.org_id}" project_folder_id = "${var.folder_id != "" ? var.folder_id : ""}" temp_project_id = "${var.random_project_id ? format("%s-%s",var.name,random_id.random_project_id_suffix.hex) : var.name}" @@ -91,7 +91,7 @@ EOD /******************************************* Project creation *******************************************/ -resource "google_project" "project" { +resource "google_project" "main" { name = "${var.name}" project_id = "${local.temp_project_id}" org_id = "${local.project_org_id}" @@ -111,7 +111,7 @@ resource "google_project" "project" { *****************************************/ resource "google_resource_manager_lien" "lien" { count = "${var.lien ? 1 : 0}" - parent = "projects/${google_project.project.number}" + parent = "projects/${google_project.main.number}" restrictions = ["resourcemanager.projects.delete"] origin = "project-factory" reason = "Project Factory lien" @@ -126,7 +126,7 @@ resource "google_project_service" "project_services" { project = "${local.project_id}" service = "${element(var.activate_apis, count.index)}" - depends_on = ["google_project.project"] + depends_on = ["google_project.main"] } /****************************************** @@ -145,7 +145,7 @@ resource "google_compute_shared_vpc_service_project" "shared_vpc_attachment" { Default compute service account retrieval *****************************************/ data "google_compute_default_service_account" "default" { - project = "${google_project.project.id}" + project = "${google_project.main.id}" } /****************************************** From c2a5cd67f2c10bf6e7a7cb19f7adafddf31d7927 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 10 Jan 2019 20:58:33 -0500 Subject: [PATCH 089/105] Minor updates to migration docs around preconditions and org_id --- docs/upgrading_to_project_factory_v1.0.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 10e1159c..1f116407 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -89,6 +89,9 @@ index d876954..ebb3b1e 100755 org_id = "${var.org_id}" ``` +Additionally, `org_id` is now required so you will need to add +it as an argument if you didn't already specify it on your projects. + ### Download the state migration script ``` @@ -154,7 +157,8 @@ terraform plan -state terraform.tfstate.new The G Suite refactor adds an additional IAM membership and needs to re-create two resources, due to how resources were split up between the `gsuite_enabled` -and `core_project_factory` modules. +and `core_project_factory` modules. Depending on the version +you are upgrading from, it might also add a `null_resource` for `preconditions` checks. ```txt @@ -275,6 +279,20 @@ Error: module "project-pfactory-development": "create_group" is not a valid argu These are related to projects which depend on G Suite functionality. Make sure to update the source of such projects to point to the [G Suite module](../modules/gsuite_enabled) +### Missing `org_id` + +If your existing configuration doesn't specify the `org_id`, +you might see some errors on upgrade: + +``` +Initializing the backend... + +Error: module "project_factory": missing required argument "org_id" +``` + +The fix for this is to explicitly set the `org_id` argument +on your projects. + ### The migration script fails to run If you get an error like this when running the migration script, it means you need upgrade From 4bff4048a694134808a655d86b84896c631dfc4c Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 10 Jan 2019 21:00:12 -0500 Subject: [PATCH 090/105] Add clean up steps --- docs/upgrading_to_project_factory_v1.0.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 1f116407..7c3f0971 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -264,6 +264,20 @@ After restoring remote state, you need to re-initialize Terraform and push your terraform init -force-copy ``` +### Clean up + +Once you are done with the migration, you can safely remove `migrate.py`. + +``` +rm migrate.py +``` + +If you are using remote state, you can also remove the local state copies. + +``` +rm -rf terraform.tfstate* +``` + ## Troubleshooting ### Errors about invalid arguments From f51a2adb60e85768460878a969e5ede4a43cc065 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 11 Jan 2019 17:09:05 -0500 Subject: [PATCH 091/105] Incorporate changes from #91 --- main.tf | 43 ++++++++++++----------- modules/core_project_factory/main.tf | 2 ++ modules/core_project_factory/variables.tf | 6 ++++ modules/gsuite_enabled/main.tf | 43 ++++++++++++----------- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/main.tf b/main.tf index 148086aa..caeaa80b 100755 --- a/main.tf +++ b/main.tf @@ -17,25 +17,26 @@ module "project-factory" { source = "modules/core_project_factory" - group_name = "${var.group_name}" - group_role = "${var.group_role}" - lien = "${var.lien}" - random_project_id = "${var.random_project_id}" - org_id = "${var.org_id}" - domain = "${var.domain}" - name = "${var.name}" - shared_vpc = "${var.shared_vpc}" - billing_account = "${var.billing_account}" - folder_id = "${var.folder_id}" - sa_role = "${var.sa_role}" - activate_apis = "${var.activate_apis}" - usage_bucket_name = "${var.usage_bucket_name}" - usage_bucket_prefix = "${var.usage_bucket_prefix}" - credentials_path = "${var.credentials_path}" - shared_vpc_subnets = "${var.shared_vpc_subnets}" - labels = "${var.labels}" - bucket_project = "${var.bucket_project}" - bucket_name = "${var.bucket_name}" - auto_create_network = "${var.auto_create_network}" - app_engine = "${var.app_engine}" + group_name = "${var.group_name}" + group_role = "${var.group_role}" + lien = "${var.lien}" + random_project_id = "${var.random_project_id}" + org_id = "${var.org_id}" + domain = "${var.domain}" + name = "${var.name}" + shared_vpc = "${var.shared_vpc}" + billing_account = "${var.billing_account}" + folder_id = "${var.folder_id}" + sa_role = "${var.sa_role}" + activate_apis = "${var.activate_apis}" + usage_bucket_name = "${var.usage_bucket_name}" + usage_bucket_prefix = "${var.usage_bucket_prefix}" + credentials_path = "${var.credentials_path}" + shared_vpc_subnets = "${var.shared_vpc_subnets}" + labels = "${var.labels}" + bucket_project = "${var.bucket_project}" + bucket_name = "${var.bucket_name}" + auto_create_network = "${var.auto_create_network}" + app_engine = "${var.app_engine}" + disable_services_on_destroy = "${var.disable_services_on_destroy}" } diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index fe81d6e4..3c9dd7cc 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -126,6 +126,8 @@ resource "google_project_service" "project_services" { project = "${local.project_id}" service = "${element(var.activate_apis, count.index)}" + disable_on_destroy = "${var.disable_services_on_destroy}" + depends_on = ["google_project.main"] } diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index b6a952ac..6708b3a5 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -117,3 +117,9 @@ variable "app_engine" { type = "map" default = {} } + +variable "disable_services_on_destroy" { + description = "Whether project services will be disabled when the resources are destroyed" + default = "true" + type = "string" +} diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index c864af72..ad865181 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -67,25 +67,26 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { module "project-factory" { source = "../core_project_factory/" - group_name = "${local.group_name}" - group_role = "${var.group_role}" - lien = "${var.lien}" - random_project_id = "${var.random_project_id}" - org_id = "${var.org_id}" - domain = "${var.domain}" - name = "${var.name}" - shared_vpc = "${var.shared_vpc}" - billing_account = "${var.billing_account}" - folder_id = "${var.folder_id}" - sa_role = "${var.sa_role}" - activate_apis = "${var.activate_apis}" - usage_bucket_name = "${var.usage_bucket_name}" - usage_bucket_prefix = "${var.usage_bucket_prefix}" - credentials_path = "${var.credentials_path}" - shared_vpc_subnets = "${var.shared_vpc_subnets}" - labels = "${var.labels}" - bucket_project = "${var.bucket_project}" - bucket_name = "${var.bucket_name}" - auto_create_network = "${var.auto_create_network}" - app_engine = "${var.app_engine}" + group_name = "${local.group_name}" + group_role = "${var.group_role}" + lien = "${var.lien}" + random_project_id = "${var.random_project_id}" + org_id = "${var.org_id}" + domain = "${var.domain}" + name = "${var.name}" + shared_vpc = "${var.shared_vpc}" + billing_account = "${var.billing_account}" + folder_id = "${var.folder_id}" + sa_role = "${var.sa_role}" + activate_apis = "${var.activate_apis}" + usage_bucket_name = "${var.usage_bucket_name}" + usage_bucket_prefix = "${var.usage_bucket_prefix}" + credentials_path = "${var.credentials_path}" + shared_vpc_subnets = "${var.shared_vpc_subnets}" + labels = "${var.labels}" + bucket_project = "${var.bucket_project}" + bucket_name = "${var.bucket_name}" + auto_create_network = "${var.auto_create_network}" + app_engine = "${var.app_engine}" + disable_services_on_destroy = "${var.disable_services_on_destroy}" } From 29e3a457b735a1d9320dc6411e986dbb9ab4af1b Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 11 Jan 2019 17:12:58 -0500 Subject: [PATCH 092/105] Update CHANGELOG --- CHANGELOG.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ceac12..0f6488ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,29 @@ -## 0.3.0 +## [1.0.0] +1.0.0 is a major backwards incompatible release. See the [upgrade guide](./docs/upgrading_to_project_factory_v1.0.md) for details. + +### ADDED +- Support for disable_services_on_destroy flag to leave service active on delete. #91 -IMPROVEMENTS: +### Changed +- Refactored project factory to eliminate the dependenency on the G Suite provider for all projects. #94 +## 0.3.0 +### ADDED - Implement billing account role. - Remove `CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE`. - Lien support. -BUG FIXES: - +### FIXED - Fix/refactor `helpers/init_debian.sh`. ## 0.2.1 - +### ADDED - Explicit dependency on `google_project_service`. ## 0.2.0 - +### ADDED - Make IAM bindings non-authoritative. ## 0.1.0 - -This is the initial release of the Project Factory Module. +### ADDED +- This is the initial release of the Project Factory Module. From ee2357dcc025de1b958c6d6df80ab37dd16c5740 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 11 Jan 2019 18:25:53 -0500 Subject: [PATCH 093/105] Try to fix example --- modules/gsuite_enabled/variables.tf | 6 ++ test/fixtures/full/variables.tf | 70 +------------------ test/fixtures/minimal/variables.tf | 34 +-------- test/fixtures/shared/terraform.tfvars.example | 4 +- 4 files changed, 10 insertions(+), 104 deletions(-) mode change 100644 => 120000 test/fixtures/full/variables.tf mode change 100644 => 120000 test/fixtures/minimal/variables.tf diff --git a/modules/gsuite_enabled/variables.tf b/modules/gsuite_enabled/variables.tf index bf61bd68..ee070fb8 100644 --- a/modules/gsuite_enabled/variables.tf +++ b/modules/gsuite_enabled/variables.tf @@ -134,3 +134,9 @@ variable "app_engine" { type = "map" default = {} } + +variable "disable_services_on_destroy" { + description = "Whether project services will be disabled when the resources are destroyed" + default = "true" + type = "string" +} diff --git a/test/fixtures/full/variables.tf b/test/fixtures/full/variables.tf deleted file mode 100644 index 1c7bc75a..00000000 --- a/test/fixtures/full/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "credentials_path" {} - -variable "name" { - default = "pf-test-int-full" -} - -variable "org_id" {} - -variable "folder_id" { - default = "" -} - -variable "domain" {} - -variable "usage_bucket_name" { - default = "" -} - -variable "usage_bucket_prefix" { - default = "" -} - -variable "billing_account" {} - -variable "group_name" { - default = "" -} - -variable "create_group" { - default = "false" -} - -variable "group_role" { - default = "roles/viewer" -} - -variable "shared_vpc" { - default = "" -} - -variable "sa_role" { - default = "roles/editor" -} - -variable "sa_group" { - default = "" -} - -variable "region" { - default = "us-east4" -} - -variable "gsuite_admin_account" {} diff --git a/test/fixtures/full/variables.tf b/test/fixtures/full/variables.tf new file mode 120000 index 00000000..c113c00a --- /dev/null +++ b/test/fixtures/full/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/fixtures/minimal/variables.tf b/test/fixtures/minimal/variables.tf deleted file mode 100644 index 755adc30..00000000 --- a/test/fixtures/minimal/variables.tf +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "credentials_path" {} - -variable "name" { - default = "pf-test-int-minimal" -} - -variable "org_id" {} - -variable "folder_id" { - default = "" -} - -variable "domain" { - default = "" -} - -variable "billing_account" {} diff --git a/test/fixtures/minimal/variables.tf b/test/fixtures/minimal/variables.tf new file mode 120000 index 00000000..c113c00a --- /dev/null +++ b/test/fixtures/minimal/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/fixtures/shared/terraform.tfvars.example b/test/fixtures/shared/terraform.tfvars.example index 688095a7..f979f86b 100644 --- a/test/fixtures/shared/terraform.tfvars.example +++ b/test/fixtures/shared/terraform.tfvars.example @@ -23,7 +23,7 @@ org_id = "000000000000" billing_account = "000000-000000-000000" # The service account credentials to use when running Terraform. -credentials_path = "credentials.json" +credentials_path = "../../../credentials.json" # The G Suite admin account to impersonate when managing G Suite groups and group membership gsuite_admin_account = "admin@example.com" @@ -45,7 +45,7 @@ shared_vpc = "host-vpc-project-id" # The GCP domain name # Example source: `domain=$(gcloud organizations list --format='get(displayName)')` -# domain = "example.com" +domain = "example.com" ##### Optional variables #### From 803892da3f00ff9a5520b805094457dd502c7dc0 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 11 Jan 2019 18:45:58 -0500 Subject: [PATCH 094/105] Set default group-name --- test/fixtures/shared/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index f3027de0..c4f4be7e 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -36,7 +36,7 @@ variable "usage_bucket_prefix" { variable "billing_account" {} variable "group_name" { - default = "" + default = "project-factory" } variable "create_group" { From 6b2029ddc8257610e4748859069b449b5ef6e1d2 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Mon, 14 Jan 2019 14:59:40 -0500 Subject: [PATCH 095/105] Remove random_string suffix from fixture modules The interpolation of random_string was causing an issue for counts based off of outputs from gsuite_group modules. It's not clear why this is happening. The tests can not be run concurrently at this point so the there is no need to avoid naming collisions any way. --- test/fixtures/full/main.tf | 10 ++-------- test/fixtures/minimal/main.tf | 8 +------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/test/fixtures/full/main.tf b/test/fixtures/full/main.tf index 483aa931..18dce841 100644 --- a/test/fixtures/full/main.tf +++ b/test/fixtures/full/main.tf @@ -40,16 +40,10 @@ locals { shared_vpc_subnets = ["projects/${var.shared_vpc}/regions/${module.vpc.subnets_regions[0]}/subnetworks/${module.vpc.subnets_names[0]}"] } -resource "random_string" "suffix" { - length = 8 - special = false - upper = false -} - module "vpc" { source = "terraform-google-modules/network/google" version = "~> 0.4.0" - network_name = "pf-test-int-full-${random_string.suffix.result}" + network_name = "pf-test-int-full" project_id = "${var.shared_vpc}" # The provided project must already be a Shared VPC host @@ -77,7 +71,7 @@ module "project-factory" { source = "../../../modules/gsuite_enabled" domain = "${var.domain}" - name = "pf-ci-test-full-${random_string.suffix.result}" + name = "pf-ci-test-full" random_project_id = true org_id = "${var.org_id}" diff --git a/test/fixtures/minimal/main.tf b/test/fixtures/minimal/main.tf index d333da67..3933d9dd 100644 --- a/test/fixtures/minimal/main.tf +++ b/test/fixtures/minimal/main.tf @@ -24,15 +24,9 @@ provider "google-beta" { version = "~> 1.19" } -resource "random_string" "suffix" { - length = 5 - special = false - upper = false -} - module "project-factory" { source = "../../../" - name = "pf-ci-test-minimal-${random_string.suffix.result}" + name = "pf-ci-test-minimal" random_project_id = true domain = "${var.domain}" org_id = "${var.org_id}" From ad6b467664c9895a75839b665e2114413da05530 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 16 Jan 2019 00:28:47 +0000 Subject: [PATCH 096/105] Fix counts which can not be computed `gsuite_group` does not need to be included in `core_project_factory` since the only relevant output is `email`. The `manage_group` variable on `core_project_factory` removes the need to check the value of the `group_name` variable which was causing an issue during graph resolution. The `count` of the `additive_service_account_grant_to_group` in the `full` fixture is removed because the `group_email` output will always be defined in that fixture. --- main.tf | 15 +++++++++++++-- modules/core_project_factory/main.tf | 21 +++++---------------- modules/core_project_factory/outputs.tf | 10 ---------- modules/core_project_factory/variables.tf | 17 +++++++++-------- modules/gsuite_enabled/main.tf | 4 ++-- modules/gsuite_enabled/outputs.tf | 2 +- outputs.tf | 4 ++-- test/fixtures/full/main.tf | 15 +++++++-------- 8 files changed, 39 insertions(+), 49 deletions(-) diff --git a/main.tf b/main.tf index 148086aa..f55d1f42 100755 --- a/main.tf +++ b/main.tf @@ -14,15 +14,26 @@ * limitations under the License. */ +/***************************************** + Organization info retrieval + *****************************************/ +module "gsuite_group" { + source = "${path.module}/modules/gsuite_group" + + domain = "${var.domain}" + name = "${var.group_name}" + org_id = "${var.org_id}" +} + module "project-factory" { source = "modules/core_project_factory" - group_name = "${var.group_name}" + group_email = "${module.gsuite_group.email}" group_role = "${var.group_role}" lien = "${var.lien}" + manage_group = "${var.group_name != "" ? "true" : "false"}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" - domain = "${var.domain}" name = "${var.name}" shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" diff --git a/modules/core_project_factory/main.tf b/modules/core_project_factory/main.tf index 3c9dd7cc..81c90c9f 100644 --- a/modules/core_project_factory/main.tf +++ b/modules/core_project_factory/main.tf @@ -25,7 +25,7 @@ resource "random_id" "random_project_id_suffix" { Locals configuration *****************************************/ locals { - group_id = "${var.group_name != "" ? format("group:%s", module.gsuite_group.email) : ""}" + group_id = "${var.manage_group ? format("group:%s", var.group_email) : ""}" project_id = "${google_project.main.project_id}" project_number = "${google_project.main.number}" project_org_id = "${var.folder_id != "" ? "" : var.org_id}" @@ -52,17 +52,6 @@ locals { } } -/***************************************** - G Suite group information retrieval - *****************************************/ -module "gsuite_group" { - source = "../gsuite_group" - - domain = "${var.domain}" - name = "${var.group_name}" - org_id = "${var.org_id}" -} - resource "null_resource" "preconditions" { triggers { credentials_path = "${var.credentials_path}" @@ -190,7 +179,7 @@ resource "google_project_iam_member" "default_service_account_membership" { Gsuite Group Role Configuration *****************************************/ resource "google_project_iam_member" "gsuite_group_role" { - count = "${local.group_id != "" ? 1 : 0}" + count = "${var.manage_group ? 1 : 0}" member = "${local.group_id}" project = "${local.project_id}" @@ -201,7 +190,7 @@ resource "google_project_iam_member" "gsuite_group_role" { Granting serviceAccountUser to group *****************************************/ resource "google_service_account_iam_member" "service_account_grant_to_group" { - count = "${local.group_id != "" ? 1 : 0}" + count = "${var.manage_group ? 1 : 0}" member = "${local.group_id}" role = "roles/iam.serviceAccountUser" @@ -246,7 +235,7 @@ resource "google_compute_subnetwork_iam_member" "service_account_role_to_vpc_sub resource "google_compute_subnetwork_iam_member" "group_role_to_vpc_subnets" { provider = "google-beta" - count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && local.group_id != "" ? length(var.shared_vpc_subnets) : 0 }" + count = "${var.shared_vpc != "" && length(compact(var.shared_vpc_subnets)) > 0 && var.manage_group ? length(var.shared_vpc_subnets) : 0 }" member = "${local.group_id}" project = "${var.shared_vpc}" @@ -299,7 +288,7 @@ resource "google_storage_bucket" "project_bucket" { Project's bucket storage.admin granting to group ***********************************************/ resource "google_storage_bucket_iam_member" "group_storage_admin_on_project_bucket" { - count = "${local.create_bucket && local.group_id != "" ? 1 : 0}" + count = "${local.create_bucket && var.manage_group ? 1 : 0}" bucket = "${google_storage_bucket.project_bucket.name}" member = "${local.group_id}" diff --git a/modules/core_project_factory/outputs.tf b/modules/core_project_factory/outputs.tf index e4930da7..0c6e52e8 100644 --- a/modules/core_project_factory/outputs.tf +++ b/modules/core_project_factory/outputs.tf @@ -22,16 +22,6 @@ output "project_number" { value = "${local.project_number}" } -output "domain" { - value = "${module.gsuite_group.domain}" - description = "The organization's domain" -} - -output "group_email" { - value = "${module.gsuite_group.email}" - description = "The email of the GSuite group with group_name" -} - output "service_account_id" { value = "${google_service_account.default_service_account.account_id}" description = "The id of the default service account" diff --git a/modules/core_project_factory/variables.tf b/modules/core_project_factory/variables.tf index 6708b3a5..5d8fc71e 100644 --- a/modules/core_project_factory/variables.tf +++ b/modules/core_project_factory/variables.tf @@ -14,12 +14,13 @@ * limitations under the License. */ -variable "group_name" { - description = "A group to control the project by being assigned group_role (defaults to project editor)" +variable "group_email" { + description = "The email address of a group to control the project by being assigned group_role." } variable "group_role" { - description = "The role to give the controlling group (group_name) over the project (defaults to project editor)" + description = "The role to give the controlling group (group_name) over the project." + default = "" } variable "lien" { @@ -28,6 +29,11 @@ variable "lien" { type = "string" } +variable "manage_group" { + description = "A toggle to indicate if a G Suite group should be managed." + default = "false" +} + variable "random_project_id" { description = "Enables project random id generation" default = "false" @@ -37,11 +43,6 @@ variable "org_id" { description = "The organization ID." } -variable "domain" { - description = "The domain name (optional)." - default = "" -} - variable "name" { description = "The name for the project" } diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index ad865181..1caa992e 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -67,12 +67,12 @@ resource "gsuite_group_member" "api_s_account_api_sa_group_member" { module "project-factory" { source = "../core_project_factory/" - group_name = "${local.group_name}" + group_email = "${module.gsuite_group.email}" group_role = "${var.group_role}" lien = "${var.lien}" + manage_group = "true" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" - domain = "${var.domain}" name = "${var.name}" shared_vpc = "${var.shared_vpc}" billing_account = "${var.billing_account}" diff --git a/modules/gsuite_enabled/outputs.tf b/modules/gsuite_enabled/outputs.tf index ec2de95c..5f4b0ec1 100644 --- a/modules/gsuite_enabled/outputs.tf +++ b/modules/gsuite_enabled/outputs.tf @@ -23,7 +23,7 @@ output "project_number" { } output "domain" { - value = "${module.project-factory.domain}" + value = "${module.gsuite_group.domain}" description = "The organization's domain" } diff --git a/outputs.tf b/outputs.tf index 2ea85388..d1a8fd90 100755 --- a/outputs.tf +++ b/outputs.tf @@ -23,12 +23,12 @@ output "project_number" { } output "domain" { - value = "${module.project-factory.domain}" + value = "${module.gsuite_group.domain}" description = "The organization's domain" } output "group_email" { - value = "${module.project-factory.group_email}" + value = "${module.gsuite_group.email}" description = "The email of the GSuite group with group_name" } diff --git a/test/fixtures/full/main.tf b/test/fixtures/full/main.tf index fc098745..f573be04 100644 --- a/test/fixtures/full/main.tf +++ b/test/fixtures/full/main.tf @@ -47,10 +47,11 @@ resource "random_string" "suffix" { } module "vpc" { - source = "terraform-google-modules/network/google" - version = "~> 0.4.0" - network_name = "pf-test-int-full-${random_string.suffix.result}" - project_id = "${var.shared_vpc}" + source = "terraform-google-modules/network/google" + version = "~> 0.4.0" + network_name = "pf-test-int-full-${random_string.suffix.result}" + project_id = "${var.shared_vpc}" + # The provided project must already be a Shared VPC host shared_vpc_host = "false" @@ -75,8 +76,8 @@ module "vpc" { module "project-factory" { source = "../../../modules/gsuite_enabled" - name = "pf-ci-test-full-${random_string.suffix.result}" - random_project_id = "true" + name = "pf-ci-test-full-${random_string.suffix.result}" + random_project_id = "true" domain = "${var.domain}" org_id = "${var.org_id}" @@ -133,8 +134,6 @@ resource "google_project_iam_member" "additive_shared_vpc_role" { } resource "google_service_account_iam_member" "additive_service_account_grant_to_group" { - count = "${module.project-factory.group_email != "" ? 1 : 0}" - service_account_id = "projects/${module.project-factory.project_id}/serviceAccounts/${module.project-factory.service_account_email}" role = "roles/iam.serviceAccountUser" From c786760a72fa8c2695486d29a9cde4792b9b3189 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 16 Jan 2019 00:29:58 +0000 Subject: [PATCH 097/105] Generate documentation --- README.md | 5 +++-- modules/core_project_factory/README.md | 9 ++++----- modules/gsuite_enabled/README.md | 1 + test/fixtures/full/README.md | 3 +-- test/fixtures/minimal/README.md | 23 ++++++++++++++++++++--- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ff12e9c1..82da88aa 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ The roles granted are specifically: | bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| disable\_services\_on\_destroy | Whether project services will be disabled when the resources are destroyed | string | `"true"` | no | | domain | The domain name (optional). | string | `""` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | | group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | `""` | no | @@ -114,7 +115,7 @@ The roles granted are specifically: |------|-------------| | app\_engine\_enabled | Whether app engine is enabled | | domain | The organization's domain | -| group\_email | The email of the created GSuite group with group_name | +| group\_email | The email of the GSuite group with group_name | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | | project\_id | | @@ -400,4 +401,4 @@ versions][release-new-version]. [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta [terraform-provider-gsuite]: https://github.com/DeviaVir/terraform-provider-gsuite [glossary]: /docs/GLOSSARY.md -[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions +[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions \ No newline at end of file diff --git a/modules/core_project_factory/README.md b/modules/core_project_factory/README.md index 8eb681dd..df2e55a8 100644 --- a/modules/core_project_factory/README.md +++ b/modules/core_project_factory/README.md @@ -13,12 +13,13 @@ | bucket\_name | A name for a GCS bucket to create (in the bucket_project project), useful for Terraform state (optional) | string | `""` | no | | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | -| domain | The domain name (optional). | string | `""` | no | +| disable\_services\_on\_destroy | Whether project services will be disabled when the resources are destroyed | string | `"true"` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | -| group\_name | A group to control the project by being assigned group_role (defaults to project editor) | string | n/a | yes | -| group\_role | The role to give the controlling group (group_name) over the project (defaults to project editor) | string | n/a | yes | +| group\_email | The email address of a group to control the project by being assigned group_role. | string | n/a | yes | +| group\_role | The role to give the controlling group (group_name) over the project. | string | `""` | no | | labels | Map of labels for project | map | `` | no | | lien | Add a lien on the project to prevent accidental deletion | string | `"false"` | no | +| manage\_group | A toggle to indicate if a G Suite group should be managed. | string | `"false"` | no | | name | The name for the project | string | n/a | yes | | org\_id | The organization ID. | string | n/a | yes | | random\_project\_id | Enables project random id generation | string | `"false"` | no | @@ -35,8 +36,6 @@ | api\_s\_account | API service account email | | api\_s\_account\_fmt | API service account email formatted for terraform use | | app\_engine\_enabled | Whether app engine is enabled | -| domain | The organization's domain | -| group\_email | The email of the created GSuite group with group_name | | project\_bucket\_name | The name of the projec's bucket | | project\_bucket\_self\_link | Project's bucket selfLink | | project\_bucket\_url | Project's bucket url | diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index 9c3ba5a3..e8374458 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -68,6 +68,7 @@ The roles granted are specifically: | bucket\_project | A project to create a GCS bucket (bucket_name) in, useful for Terraform state (optional) | string | `""` | no | | create\_group | Whether to create the group or not | string | `"false"` | no | | credentials\_path | Path to a Service Account credentials file with permissions documented in the readme | string | n/a | yes | +| disable\_services\_on\_destroy | Whether project services will be disabled when the resources are destroyed | string | `"true"` | no | | domain | The domain name (optional). | string | `""` | no | | folder\_id | The ID of a folder to host this project | string | `""` | no | | group\_name | A group to control the project by being assigned group_role - defaults to ${project_name}-editors | string | `""` | no | diff --git a/test/fixtures/full/README.md b/test/fixtures/full/README.md index b9c27f55..616ca90e 100644 --- a/test/fixtures/full/README.md +++ b/test/fixtures/full/README.md @@ -14,7 +14,6 @@ | group\_name | | string | `""` | no | | group\_role | | string | `"roles/viewer"` | no | | gsuite\_admin\_account | | string | n/a | yes | -| name | | string | `"pf-test-int-full"` | no | | org\_id | | string | n/a | yes | | region | | string | `"us-east4"` | no | | sa\_group | | string | `""` | no | @@ -27,7 +26,7 @@ | Name | Description | |------|-------------| -| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | +| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials | | domain | | | extra\_service\_account\_email | | | group\_email | | diff --git a/test/fixtures/minimal/README.md b/test/fixtures/minimal/README.md index 159d992f..f96a107f 100644 --- a/test/fixtures/minimal/README.md +++ b/test/fixtures/minimal/README.md @@ -7,20 +7,37 @@ | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | billing\_account | | string | n/a | yes | +| create\_group | | string | `"false"` | no | | credentials\_path | | string | n/a | yes | -| domain | | string | `""` | no | +| domain | | string | n/a | yes | | folder\_id | | string | `""` | no | -| name | | string | `"pf-test-int-minimal"` | no | +| group\_name | | string | `""` | no | +| group\_role | | string | `"roles/viewer"` | no | +| gsuite\_admin\_account | | string | n/a | yes | | org\_id | | string | n/a | yes | +| region | | string | `"us-east4"` | no | +| sa\_group | | string | `""` | no | +| sa\_role | | string | `"roles/editor"` | no | +| shared\_vpc | | string | `""` | no | +| usage\_bucket\_name | | string | `""` | no | +| usage\_bucket\_prefix | | string | `""` | no | ## Outputs | Name | Description | |------|-------------| -| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials. | +| credentials\_path | Pass through the `credentials_path` variable so that InSpec can reuse the credentials | | domain | | +| group\_email | | +| group\_role | | +| gsuite\_admin\_account | | | project\_id | | | project\_number | | +| region | | +| sa\_role | | | service\_account\_email | | +| shared\_vpc | | +| usage\_bucket\_name | | +| usage\_bucket\_prefix | | [^]: (autogen_docs_end) \ No newline at end of file From 673f6521628e0ac475f955f19ce3dd8978736b39 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 15 Jan 2019 20:40:15 -0500 Subject: [PATCH 098/105] Remove interpolation from module source --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index f55d1f42..a4faa6a0 100755 --- a/main.tf +++ b/main.tf @@ -18,7 +18,7 @@ Organization info retrieval *****************************************/ module "gsuite_group" { - source = "${path.module}/modules/gsuite_group" + source = "modules/gsuite_group" domain = "${var.domain}" name = "${var.group_name}" From 50aa1d8025a1441497f92fbe7420fda22748d5e6 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 16 Jan 2019 13:57:57 -0500 Subject: [PATCH 099/105] Only manage group if it is existent This commit patches gsuite_enabled to only manage a group when a name is provided or the toggle to create a group is set. --- modules/gsuite_enabled/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 1caa992e..4815190d 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -70,7 +70,7 @@ module "project-factory" { group_email = "${module.gsuite_group.email}" group_role = "${var.group_role}" lien = "${var.lien}" - manage_group = "true" + manage_group = "${var.group_name != "" || var.create_group ? "true" : "false"}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" name = "${var.name}" From c347a37760cd14acef56b636fa4e5bfb18db3e19 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 16 Jan 2019 14:02:14 -0500 Subject: [PATCH 100/105] Remove redundant boolean logic --- modules/gsuite_enabled/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gsuite_enabled/main.tf b/modules/gsuite_enabled/main.tf index 4815190d..a098c0cf 100644 --- a/modules/gsuite_enabled/main.tf +++ b/modules/gsuite_enabled/main.tf @@ -70,7 +70,7 @@ module "project-factory" { group_email = "${module.gsuite_group.email}" group_role = "${var.group_role}" lien = "${var.lien}" - manage_group = "${var.group_name != "" || var.create_group ? "true" : "false"}" + manage_group = "${var.group_name != "" || var.create_group}" random_project_id = "${var.random_project_id}" org_id = "${var.org_id}" name = "${var.name}" From beb050f17b3e82f95d92662e2e3ba7d612de2b43 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 17 Jan 2019 10:49:41 -0500 Subject: [PATCH 101/105] Capitalize Changed header in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f6488ef..eb3bb095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### ADDED - Support for disable_services_on_destroy flag to leave service active on delete. #91 -### Changed +### CHANGED - Refactored project factory to eliminate the dependenency on the G Suite provider for all projects. #94 ## 0.3.0 From 633ffc8d0499fe5f3bc2e63b3f5c476d3539c4d8 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 17 Jan 2019 10:50:34 -0500 Subject: [PATCH 102/105] Remove brackets from 1.0.0 header in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3bb095..c2fdc1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## [1.0.0] +## 1.0.0 1.0.0 is a major backwards incompatible release. See the [upgrade guide](./docs/upgrading_to_project_factory_v1.0.md) for details. ### ADDED From 383b5cb89c5da1e613de99c318e37437a16b3108 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 17 Jan 2019 10:56:59 -0500 Subject: [PATCH 103/105] Pin module to 1.0 in README example --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 746493c6..6190604a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ There are multiple examples included in the [examples](./examples/) folder but s ```hcl module "project-factory" { source = "terraform-google-modules/project-factory/google" - version = "v0.3.0" + version = "~> 1.0" name = "pf-test-1" random_project_id = "true" @@ -401,4 +401,4 @@ versions][release-new-version]. [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta [terraform-provider-gsuite]: https://github.com/DeviaVir/terraform-provider-gsuite [glossary]: /docs/GLOSSARY.md -[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions \ No newline at end of file +[release-new-version]: https://www.terraform.io/docs/registry/modules/publish.html#releasing-new-versions From ae652cb0c0b72bae85772f29c59c6f41fbdf4f14 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 17 Jan 2019 11:04:44 -0500 Subject: [PATCH 104/105] Target v1.0.0 in migration script download --- docs/upgrading_to_project_factory_v1.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrading_to_project_factory_v1.0.md b/docs/upgrading_to_project_factory_v1.0.md index 7c3f0971..8e2458be 100644 --- a/docs/upgrading_to_project_factory_v1.0.md +++ b/docs/upgrading_to_project_factory_v1.0.md @@ -95,7 +95,7 @@ it as an argument if you didn't already specify it on your projects. ### Download the state migration script ``` -curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/1.0-rc1/helpers/migrate.py +curl -O https://raw.githubusercontent.com/terraform-google-modules/terraform-google-project-factory/v1.0.0/helpers/migrate.py chmod +x migrate.py ``` From 574ca009f0c3522eedd613a6c8f942c746142b09 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 17 Jan 2019 11:08:45 -0500 Subject: [PATCH 105/105] Pin gsuite_enabled module to 1.0 in README --- modules/gsuite_enabled/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gsuite_enabled/README.md b/modules/gsuite_enabled/README.md index ab12cef6..225639e3 100644 --- a/modules/gsuite_enabled/README.md +++ b/modules/gsuite_enabled/README.md @@ -10,7 +10,7 @@ There are multiple examples included in the [examples] folder but simple usage i ```hcl module "project-factory" { source = "terraform-google-modules/project-factory/google//modules/gsuite_enabled" - version = "0.2.1" + version = "~> 1.0" billing_account = "ABCDEF-ABCDEF-ABCDEF" create_group = "true" @@ -105,4 +105,4 @@ The roles granted are specifically: [^]: (autogen_docs_end) [examples]: ../../examples/ -[root-module]: ../../README.md \ No newline at end of file +[root-module]: ../../README.md