From 8353df6b8ba2bc388081a0841852ba1ea8843868 Mon Sep 17 00:00:00 2001 From: Nat Henderson Date: Thu, 23 Sep 2021 17:54:33 -0700 Subject: [PATCH] Re-add monitoring monitored project, without metrics scope resource (#5235) * Added monitoring metrics scope and monitored project resources. (#5218) * Added monitoring metrics scope and monitored project resources. * Ran make upgrade-dcl * Fixed indentation * Copied tpgtools go.sum lines into mmv1 go.sum. * Apply suggestions from code review Co-authored-by: Scott Suarez * Attempted to fix strange downstream change in iam.go. * Added missing custom serializer to GA version, upgraded DCL again. * Update ID format so import is possible. Co-authored-by: Scott Suarez Co-authored-by: Nathan Mckinley * Revert reversion of metrics scopes * Remove scope from provider. * Remove from GA, remove metrics scope from provider. * Added missing strings import for mmv1 dataproc cluster test. * Ran make serialize. * Removed unneeded overrides. Re-ran make serialize. * Removed undeletable override. Co-authored-by: Thomas Rodgers Co-authored-by: Scott Suarez --- .../resource_dataproc_cluster_test.go.erb | 1 + mmv1/third_party/terraform/utils/iam.go.erb | 1 + .../terraform/utils/provider.go.erb | 3 +- .../monitoring/beta/monitored_project.yaml | 86 +++++++++++++++++++ .../basic.cloudresourcemanager.project.json | 4 + .../samples/basic.monitored_project.json | 4 + .../samples/basic_monitored_project.yaml | 28 ++++++ tpgtools/override.go | 3 +- tpgtools/override_details.go | 6 ++ .../monitoring/beta/monitored_project.yaml | 17 ++++ .../monitoring/samples/metricsscope/meta.yaml | 3 + .../samples/monitoredproject/meta.yaml | 7 ++ tpgtools/resource.go | 19 +++- tpgtools/serialization.go | 28 +++++- tpgtools/serialization_helpers.go | 1 - tpgtools/templates/resource.go.tmpl | 28 ++++-- tpgtools/templates/serialization.go.tmpl | 10 ++- tpgtools/templates/sweeper.go.tmpl | 6 +- 18 files changed, 234 insertions(+), 21 deletions(-) create mode 100755 tpgtools/api/monitoring/beta/monitored_project.yaml create mode 100755 tpgtools/api/monitoring/samples/basic.cloudresourcemanager.project.json create mode 100755 tpgtools/api/monitoring/samples/basic.monitored_project.json create mode 100755 tpgtools/api/monitoring/samples/basic_monitored_project.yaml create mode 100644 tpgtools/overrides/monitoring/beta/monitored_project.yaml create mode 100644 tpgtools/overrides/monitoring/samples/metricsscope/meta.yaml create mode 100644 tpgtools/overrides/monitoring/samples/monitoredproject/meta.yaml diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb index a2c1ea8e8406..e314a7258f07 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "strconv" + "strings" "testing" "time" diff --git a/mmv1/third_party/terraform/utils/iam.go.erb b/mmv1/third_party/terraform/utils/iam.go.erb index f6ad971b7834..181128798f0a 100644 --- a/mmv1/third_party/terraform/utils/iam.go.erb +++ b/mmv1/third_party/terraform/utils/iam.go.erb @@ -7,6 +7,7 @@ import ( "fmt" "log" "reflect" + "sort" "strings" "time" diff --git a/mmv1/third_party/terraform/utils/provider.go.erb b/mmv1/third_party/terraform/utils/provider.go.erb index 90d1cee31014..ba06f61565dc 100644 --- a/mmv1/third_party/terraform/utils/provider.go.erb +++ b/mmv1/third_party/terraform/utils/provider.go.erb @@ -455,8 +455,9 @@ end # products.each do <% unless version == 'ga' -%> "google_gke_hub_feature": resourceGkeHubFeature(), "google_gke_hub_feature_membership": resourceGkeHubFeatureMembership(), + "google_monitoring_monitored_project": resourceMonitoringMonitoredProject(), <% end -%> - "google_org_policy_policy": resourceOrgPolicyPolicy(), + "google_org_policy_policy": resourceOrgPolicyPolicy(), "google_privateca_certificate_template": resourcePrivatecaCertificateTemplate(), }, // ------------------------------------ diff --git a/tpgtools/api/monitoring/beta/monitored_project.yaml b/tpgtools/api/monitoring/beta/monitored_project.yaml new file mode 100755 index 000000000000..c85c5ac7138d --- /dev/null +++ b/tpgtools/api/monitoring/beta/monitored_project.yaml @@ -0,0 +1,86 @@ +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +info: + title: Monitoring/MonitoredProject + description: DCL Specification for the Monitoring MonitoredProject resource + x-dcl-has-iam: false +paths: + get: + description: The function used to get information about a MonitoredProject + parameters: + - name: MonitoredProject + required: true + description: A full instance of a MonitoredProject + apply: + description: The function used to apply information about a MonitoredProject + parameters: + - name: MonitoredProject + required: true + description: A full instance of a MonitoredProject + delete: + description: The function used to delete a MonitoredProject + parameters: + - name: MonitoredProject + required: true + description: A full instance of a MonitoredProject + deleteAll: + description: The function used to delete all MonitoredProject + parameters: + - name: metricsscope + required: true + schema: + type: string + list: + description: The function used to list information about many MonitoredProject + parameters: + - name: metricsscope + required: true + schema: + type: string +components: + schemas: + MonitoredProject: + title: MonitoredProject + x-dcl-id: locations/global/metricsScopes/{{metrics_scope}}/projects/{{name}} + x-dcl-locations: + - global + type: object + required: + - name + - metricsScope + properties: + createTime: + type: string + format: date-time + x-dcl-go-name: CreateTime + readOnly: true + description: Output only. The time when this `MonitoredProject` was created. + x-kubernetes-immutable: true + metricsScope: + type: string + x-dcl-go-name: MetricsScope + description: 'Required. The resource name of the existing Metrics Scope + that will monitor this project. Example: locations/global/metricsScopes/{SCOPING_PROJECT_ID_OR_NUMBER}' + x-kubernetes-immutable: true + x-dcl-references: + - resource: Monitoring/MetricsScope + field: name + name: + type: string + x-dcl-go-name: Name + description: 'Immutable. The resource name of the `MonitoredProject`. On + input, the resource name includes the scoping project ID and monitored + project ID. On output, it contains the equivalent project numbers. Example: + `locations/global/metricsScopes/{SCOPING_PROJECT_ID_OR_NUMBER}/projects/{MONITORED_PROJECT_ID_OR_NUMBER}`' + x-kubernetes-immutable: true diff --git a/tpgtools/api/monitoring/samples/basic.cloudresourcemanager.project.json b/tpgtools/api/monitoring/samples/basic.cloudresourcemanager.project.json new file mode 100755 index 000000000000..ecb8396416a9 --- /dev/null +++ b/tpgtools/api/monitoring/samples/basic.cloudresourcemanager.project.json @@ -0,0 +1,4 @@ +{ + "name": "{{id}}", + "parent": "organizations/{{org_id}}" +} diff --git a/tpgtools/api/monitoring/samples/basic.monitored_project.json b/tpgtools/api/monitoring/samples/basic.monitored_project.json new file mode 100755 index 000000000000..2ae0a9f4ea78 --- /dev/null +++ b/tpgtools/api/monitoring/samples/basic.monitored_project.json @@ -0,0 +1,4 @@ +{ + "name": "{{ref:basic.cloudresourcemanager.project.json:name}}", + "metricsScope": "{{project}}" +} diff --git a/tpgtools/api/monitoring/samples/basic_monitored_project.yaml b/tpgtools/api/monitoring/samples/basic_monitored_project.yaml new file mode 100755 index 000000000000..13743558c47e --- /dev/null +++ b/tpgtools/api/monitoring/samples/basic_monitored_project.yaml @@ -0,0 +1,28 @@ +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +dependencies: +- samples/basic.cloudresourcemanager.project.json +description: A basic example of a monitoring monitored project +name: basic_monitored_project +resource: samples/basic.monitored_project.json +type: monitored_project +variables: +- name: id + type: resource_name +- name: org_id + type: org_id +- name: project + type: project +versions: +- beta diff --git a/tpgtools/override.go b/tpgtools/override.go index 10b39d7636b0..54fd5846b183 100644 --- a/tpgtools/override.go +++ b/tpgtools/override.go @@ -37,6 +37,7 @@ const ( CustomID = "CUSTOM_ID" CustomizeDiff = "CUSTOMIZE_DIFF" ImportFormat = "IMPORT_FORMAT" + AppendToBasePath = "APPEND_TO_BASE_PATH" Mutex = "MUTEX" PreCreate = "PRE_CREATE_FUNCTION" PostCreate = "POST_CREATE_FUNCTION" @@ -45,7 +46,7 @@ const ( NoSweeper = "NO_SWEEPER" CustomImport = "CUSTOM_IMPORT_FUNCTION" CustomCreateDirective = "CUSTOM_CREATE_DIRECTIVE_FUNCTION" - SkipDelete = "SKIP_DELETE_FUNCTION" + SkipDeleteFunction = "SKIP_DELETE_FUNCTION" SerializationOnly = "SERIALIZATION_ONLY" CustomSerializer = "CUSTOM_SERIALIZER" TerraformProductName = "CUSTOM_TERRAFORM_PRODUCT_NAME" diff --git a/tpgtools/override_details.go b/tpgtools/override_details.go index e6d209bdd5fc..3d456dfa5597 100644 --- a/tpgtools/override_details.go +++ b/tpgtools/override_details.go @@ -66,6 +66,12 @@ type CustomImportFunctionDetails struct { Function string } +type AppendToBasePathDetails struct { + // Append to base path appends this string to the end of the resource's + // base path. + String string +} + type CustomizeDiffDetails struct { // Functions is a list of CustomizeDiffFunc to use with // customdiff.All(...). diff --git a/tpgtools/overrides/monitoring/beta/monitored_project.yaml b/tpgtools/overrides/monitoring/beta/monitored_project.yaml new file mode 100644 index 000000000000..7cf83cf69dcc --- /dev/null +++ b/tpgtools/overrides/monitoring/beta/monitored_project.yaml @@ -0,0 +1,17 @@ +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- type: APPEND_TO_BASE_PATH + details: + string: v1 diff --git a/tpgtools/overrides/monitoring/samples/metricsscope/meta.yaml b/tpgtools/overrides/monitoring/samples/metricsscope/meta.yaml new file mode 100644 index 000000000000..a4fe31c7b534 --- /dev/null +++ b/tpgtools/overrides/monitoring/samples/metricsscope/meta.yaml @@ -0,0 +1,3 @@ +substitutions: + - substitution: "project" + value: ":PROJECT" diff --git a/tpgtools/overrides/monitoring/samples/monitoredproject/meta.yaml b/tpgtools/overrides/monitoring/samples/monitoredproject/meta.yaml new file mode 100644 index 000000000000..700fd218cc00 --- /dev/null +++ b/tpgtools/overrides/monitoring/samples/monitoredproject/meta.yaml @@ -0,0 +1,7 @@ +substitutions: + - substitution: "id" + value: "id" + - substitution: "org_id" + value: ":ORG_ID" + - substitution: "project" + value: ":PROJECT" diff --git a/tpgtools/resource.go b/tpgtools/resource.go index 9bac2c5bcf3f..351419c80b1b 100644 --- a/tpgtools/resource.go +++ b/tpgtools/resource.go @@ -47,6 +47,11 @@ type Resource struct { // TODO: if none are set, the resource does not support import. ImportFormats []string + // AppendToBasePath is a string that will be appended to the end of the API base path. + // rarely needed in cases where the shared mm basepath does not include the version + // as in Montioring https://git.io/Jz4Wn + AppendToBasePath string + // title is the name of the resource in snake_case. For example, // "instance", "backend_service". title string @@ -419,6 +424,16 @@ func createResource(schema *openapi.Schema, typeFetcher *TypeFetcher, overrides res.CustomImportFunction = &cifd.Function } + // Resource Override: Append to Base Path + atbpd := AppendToBasePathDetails{} + atbpOk, err := overrides.ResourceOverrideWithDetails(AppendToBasePath, &atbpd, location) + if err != nil { + return nil, fmt.Errorf("failed to decode append to base path details: %v", err) + } + if atbpOk { + res.AppendToBasePath = atbpd.String + } + // Resource Override: Import formats ifd := ImportFormatDetails{} ifdOk, err := overrides.ResourceOverrideWithDetails(ImportFormat, &ifd, location) @@ -551,9 +566,9 @@ func createResource(schema *openapi.Schema, typeFetcher *TypeFetcher, overrides // Resource Override: SkipDeleteFunction skipDeleteFunc := SkipDeleteFunctionDetails{} - skipDeleteFuncOk, err := overrides.ResourceOverrideWithDetails(SkipDelete, &skipDeleteFunc, location) + skipDeleteFuncOk, err := overrides.ResourceOverrideWithDetails(SkipDeleteFunction, &skipDeleteFunc, location) if err != nil { - return nil, fmt.Errorf("failed to decode skip delete function details: %v", err) + return nil, fmt.Errorf("failed to decode skip delete details: %v", err) } if skipDeleteFuncOk { res.SkipDeleteFunction = &skipDeleteFunc.Function diff --git a/tpgtools/serialization.go b/tpgtools/serialization.go index 166705e0d14b..d18ae47fd7dd 100644 --- a/tpgtools/serialization.go +++ b/tpgtools/serialization.go @@ -33,6 +33,7 @@ import ( eventarc "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/eventarc" eventarcBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/eventarc/beta" gkehubBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub/beta" + monitoringBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/monitoring/beta" orgpolicy "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/orgpolicy" orgpolicyBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/orgpolicy/beta" privateca "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/privateca" @@ -71,6 +72,8 @@ func DCLToTerraformReference(resourceType, version string) (string, error) { return "google_gke_hub_feature", nil case "GkeHubFeatureMembership": return "google_gke_hub_feature_membership", nil + case "MonitoringMonitoredProject": + return "google_monitoring_monitored_project", nil case "OrgPolicyPolicy": return "google_org_policy_policy", nil case "PrivatecaCertificateTemplate": @@ -224,6 +227,12 @@ func ConvertSampleJSONToHCL(resourceType string, version string, b []byte) (stri return "", err } return GkeHubFeatureMembershipBetaAsHCL(*r) + case "MonitoringMonitoredProject": + r := &monitoringBeta.MonitoredProject{} + if err := json.Unmarshal(b, r); err != nil { + return "", err + } + return MonitoringMonitoredProjectBetaAsHCL(*r) case "OrgPolicyPolicy": r := &orgpolicyBeta.Policy{} if err := json.Unmarshal(b, r); err != nil { @@ -257,7 +266,7 @@ func ConvertSampleJSONToHCL(resourceType string, version string, b []byte) (stri if err := json.Unmarshal(b, r); err != nil { return "", err } - return CloudResourceManagerProjectAsHCL(*r) + return serializeGAProjectToHCL(*r) case "ComputeFirewallPolicy": r := &compute.FirewallPolicy{} if err := json.Unmarshal(b, r); err != nil { @@ -2033,6 +2042,23 @@ func convertGkeHubFeatureMembershipBetaConfigmanagementPolicyControllerToHCL(r * return outputConfig + "}" } +// MonitoringMonitoredProjectBetaAsHCL returns a string representation of the specified resource in HCL. +// The generated HCL will include every settable field as a literal - that is, no +// variables, no references. This may not be the best possible representation, but +// the crucial point is that `terraform import; terraform apply` will not produce +// any changes. We do not validate that the resource specified will pass terraform +// validation unless is an object returned from the API after an Apply. +func MonitoringMonitoredProjectBetaAsHCL(r monitoringBeta.MonitoredProject) (string, error) { + outputConfig := "resource \"google_monitoring_monitored_project\" \"output\" {\n" + if r.MetricsScope != nil { + outputConfig += fmt.Sprintf("\tmetrics_scope = %#v\n", *r.MetricsScope) + } + if r.Name != nil { + outputConfig += fmt.Sprintf("\tname = %#v\n", *r.Name) + } + return formatHCL(outputConfig + "}") +} + // OrgPolicyPolicyBetaAsHCL returns a string representation of the specified resource in HCL. // The generated HCL will include every settable field as a literal - that is, no // variables, no references. This may not be the best possible representation, but diff --git a/tpgtools/serialization_helpers.go b/tpgtools/serialization_helpers.go index dd104a5e9f76..34f7cfaf774b 100644 --- a/tpgtools/serialization_helpers.go +++ b/tpgtools/serialization_helpers.go @@ -21,7 +21,6 @@ func serializeBetaProjectToHCL(r cloudresourcemanagerBeta.Project) (string, erro return serializeProjectToHCL(m) } - func serializeGAProjectToHCL(r cloudresourcemanager.Project) (string, error) { b, err := json.Marshal(r) if err != nil { diff --git a/tpgtools/templates/resource.go.tmpl b/tpgtools/templates/resource.go.tmpl index 6a5a1601dfc2..218fbf9fd697 100644 --- a/tpgtools/templates/resource.go.tmpl +++ b/tpgtools/templates/resource.go.tmpl @@ -266,6 +266,9 @@ should be converted to use the DCL's ID method, so normalization can be uniform. billingProject = bp } client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject) + {{- if $.AppendToBasePath }} + client.Config.BasePath += "{{$.AppendToBasePath}}" + {{- end }} res, err := client.Apply{{$.Type}}(context.Background(), obj, createDirective...) if _, ok := err.(dcl.DiffAfterApplyError); ok { @@ -337,6 +340,9 @@ func resource{{$.PathType}}Read(d *schema.ResourceData, meta interface{}) error billingProject = bp } client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject) + {{- if $.AppendToBasePath }} + client.Config.BasePath += "{{$.AppendToBasePath}}" + {{- end }} res, err := client.Get{{$.Type}}(context.Background(), obj) if err != nil { resourceName := fmt.Sprintf("{{$.PathType}} %q", d.Id()) @@ -410,6 +416,9 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro billingProject = bp } client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject) + {{- if $.AppendToBasePath }} + client.Config.BasePath += "{{$.AppendToBasePath}}" + {{- end }} res, err := client.Apply{{$.Type}}(context.Background(), obj, directive...) if _, ok := err.(dcl.DiffAfterApplyError); ok { @@ -462,14 +471,14 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro {{ end }} {{- if $.SkipDeleteFunction }} - skip, err := {{$.SkipDeleteFunction}}(config, obj) - if err != nil { - return fmt.Errorf("error encountered in skip-delete: %v", err) - } - if skip { - return nil - } -{{- end}} + skip, err := {{$.SkipDeleteFunction}}(config, obj) + if err != nil { + return fmt.Errorf("error encountered in skip-delete: %v", err) + } + if skip { + return nil + } +{{- end }} log.Printf("[DEBUG] Deleting {{$.Type}} %q", d.Id()) userAgent, err := generateUserAgentString(d, config.userAgent) @@ -486,6 +495,9 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro billingProject = bp } client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject) + {{- if $.AppendToBasePath }} + client.Config.BasePath += "{{$.AppendToBasePath}}" + {{- end }} if err := client.Delete{{$.Type}}(context.Background(), obj); err != nil { return fmt.Errorf("Error deleting {{$.Type}}: %s", err) } diff --git a/tpgtools/templates/serialization.go.tmpl b/tpgtools/templates/serialization.go.tmpl index 260700234c7b..0dec8ca53bf6 100644 --- a/tpgtools/templates/serialization.go.tmpl +++ b/tpgtools/templates/serialization.go.tmpl @@ -110,14 +110,18 @@ func ConvertSampleJSONToHCL(resourceType string, version string, b []byte) (stri {{- else }} // If not found in sample version, fallthrough to GA switch resourceType { - {{- range $res := $resList }} + {{- range $res := $resList }} case "{{$res.PathType}}": r := &{{$res.Package}}{{$version.SerializationSuffix}}.{{$res.Type}}{} if err := json.Unmarshal(b, r); err != nil { return "", err } - return {{$res.PathType}}{{$version.SerializationSuffix}}AsHCL(*r) - {{- end }} + {{- if $res.CustomSerializer }} + return {{$res.CustomSerializer}}(*r) + {{- else }} + return {{$res.PathType}}{{$version.SerializationSuffix}}AsHCL(*r) + {{- end }} + {{- end }} default: //return fmt.Sprintf("%s resource not supported in tpgtools", resourceType), nil return "", fmt.Errorf("Error converting sample JSON to HCL: %s not found", resourceType) diff --git a/tpgtools/templates/sweeper.go.tmpl b/tpgtools/templates/sweeper.go.tmpl index 01992e80076e..d0d79622d247 100644 --- a/tpgtools/templates/sweeper.go.tmpl +++ b/tpgtools/templates/sweeper.go.tmpl @@ -33,9 +33,8 @@ import( "log" "testing" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" {{$.Package}} "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/{{$.DCLPackage}}" - ) func init() { @@ -46,8 +45,7 @@ func init() { } func testSweep{{$.SweeperName}}(region string) error { - resourceName := "{{ $.SweeperName}}" - log.Printf("[INFO][SWEEPER_LOG] Starting sweeper for %s",resourceName) + log.Print("[INFO][SWEEPER_LOG] Starting sweeper for {{$.SweeperName}}") config, err := sharedConfigForRegion(region) if err != nil {