Re-add monitoring monitored project, without metrics scope resource (#…

…5235)

* Added monitoring metrics scope and monitored project resources. (#5218)

* Added monitoring metrics scope and monitored project resources.

* Ran make upgrade-dcl

* Fixed indentation

* Copied tpgtools go.sum lines into mmv1 go.sum.

* Apply suggestions from code review

Co-authored-by: Scott Suarez <[email protected]>

* Attempted to fix strange downstream change in iam.go.

* Added missing custom serializer to GA version, upgraded DCL again.

* Update ID format so import is possible.

Co-authored-by: Scott Suarez <[email protected]>
Co-authored-by: Nathan Mckinley <[email protected]>

* Revert reversion of metrics scopes

* Remove scope from provider.

* Remove from GA, remove metrics scope from provider.

* Added missing strings import for mmv1 dataproc cluster test.

* Ran make serialize.

* Removed unneeded overrides. Re-ran make serialize.

* Removed undeletable override.

Co-authored-by: Thomas Rodgers <[email protected]>
Co-authored-by: Scott Suarez <[email protected]>

mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 

mmv1/third_party/terraform/utils/iam.go.erb

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"reflect"
+	"sort"
 	"strings"
 	"time"
 

mmv1/third_party/terraform/utils/provider.go.erb

@@ -455,8 +455,9 @@ end     # products.each do
 				<% unless version == 'ga' -%>
 				"google_gke_hub_feature":                       resourceGkeHubFeature(),
 				"google_gke_hub_feature_membership":            resourceGkeHubFeatureMembership(),
+				"google_monitoring_monitored_project":          resourceMonitoringMonitoredProject(),
 				<% end -%>
-                                "google_org_policy_policy":                      resourceOrgPolicyPolicy(),
+				"google_org_policy_policy":             resourceOrgPolicyPolicy(),
 				"google_privateca_certificate_template":        resourcePrivatecaCertificateTemplate(),
 			},
 			// ------------------------------------

tpgtools/api/monitoring/beta/monitored_project.yaml

@@ -0,0 +1,86 @@
+# Copyright 2021 Google LLC. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+info:
+  title: Monitoring/MonitoredProject
+  description: DCL Specification for the Monitoring MonitoredProject resource
+  x-dcl-has-iam: false
+paths:
+  get:
+    description: The function used to get information about a MonitoredProject
+    parameters:
+    - name: MonitoredProject
+      required: true
+      description: A full instance of a MonitoredProject
+  apply:
+    description: The function used to apply information about a MonitoredProject
+    parameters:
+    - name: MonitoredProject
+      required: true
+      description: A full instance of a MonitoredProject
+  delete:
+    description: The function used to delete a MonitoredProject
+    parameters:
+    - name: MonitoredProject
+      required: true
+      description: A full instance of a MonitoredProject
+  deleteAll:
+    description: The function used to delete all MonitoredProject
+    parameters:
+    - name: metricsscope
+      required: true
+      schema:
+        type: string
+  list:
+    description: The function used to list information about many MonitoredProject
+    parameters:
+    - name: metricsscope
+      required: true
+      schema:
+        type: string
+components:
+  schemas:
+    MonitoredProject:
+      title: MonitoredProject
+      x-dcl-id: locations/global/metricsScopes/{{metrics_scope}}/projects/{{name}}
+      x-dcl-locations:
+      - global
+      type: object
+      required:
+      - name
+      - metricsScope
+      properties:
+        createTime:
+          type: string
+          format: date-time
+          x-dcl-go-name: CreateTime
+          readOnly: true
+          description: Output only. The time when this `MonitoredProject` was created.
+          x-kubernetes-immutable: true
+        metricsScope:
+          type: string
+          x-dcl-go-name: MetricsScope
+          description: 'Required. The resource name of the existing Metrics Scope
+            that will monitor this project. Example: locations/global/metricsScopes/{SCOPING_PROJECT_ID_OR_NUMBER}'
+          x-kubernetes-immutable: true
+          x-dcl-references:
+          - resource: Monitoring/MetricsScope
+            field: name
+        name:
+          type: string
+          x-dcl-go-name: Name
+          description: 'Immutable. The resource name of the `MonitoredProject`. On
+            input, the resource name includes the scoping project ID and monitored
+            project ID. On output, it contains the equivalent project numbers. Example:
+            `locations/global/metricsScopes/{SCOPING_PROJECT_ID_OR_NUMBER}/projects/{MONITORED_PROJECT_ID_OR_NUMBER}`'
+          x-kubernetes-immutable: true

tpgtools/api/monitoring/samples/basic.cloudresourcemanager.project.json

@@ -0,0 +1,4 @@
+{
+  "name": "{{id}}",
+  "parent": "organizations/{{org_id}}"
+}

tpgtools/api/monitoring/samples/basic.monitored_project.json

@@ -0,0 +1,4 @@
+{
+  "name": "{{ref:basic.cloudresourcemanager.project.json:name}}",
+  "metricsScope": "{{project}}"
+}

tpgtools/api/monitoring/samples/basic_monitored_project.yaml

@@ -0,0 +1,28 @@
+# Copyright 2021 Google LLC. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+dependencies:
+- samples/basic.cloudresourcemanager.project.json
+description: A basic example of a monitoring monitored project
+name: basic_monitored_project
+resource: samples/basic.monitored_project.json
+type: monitored_project
+variables:
+- name: id
+  type: resource_name
+- name: org_id
+  type: org_id
+- name: project
+  type: project
+versions:
+- beta

tpgtools/override.go

@@ -37,6 +37,7 @@ const (
 	CustomID                           = "CUSTOM_ID"
 	CustomizeDiff                      = "CUSTOMIZE_DIFF"
 	ImportFormat                       = "IMPORT_FORMAT"
+	AppendToBasePath                   = "APPEND_TO_BASE_PATH"
 	Mutex                              = "MUTEX"
 	PreCreate                          = "PRE_CREATE_FUNCTION"
 	PostCreate                         = "POST_CREATE_FUNCTION"
@@ -45,7 +46,7 @@ const (
 	NoSweeper                          = "NO_SWEEPER"
 	CustomImport                       = "CUSTOM_IMPORT_FUNCTION"
 	CustomCreateDirective              = "CUSTOM_CREATE_DIRECTIVE_FUNCTION"
-	SkipDelete                         = "SKIP_DELETE_FUNCTION"
+	SkipDeleteFunction                 = "SKIP_DELETE_FUNCTION"
 	SerializationOnly                  = "SERIALIZATION_ONLY"
 	CustomSerializer                   = "CUSTOM_SERIALIZER"
 	TerraformProductName               = "CUSTOM_TERRAFORM_PRODUCT_NAME"

tpgtools/override_details.go

@@ -66,6 +66,12 @@ type CustomImportFunctionDetails struct {
 	Function string
 }
 
+type AppendToBasePathDetails struct {
+	// Append to base path appends this string to the end of the resource's
+	// base path.
+	String string
+}
+
 type CustomizeDiffDetails struct {
 	// Functions is a list of CustomizeDiffFunc to use with
 	// customdiff.All(...).

tpgtools/overrides/monitoring/beta/monitored_project.yaml

@@ -0,0 +1,17 @@
+# Copyright 2021 Google LLC. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- type: APPEND_TO_BASE_PATH
+  details:
+    string: v1

tpgtools/overrides/monitoring/samples/metricsscope/meta.yaml

@@ -0,0 +1,3 @@
+substitutions:
+  - substitution: "project"
+    value: ":PROJECT"

tpgtools/overrides/monitoring/samples/monitoredproject/meta.yaml

@@ -0,0 +1,7 @@
+substitutions:
+  - substitution: "id"
+    value: "id"
+  - substitution: "org_id"
+    value: ":ORG_ID"
+  - substitution: "project"
+    value: ":PROJECT"

tpgtools/resource.go

@@ -47,6 +47,11 @@ type Resource struct {
 	// TODO: if none are set, the resource does not support import.
 	ImportFormats []string
 
+	// AppendToBasePath is a string that will be appended to the end of the API base path.
+	// rarely needed in cases where the shared mm basepath does not include the version
+	// as in Montioring https://git.io/Jz4Wn
+	AppendToBasePath string
+
 	// title is the name of the resource in snake_case. For example,
 	// "instance", "backend_service".
 	title string
@@ -419,6 +424,16 @@ func createResource(schema *openapi.Schema, typeFetcher *TypeFetcher, overrides
 		res.CustomImportFunction = &cifd.Function
 	}
 
+	// Resource Override: Append to Base Path
+	atbpd := AppendToBasePathDetails{}
+	atbpOk, err := overrides.ResourceOverrideWithDetails(AppendToBasePath, &atbpd, location)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode append to base path details: %v", err)
+	}
+	if atbpOk {
+		res.AppendToBasePath = atbpd.String
+	}
+
 	// Resource Override: Import formats
 	ifd := ImportFormatDetails{}
 	ifdOk, err := overrides.ResourceOverrideWithDetails(ImportFormat, &ifd, location)
@@ -551,9 +566,9 @@ func createResource(schema *openapi.Schema, typeFetcher *TypeFetcher, overrides
 
 	// Resource Override: SkipDeleteFunction
 	skipDeleteFunc := SkipDeleteFunctionDetails{}
-	skipDeleteFuncOk, err := overrides.ResourceOverrideWithDetails(SkipDelete, &skipDeleteFunc, location)
+	skipDeleteFuncOk, err := overrides.ResourceOverrideWithDetails(SkipDeleteFunction, &skipDeleteFunc, location)
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode skip delete function details: %v", err)
+		return nil, fmt.Errorf("failed to decode skip delete details: %v", err)
 	}
 	if skipDeleteFuncOk {
 		res.SkipDeleteFunction = &skipDeleteFunc.Function

tpgtools/serialization.go

@@ -33,6 +33,7 @@ import (
 	eventarc "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/eventarc"
 	eventarcBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/eventarc/beta"
 	gkehubBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/gkehub/beta"
+	monitoringBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/monitoring/beta"
 	orgpolicy "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/orgpolicy"
 	orgpolicyBeta "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/orgpolicy/beta"
 	privateca "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/privateca"
@@ -71,6 +72,8 @@ func DCLToTerraformReference(resourceType, version string) (string, error) {
 			return "google_gke_hub_feature", nil
 		case "GkeHubFeatureMembership":
 			return "google_gke_hub_feature_membership", nil
+		case "MonitoringMonitoredProject":
+			return "google_monitoring_monitored_project", nil
 		case "OrgPolicyPolicy":
 			return "google_org_policy_policy", nil
 		case "PrivatecaCertificateTemplate":
@@ -224,6 +227,12 @@ func ConvertSampleJSONToHCL(resourceType string, version string, b []byte) (stri
 				return "", err
 			}
 			return GkeHubFeatureMembershipBetaAsHCL(*r)
+		case "MonitoringMonitoredProject":
+			r := &monitoringBeta.MonitoredProject{}
+			if err := json.Unmarshal(b, r); err != nil {
+				return "", err
+			}
+			return MonitoringMonitoredProjectBetaAsHCL(*r)
 		case "OrgPolicyPolicy":
 			r := &orgpolicyBeta.Policy{}
 			if err := json.Unmarshal(b, r); err != nil {
@@ -257,7 +266,7 @@ func ConvertSampleJSONToHCL(resourceType string, version string, b []byte) (stri
 		if err := json.Unmarshal(b, r); err != nil {
 			return "", err
 		}
-		return CloudResourceManagerProjectAsHCL(*r)
+		return serializeGAProjectToHCL(*r)
 	case "ComputeFirewallPolicy":
 		r := &compute.FirewallPolicy{}
 		if err := json.Unmarshal(b, r); err != nil {
@@ -2033,6 +2042,23 @@ func convertGkeHubFeatureMembershipBetaConfigmanagementPolicyControllerToHCL(r *
 	return outputConfig + "}"
 }
 
+// MonitoringMonitoredProjectBetaAsHCL returns a string representation of the specified resource in HCL.
+// The generated HCL will include every settable field as a literal - that is, no
+// variables, no references.  This may not be the best possible representation, but
+// the crucial point is that `terraform import; terraform apply` will not produce
+// any changes.  We do not validate that the resource specified will pass terraform
+// validation unless is an object returned from the API after an Apply.
+func MonitoringMonitoredProjectBetaAsHCL(r monitoringBeta.MonitoredProject) (string, error) {
+	outputConfig := "resource \"google_monitoring_monitored_project\" \"output\" {\n"
+	if r.MetricsScope != nil {
+		outputConfig += fmt.Sprintf("\tmetrics_scope = %#v\n", *r.MetricsScope)
+	}
+	if r.Name != nil {
+		outputConfig += fmt.Sprintf("\tname = %#v\n", *r.Name)
+	}
+	return formatHCL(outputConfig + "}")
+}
+
 // OrgPolicyPolicyBetaAsHCL returns a string representation of the specified resource in HCL.
 // The generated HCL will include every settable field as a literal - that is, no
 // variables, no references.  This may not be the best possible representation, but

tpgtools/serialization_helpers.go

@@ -21,7 +21,6 @@ func serializeBetaProjectToHCL(r cloudresourcemanagerBeta.Project) (string, erro
 	return serializeProjectToHCL(m)
 }
 
-
 func serializeGAProjectToHCL(r cloudresourcemanager.Project) (string, error) {
 	b, err := json.Marshal(r)
 	if err != nil {

tpgtools/templates/resource.go.tmpl

@@ -266,6 +266,9 @@ should be converted to use the DCL's ID method, so normalization can be uniform.
 		billingProject = bp
 	}
 	client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject)
+	{{- if $.AppendToBasePath }}
+	client.Config.BasePath += "{{$.AppendToBasePath}}"
+	{{- end }}
 	res, err := client.Apply{{$.Type}}(context.Background(), obj, createDirective...)
 
 	if _, ok := err.(dcl.DiffAfterApplyError); ok {
@@ -337,6 +340,9 @@ func resource{{$.PathType}}Read(d *schema.ResourceData, meta interface{}) error
 		billingProject = bp
 	}
 	client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject)
+	{{- if $.AppendToBasePath }}
+	client.Config.BasePath += "{{$.AppendToBasePath}}"
+	{{- end }}
 	res, err := client.Get{{$.Type}}(context.Background(), obj)
 	if err != nil {
 		resourceName := fmt.Sprintf("{{$.PathType}} %q", d.Id())
@@ -410,6 +416,9 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro
 		billingProject = bp
 	}
 	client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject)
+	{{- if $.AppendToBasePath }}
+	client.Config.BasePath += "{{$.AppendToBasePath}}"
+	{{- end }}
 	res, err := client.Apply{{$.Type}}(context.Background(), obj, directive...)
 
 	if _, ok := err.(dcl.DiffAfterApplyError); ok {
@@ -462,14 +471,14 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro
 {{ end }}
 
 {{- if $.SkipDeleteFunction }}
-  skip, err := {{$.SkipDeleteFunction}}(config, obj)
-  if err != nil {
-    return fmt.Errorf("error encountered in skip-delete: %v", err)
-  }
-  if skip {
-    return nil
-  }
-{{- end}}
+	skip, err := {{$.SkipDeleteFunction}}(config, obj)
+	if err != nil {
+		return fmt.Errorf("error encountered in skip-delete: %v", err)
+	}
+	if skip {
+		return nil
+	}
+{{- end }}
 
 	log.Printf("[DEBUG] Deleting {{$.Type}} %q", d.Id())
 	userAgent, err := generateUserAgentString(d, config.userAgent)
@@ -486,6 +495,9 @@ func resource{{$.PathType}}Delete(d *schema.ResourceData, meta interface{}) erro
 		billingProject = bp
 	}
 	client := NewDCL{{$.ProductType}}Client(config, userAgent, billingProject)
+	{{- if $.AppendToBasePath }}
+	client.Config.BasePath += "{{$.AppendToBasePath}}"
+	{{- end }}
 	if err := client.Delete{{$.Type}}(context.Background(), obj); err != nil {
 		return fmt.Errorf("Error deleting {{$.Type}}: %s", err)
 	}