From 616cb8960c1b6326ed529a0b15ddaf3b9e14b545 Mon Sep 17 00:00:00 2001 From: William Yardley Date: Wed, 28 Aug 2024 20:21:21 -0700 Subject: [PATCH] container: Add `node_kublet_config` support for autopilot clusters Add support for `node_kubelet_config` in `node_pool_auto_config`. See: https://github.com/hashicorp/terraform-provider-google/issues/15208#issuecomment-2299669810 Per: https://pkg.go.dev/google.golang.org/api/container/v1#NodePoolAutoConfig Currently only `insecure_kubelet_readonly_port_enabled` can be set here. Fixes https://github.com/hashicorp/terraform-provider-google/issues/19236 Fixes https://github.com/hashicorp/terraform-provider-google/issues/19153 --- .../services/container/node_config.go.erb | 69 ++++++++-------- .../resource_container_cluster.go.erb | 26 ++++++ .../resource_container_cluster_test.go.erb | 81 +++++++++++++++++++ .../docs/r/container_cluster.html.markdown | 3 + 4 files changed, 146 insertions(+), 33 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.erb b/mmv1/third_party/terraform/services/container/node_config.go.erb index 14ec2f469a44..d5fcc2534d2c 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.erb +++ b/mmv1/third_party/terraform/services/container/node_config.go.erb @@ -120,6 +120,41 @@ func schemaGcfsConfig(forceNew bool) *schema.Schema { } } +func schemaKubeletConfig() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Description: `Node kubelet configs.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "cpu_manager_policy": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"static", "none", ""}, false), + Description: `Control the CPU management policy on the node.`, + }, + "cpu_cfs_quota": { + Type: schema.TypeBool, + Optional: true, + Description: `Enable CPU CFS quota enforcement for containers that specify CPU limits.`, + }, + "cpu_cfs_quota_period": { + Type: schema.TypeString, + Optional: true, + Description: `Set the CPU CFS quota period value 'cpu.cfs_period_us'.`, + }, + "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), + "pod_pids_limit": { + Type: schema.TypeInt, + Optional: true, + Description: `Controls the maximum number of processes allowed to run in a pod.`, + }, + }, + }, + } +} + func schemaNodeConfig() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, @@ -586,39 +621,7 @@ func schemaNodeConfig() *schema.Schema { }, // Note that AtLeastOneOf can't be set because this schema is reused by // two different resources. - "kubelet_config": { - Type: schema.TypeList, - Optional: true, - MaxItems: 1, - Description: `Node kubelet configs.`, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "cpu_manager_policy": { - Type: schema.TypeString, - Required: true, - ValidateFunc: validation.StringInSlice([]string{"static", "none", ""}, false), - Description: `Control the CPU management policy on the node.`, - }, - "cpu_cfs_quota": { - Type: schema.TypeBool, - Optional: true, - Description: `Enable CPU CFS quota enforcement for containers that specify CPU limits.`, - }, - "cpu_cfs_quota_period": { - Type: schema.TypeString, - Optional: true, - Description: `Set the CPU CFS quota period value 'cpu.cfs_period_us'.`, - }, - "insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(), - "pod_pids_limit": { - Type: schema.TypeInt, - Optional: true, - Description: `Controls the maximum number of processes allowed to run in a pod.`, - }, - }, - }, - }, - + "kubelet_config": schemaKubeletConfig(), "linux_node_config": { Type: schema.TypeList, Optional: true, diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb index 9a2ce54a0d49..00cb9e5713bf 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb @@ -1505,6 +1505,7 @@ func ResourceContainerCluster() *schema.Resource { Description: `Node pool configs that apply to all auto-provisioned node pools in autopilot clusters and node auto-provisioning enabled clusters.`, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ + "node_kubelet_config": schemaKubeletConfig(), "network_tags": { Type: schema.TypeList, Optional: true, @@ -4403,6 +4404,24 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er } } + if d.HasChange("node_pool_auto_config.0.node_kubelet_config") { + req := &container.UpdateClusterRequest{ + Update: &container.ClusterUpdate{ + DesiredNodePoolAutoConfigKubeletConfig: expandKubeletConfig( + d.Get("node_pool_auto_config.0.node_kubelet_config"), + ), + }, + } + + updateF := updateFunc(req, "updating GKE cluster node pool auto config node_kubelet_config parameters") + // Call update serially. + if err := transport_tpg.LockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s node pool auto config node_kubelet_config parameters have been updated", d.Id()) + } + if d.HasChange("node_pool_auto_config.0.network_tags.0.tags") { tags := d.Get("node_pool_auto_config.0.network_tags.0.tags").([]interface{}) @@ -5737,6 +5756,10 @@ func expandNodePoolAutoConfig(configured interface{}) *container.NodePoolAutoCon npac := &container.NodePoolAutoConfig{} config := l[0].(map[string]interface{}) + if v, ok := config["node_kubelet_config"]; ok { + npac.NodeKubeletConfig = expandKubeletConfig(v) + } + if v, ok := config["network_tags"]; ok && len(v.([]interface{})) > 0 { npac.NetworkTags = expandNodePoolAutoConfigNetworkTags(v) } @@ -6575,6 +6598,9 @@ func flattenNodePoolAutoConfig(c *container.NodePoolAutoConfig) []map[string]int } result := make(map[string]interface{}) + if c.NodeKubeletConfig != nil { + result["node_kubelet_config"] = flattenKubeletConfig(c.NodeKubeletConfig) + } if c.NetworkTags != nil { result["network_tags"] = flattenNodePoolAutoConfigNetworkTags(c.NetworkTags) } diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb index b5621ab74e7c..e0bfcaee9203 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb @@ -3248,6 +3248,52 @@ func TestAccContainerCluster_withAutopilotNetworkTags(t *testing.T) { }) } +func TestAccContainerCluster_withAutopilotKubeletConfig(t *testing.T) { + t.Parallel() + + randomSuffix := acctest.RandString(t, 10) + clusterName := fmt.Sprintf("tf-test-cluster-%s", randomSuffix) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + ExternalProviders: map[string]resource.ExternalProvider{ + "time": {}, + }, + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withAutopilotKubeletConfigBaseline(clusterName), + }, + { + ResourceName: "google_container_cluster.with_autopilot_kubelet_config", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + { + Config: testAccContainerCluster_withAutopilotKubeletConfigUpdates(clusterName, "FALSE"), + }, + { + ResourceName: "google_container_cluster.with_autopilot_kubelet_config", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + { + Config: testAccContainerCluster_withAutopilotKubeletConfigUpdates(clusterName, "TRUE"), + }, + { + ResourceName: "google_container_cluster.with_autopilot_kubelet_config", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"}, + }, + }, + }) +} + + func TestAccContainerCluster_withAutopilotResourceManagerTags(t *testing.T) { t.Parallel() @@ -10517,6 +10563,41 @@ func testAccContainerCluster_withWorkloadALTSConfigAutopilot(projectID, name str <% end -%> +func testAccContainerCluster_withAutopilotKubeletConfigBaseline(name string) string { + return fmt.Sprintf(` + resource "google_container_cluster" "with_autopilot_kubelet_config" { + name = "%s" + location = "us-central1" + initial_node_count = 1 + enable_autopilot = true + deletion_protection = false + } +`, name) +} + +func testAccContainerCluster_withAutopilotKubeletConfigUpdates(name, insecureKubeletReadonlyPortEnabled string) string { + return fmt.Sprintf(` + resource "google_container_cluster" "with_autopilot_kubelet_config" { + name = "%s" + location = "us-central1" + initial_node_count = 1 + + node_pool_auto_config { + node_kubelet_config { + # Needed to work around being required but not actually needed + cpu_manager_policy = "" + + # Currently the only supported parameter here + insecure_kubelet_readonly_port_enabled = "%s" + } + } + + enable_autopilot = true + deletion_protection = false + } +`, name, insecureKubeletReadonlyPortEnabled) +} + func testAccContainerCluster_resourceManagerTags(projectID, clusterName, networkName, subnetworkName, randomSuffix string) string { return fmt.Sprintf(` data "google_project" "project" { diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index c395f44b87f3..c6c6f79afa23 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -1084,6 +1084,9 @@ workload_identity_config { The `node_pool_auto_config` block supports: +* `node_kubelet_config` - (Optional) Kubelet configuration. Currently, only `insecure_kubelet_readonly_port_enabled` is supported here. +Structure is [documented below](#nested_kubelet_config). + * `resource_manager_tags` - (Optional) A map of resource manager tag keys and values to be attached to the nodes for managing Compute Engine firewalls using Network Firewall Policies. Tags must be according to specifications found [here](https://cloud.google.com/vpc/docs/tags-firewalls-overview#specifications). A maximum of 5 tag key-value pairs can be specified. Existing tags will be replaced with new values. Tags must be in one of the following formats ([KEY]=[VALUE]) 1. `tagKeys/{tag_key_id}=tagValues/{tag_value_id}` 2. `{org_id}/{tag_key_name}={tag_value_name}` 3. `{project_id}/{tag_key_name}={tag_value_name}`. * `network_tags` (Optional) - The network tag config for the cluster's automatically provisioned node pools.