azurerm_kubernetes_cluster - support for the kube_proxy property (#…

…19567) * `azurerm_kubernetes_cluster` - `kube_proxy` * fmt * update * fix testcase * update * add note on when kube_proxy can be disabled * fmt * add preview note Co-authored-by: Steph <[email protected]>
hashicorp · Jan 15, 2023 · f212e5b · f212e5b
1 parent faba3a5
commit f212e5b
Show file tree

Hide file tree

Showing 3 changed files with 255 additions and 1 deletion.
diff --git a/internal/services/containers/kubernetes_cluster_network_resource_test.go b/internal/services/containers/kubernetes_cluster_network_resource_test.go
@@ -794,6 +794,28 @@ func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) {
 	})
 }
 
+func TestAccKubernetesCluster_kubeProxy(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
+	r := KubernetesClusterResource{}
+
+	data.ResourceTest(t, r, []acceptance.TestStep{
+		{
+			Config: r.kubeProxy(data),
+			Check: acceptance.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+			),
+		},
+		data.ImportStep(),
+		{
+			Config: r.kubeProxyDisabled(data),
+			Check: acceptance.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+			),
+		},
+		data.ImportStep(),
+	})
+}
+
 func (KubernetesClusterResource) apiServerInBYOSubnet(data acceptance.TestData) string {
 	return fmt.Sprintf(`
 provider "azurerm" {
@@ -3387,3 +3409,72 @@ resource "azurerm_kubernetes_cluster" "test" {
 }
 `, "westcentralus", data.RandomInteger)
 }
+
+func (KubernetesClusterResource) kubeProxy(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+ provider "azurerm" {
+   features {}
+ }
+ resource "azurerm_resource_group" "test" {
+   name     = "acctestRG-aks-%d"
+   location = "%s"
+ }
+ resource "azurerm_kubernetes_cluster" "test" {
+   name                = "acctestaks%d"
+   location            = azurerm_resource_group.test.location
+   resource_group_name = azurerm_resource_group.test.name
+   dns_prefix          = "acctestaks%d"
+   default_node_pool {
+     name       = "default"
+     node_count = 1
+     vm_size    = "Standard_D2s_v3"
+   }
+   identity {
+     type = "SystemAssigned"
+   }
+
+   network_profile {
+     network_plugin = "none"
+     kube_proxy {
+       mode = "IPVS"
+       ipvs {
+         scheduler                  = "LeastConnection"
+         tcp_fin_timeout_in_seconds = 1000
+         tcp_timeout_in_seconds     = 1000
+         udp_timeout_in_seconds     = 1000
+       }
+     }
+   }
+ }
+  `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger)
+}
+
+func (KubernetesClusterResource) kubeProxyDisabled(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+ provider "azurerm" {
+   features {}
+ }
+ resource "azurerm_resource_group" "test" {
+   name     = "acctestRG-aks-%d"
+   location = "%s"
+ }
+ resource "azurerm_kubernetes_cluster" "test" {
+   name                = "acctestaks%d"
+   location            = azurerm_resource_group.test.location
+   resource_group_name = azurerm_resource_group.test.name
+   dns_prefix          = "acctestaks%d"
+   default_node_pool {
+     name       = "default"
+     node_count = 1
+     vm_size    = "Standard_D2s_v3"
+   }
+   identity {
+     type = "SystemAssigned"
+   }
+
+   network_profile {
+     network_plugin = "none"
+   }
+ }
+  `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger)
+}
diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go
@@ -864,6 +864,60 @@ func resourceKubernetesCluster() *pluginsdk.Resource {
 							}, false),
 						},
 
+						"kube_proxy": {
+							Type:     pluginsdk.TypeList,
+							MaxItems: 1,
+							Optional: true,
+							Elem: &pluginsdk.Resource{
+								Schema: map[string]*pluginsdk.Schema{
+									"mode": {
+										Type:     pluginsdk.TypeString,
+										Required: true,
+										ValidateFunc: validation.StringInSlice([]string{
+											string(managedclusters.ModeIPVS),
+											string(managedclusters.ModeIPTABLES),
+										}, false),
+									},
+
+									"ipvs": {
+										Type:     pluginsdk.TypeList,
+										MaxItems: 1,
+										Optional: true,
+										Elem: &pluginsdk.Resource{
+											Schema: map[string]*pluginsdk.Schema{
+												"scheduler": {
+													Type:     pluginsdk.TypeString,
+													Optional: true,
+													ValidateFunc: validation.StringInSlice([]string{
+														string(managedclusters.IPvsSchedulerLeastConnection),
+														string(managedclusters.IPvsSchedulerRoundRobin),
+													}, false),
+												},
+
+												"tcp_fin_timeout_in_seconds": {
+													Type:         pluginsdk.TypeInt,
+													Optional:     true,
+													ValidateFunc: validation.IntAtLeast(0),
+												},
+
+												"tcp_timeout_in_seconds": {
+													Type:         pluginsdk.TypeInt,
+													Optional:     true,
+													ValidateFunc: validation.IntAtLeast(0),
+												},
+
+												"udp_timeout_in_seconds": {
+													Type:         pluginsdk.TypeInt,
+													Optional:     true,
+													ValidateFunc: validation.IntAtLeast(0),
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+
 						"load_balancer_profile": {
 							Type:     pluginsdk.TypeList,
 							MaxItems: 1,
@@ -1764,6 +1818,10 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{}
 
 			existing.Model.Properties.NetworkProfile.NatGatewayProfile = &natGatewayProfile
 		}
+
+		if key := "network_profile.0.kube_proxy"; d.HasChange(key) {
+			existing.Model.Properties.NetworkProfile.KubeProxyConfig = expandKubernetesClusterKubeProxyConfig(d.Get(key).([]interface{}))
+		}
 	}
 
 	if d.HasChange("tags") {
@@ -2671,6 +2729,10 @@ func expandKubernetesClusterNetworkProfile(input []interface{}) (*managedcluster
 		IPFamilies:      ipVersions,
 	}
 
+	if kubeProxyRaw := config["kube_proxy"].([]interface{}); len(kubeProxyRaw) != 0 {
+		networkProfile.KubeProxyConfig = expandKubernetesClusterKubeProxyConfig(kubeProxyRaw)
+	}
+
 	if ebpfDataPlane := config["ebpf_data_plane"].(string); ebpfDataPlane != "" {
 		networkProfile.EbpfDataplane = utils.ToPtr(managedclusters.EbpfDataplane(ebpfDataPlane))
 	}
@@ -2964,6 +3026,8 @@ func flattenKubernetesClusterNetworkProfile(profile *managedclusters.ContainerSe
 		}
 	}
 
+	kubeProxy := flattenKubernetesClusterKubeProxyConfig(profile.KubeProxyConfig)
+
 	networkPluginMode := ""
 	if profile.NetworkPluginMode != nil {
 		// The returned value has inconsistent casing
@@ -2977,11 +3041,13 @@ func flattenKubernetesClusterNetworkProfile(profile *managedclusters.ContainerSe
 	if profile.EbpfDataplane != nil {
 		ebpfDataPlane = string(*profile.EbpfDataplane)
 	}
+
 	return []interface{}{
 		map[string]interface{}{
 			"dns_service_ip":        dnsServiceIP,
 			"docker_bridge_cidr":    dockerBridgeCidr,
 			"ebpf_data_plane":       ebpfDataPlane,
+			"kube_proxy":            kubeProxy,
 			"load_balancer_sku":     string(*sku),
 			"load_balancer_profile": lbProfiles,
 			"nat_gateway_profile":   ngwProfiles,
@@ -3665,6 +3731,49 @@ func flattenKubernetesClusterIngressProfile(input *managedclusters.ManagedCluste
 	}
 }
 
+func expandKubernetesClusterKubeProxyConfig(input []interface{}) *managedclusters.ContainerServiceNetworkProfileKubeProxyConfig {
+	if len(input) == 0 {
+		return &managedclusters.ContainerServiceNetworkProfileKubeProxyConfig{
+			Enabled: utils.Bool(false),
+		}
+	}
+	config := input[0].(map[string]interface{})
+	return &managedclusters.ContainerServiceNetworkProfileKubeProxyConfig{
+		Enabled:    utils.Bool(true),
+		IPvsConfig: expandKubernetesClusterKubeProxyIPvsConfig(config["ipvs"].([]interface{})),
+		Mode:       utils.ToPtr(managedclusters.Mode(config["mode"].(string))),
+	}
+}
+
+func expandKubernetesClusterKubeProxyIPvsConfig(input []interface{}) *managedclusters.ContainerServiceNetworkProfileKubeProxyConfigIPvsConfig {
+	if len(input) == 0 || input[0] == nil {
+		return nil
+	}
+	config := input[0].(map[string]interface{})
+	return &managedclusters.ContainerServiceNetworkProfileKubeProxyConfigIPvsConfig{
+		Scheduler:            utils.ToPtr(managedclusters.IPvsScheduler(config["scheduler"].(string))),
+		TcpFinTimeoutSeconds: utils.Int64(int64(config["tcp_fin_timeout_in_seconds"].(int))),
+		TcpTimeoutSeconds:    utils.Int64(int64(config["tcp_timeout_in_seconds"].(int))),
+		UdpTimeoutSeconds:    utils.Int64(int64(config["udp_timeout_in_seconds"].(int))),
+	}
+}
+
+func flattenKubernetesClusterKubeProxyConfig(input *managedclusters.ContainerServiceNetworkProfileKubeProxyConfig) []interface{} {
+	if input == nil || input.Enabled == nil || !*input.Enabled {
+		return []interface{}{}
+	}
+	mode := ""
+	if input.Mode != nil {
+		mode = string(*input.Mode)
+	}
+	return []interface{}{
+		map[string]interface{}{
+			"mode": mode,
+			"ipvs": flattenKubernetesClusterKubeProxyIPvsConfig(input.IPvsConfig),
+		},
+	}
+}
+
 func expandKubernetesClusterAzureMonitorProfile(input []interface{}) *managedclusters.ManagedClusterAzureMonitorProfile {
 	if len(input) == 0 {
 		return &managedclusters.ManagedClusterAzureMonitorProfile{
@@ -3692,6 +3801,34 @@ func expandKubernetesClusterAzureMonitorProfile(input []interface{}) *managedclu
 	}
 }
 
+func flattenKubernetesClusterKubeProxyIPvsConfig(input *managedclusters.ContainerServiceNetworkProfileKubeProxyConfigIPvsConfig) []interface{} {
+	if input == nil {
+		return []interface{}{}
+	}
+	scheduler := ""
+	if input.Scheduler != nil {
+		scheduler = string(*input.Scheduler)
+	}
+	var tcpFinTimeout, tcpTimeout, udpTimeout int64
+	if input.TcpFinTimeoutSeconds != nil {
+		tcpFinTimeout = *input.TcpFinTimeoutSeconds
+	}
+	if input.TcpTimeoutSeconds != nil {
+		tcpTimeout = *input.TcpTimeoutSeconds
+	}
+	if input.UdpTimeoutSeconds != nil {
+		udpTimeout = *input.UdpTimeoutSeconds
+	}
+	return []interface{}{
+		map[string]interface{}{
+			"scheduler":                  scheduler,
+			"tcp_fin_timeout_in_seconds": tcpFinTimeout,
+			"tcp_timeout_in_seconds":     tcpTimeout,
+			"udp_timeout_in_seconds":     udpTimeout,
+		},
+	}
+}
+
 func flattenKubernetesClusterAzureMonitorProfile(input *managedclusters.ManagedClusterAzureMonitorProfile) []interface{} {
 	if input == nil || input.Metrics == nil || !input.Metrics.Enabled {
 		return nil

diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown
@@ -604,10 +604,16 @@ Examples of how to use [AKS with Advanced Networking](https://docs.microsoft.com
 
 * `ip_versions` - (Optional) Specifies a list of IP versions the Kubernetes Cluster will use to assign IP addresses to its nodes and pods. Possible values are `IPv4` and/or `IPv6`. `IPv4` must always be specified. Changing this forces a new resource to be created.
 
-->**Note:** To configure dual-stack networking `ip_versions` should be set to `["IPv4", "IPv6"]`
+->**Note:** To configure dual-stack networking `ip_versions` should be set to `["IPv4", "IPv6"]`.
 
 ->**Note:** Dual-stack networking requires that the Preview Feature `Microsoft.ContainerService/AKS-EnableDualStack` is enabled and the Resource Provider is re-registered, see [the documentation](https://docs.microsoft.com/azure/aks/configure-kubenet-dual-stack?tabs=azure-cli%2Ckubectl#register-the-aks-enabledualstack-preview-feature) for more information.
 
+* `kube_proxy` - (Optional) A `kube_proxy` block.
+
+->**Note:** `kube_proxy` requires that the Preview Feature `Microsoft.ContainerService/KubeProxyConfigurationPreview` is enabled and the Resource Provider re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/configure-kube-proxy#register-the-kubeproxyconfigurationpreview-feature-flag) for more information.
+
+->**Note:** `kube_proxy` can only be disabled if the cluster is configured with its own (BYO) CNI, i.e. if `network_plugin` is set to `none`.
+
 * `load_balancer_sku` - (Optional) Specifies the SKU of the Load Balancer used for this Kubernetes Cluster. Possible values are `basic` and `standard`. Defaults to `standard`. Changing this forces a new resource to be created.
 
 * `load_balancer_profile` - (Optional) A `load_balancer_profile` block as defined below. This can only be specified when `load_balancer_sku` is set to `standard`. Changing this forces a new resource to be created.
@@ -616,6 +622,26 @@ Examples of how to use [AKS with Advanced Networking](https://docs.microsoft.com
 
 ---
 
+A `kube_proxy` block supports the following:
+
+* `mode` - (Required) Specifies which proxy mode to use. Possible values are `IPTABLES` and `IPVS`(Must be using Kubernetes version >= 1.22).
+
+* `ipvs` - (Optional) An `ipvs` block.
+
+---
+
+An `ipvs` block supports the following:
+
+* `scheduler` - (Optional) Specifies the IPVS scheduler, more information please see [this document](http://www.linuxvirtualserver.org/docs/scheduling.html).
+
+* `tcp_fin_timeout_in_seconds` - (Optional) Specifies the timeout value used for IPVS TCP sessions after receiving a FIN in seconds.
+
+* `tcp_timeout_in_seconds` - (Optional) Specifies the timeout value used for idle IPVS TCP sessions in seconds.
+
+* `udp_timeout_in_seconds` - (Optional) Specifies the timeout value used for IPVS UDP packets in seconds.
+
+---
+
 A `load_balancer_profile` block supports the following:
 
 ~> **Note:** The fields `managed_outbound_ip_count`, `outbound_ip_address_ids` and `outbound_ip_prefix_ids` are mutually exclusive. Note that when specifying `outbound_ip_address_ids` ([azurerm_public_ip](/docs/providers/azurerm/r/public_ip.html)) the SKU must be `standard`.