Merge main

Signed-off-by: Dainius Serplis <[email protected]>

.changes/v3.14.1/1357-improvements.md

@@ -0,0 +1,2 @@
+* Add support of CSE 4.2.2 and 4.2.3 versions by improving `vcd_cse_kubernetes cluster` and updating
+  the installation guide and examples [GH-1357]

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 Changes in progress for v4.0.0 are available at [.changes/v4.0.0](https://github.com/vmware/terraform-provider-vcd/tree/main/.changes/v4.0.0) until the release.
 
+## 3.14.1 (December 5, 2024)
+
+### IMPROVEMENTS
+* Add support of CSE 4.2.2 and 4.2.3 versions by improving `vcd_cse_kubernetes cluster` and updating
+  the installation guide and examples ([#1357](https://github.com/vmware/terraform-provider-vcd/pull/1357))
+
+### NOTES
+* Bump [`go-vcloud-director`](https://github.com/vmware/go-vcloud-director/tree/release/v2.x) to v2.26.1
+  (SDK this provider uses for low level access to the VCD) ([#1363](https://github.com/vmware/terraform-provider-vcd/pull/1363))
+
 ## 3.14.0 (September 17, 2024)
 
 ### FEATURES

PREVIOUS_VERSION

@@ -1 +1 @@
-v3.14.0
+v3.14.1

examples/container-service-extension/v4.2/install/step1/3.11-cse-install-1-provider-config.tf

@@ -11,12 +11,12 @@
 #   Other than that, this snippet should be applied as it is.
 # ------------------------------------------------------------------------------------------------------------
 
-# VCD Provider configuration. It must be at least v3.12.0 and configured with a System administrator account.
+# VCD Provider configuration. It must be v3.14.1 and configured with a System administrator account.
 terraform {
   required_providers {
     vcd = {
       source  = "vmware/vcd"
-      version = ">= 3.12"
+      version = "3.14.1"
     }
   }
 }

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.2

@@ -0,0 +1,61 @@
+# Change configuration to your needs and rename to 'terraform.tfvars'
+# For more details about the variables specified here, please read the guide first:
+# https://registry.terraform.io/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install
+
+# ------------------------------------------------
+# VCD Provider config
+# ------------------------------------------------
+
+vcd_url                = "https://vcd.my-awesome-corp.com"
+administrator_user     = "administrator"
+administrator_password = "change-me"
+administrator_org      = "System"
+insecure_login         = "false"
+
+# ------------------------------------------------
+# CSE Server Pre-requisites
+# ------------------------------------------------
+
+# This user will be created by the Terraform configuration, so you can
+# customise what its username and password will be.
+# This user will have an API token that must be consumed by the CSE Server.
+cse_admin_username = "cse_admin"
+cse_admin_password = "change-me"
+
+# ------------------------------------------------
+# CSE Server Settings
+# ------------------------------------------------
+
+# These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
+# To know more about the specific versions, please refer to the CSE documentation.
+# The values set here correspond to CSE 4.2.2:
+vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
+capvcd_version                = "1.3.0" # 4.2.2
+cpi_version                   = "1.6.0" # 4.2.2
+csi_version                   = "1.6.0" # 4.2.2
+rde_projector_version         = "0.7.1" # 4.2.2
+
+# Optional but recommended to avoid rate limiting when configuring the TKGm clusters.
+# Create this one in https://github.com/settings/tokens
+github_personal_access_token = ""
+
+# Node will be considered unhealthy and remediated if joining the cluster takes longer than this timeout (seconds)
+node_startup_timeout = "900"
+# A newly joined node will be considered unhealthy and remediated if it cannot host workloads for longer than this timeout (seconds)
+node_not_ready_timeout = "300"
+# A healthy node will be considered unhealthy and remediated if it is unreachable for longer than this timeout (seconds)
+node_unknown_timeout = "300"
+# Remediation will be suspended when the number of unhealthy nodes exceeds this percentage.
+# (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated)
+max_unhealthy_node_percentage = 100
+
+# URL from where TKG clusters will fetch container images
+container_registry_url = "projects.registry.vmware.com"
+
+# Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+k8s_cluster_certificates = []
+
+# Certificate(s) to allow clusters to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+bootstrap_vm_certificates = []

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.3

@@ -0,0 +1,61 @@
+# Change configuration to your needs and rename to 'terraform.tfvars'
+# For more details about the variables specified here, please read the guide first:
+# https://registry.terraform.io/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install
+
+# ------------------------------------------------
+# VCD Provider config
+# ------------------------------------------------
+
+vcd_url                = "https://vcd.my-awesome-corp.com"
+administrator_user     = "administrator"
+administrator_password = "change-me"
+administrator_org      = "System"
+insecure_login         = "false"
+
+# ------------------------------------------------
+# CSE Server Pre-requisites
+# ------------------------------------------------
+
+# This user will be created by the Terraform configuration, so you can
+# customise what its username and password will be.
+# This user will have an API token that must be consumed by the CSE Server.
+cse_admin_username = "cse_admin"
+cse_admin_password = "change-me"
+
+# ------------------------------------------------
+# CSE Server Settings
+# ------------------------------------------------
+
+# These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
+# To know more about the specific versions, please refer to the CSE documentation.
+# The values set here correspond to CSE 4.2.3:
+vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
+capvcd_version                = "1.3.2" # 4.2.3
+cpi_version                   = "1.6.1" # 4.2.3
+csi_version                   = "1.6.0" # 4.2.3
+rde_projector_version         = "0.7.1" # 4.2.3
+
+# Optional but recommended to avoid rate limiting when configuring the TKGm clusters.
+# Create this one in https://github.com/settings/tokens
+github_personal_access_token = ""
+
+# Node will be considered unhealthy and remediated if joining the cluster takes longer than this timeout (seconds)
+node_startup_timeout = "900"
+# A newly joined node will be considered unhealthy and remediated if it cannot host workloads for longer than this timeout (seconds)
+node_not_ready_timeout = "300"
+# A healthy node will be considered unhealthy and remediated if it is unreachable for longer than this timeout (seconds)
+node_unknown_timeout = "300"
+# Remediation will be suspended when the number of unhealthy nodes exceeds this percentage.
+# (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated)
+max_unhealthy_node_percentage = 100
+
+# URL from where TKG clusters will fetch container images
+container_registry_url = "projects.registry.vmware.com"
+
+# Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+k8s_cluster_certificates = []
+
+# Certificate(s) to allow clusters to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+bootstrap_vm_certificates = []

examples/container-service-extension/v4.2/install/step2/3.11-cse-install-4-provider-config.tf

@@ -11,12 +11,12 @@
 #   Other than that, this snippet should be applied as it is.
 # ------------------------------------------------------------------------------------------------------------
 
-# VCD Provider configuration. It must be at least v3.12.0 and configured with a System administrator account.
+# VCD Provider configuration. It must be v3.14.1 and configured with a System administrator account.
 terraform {
   required_providers {
     vcd = {
       source  = "vmware/vcd"
-      version = ">= 3.12"
+      version = "3.14.1"
     }
     time = {
       source  = "hashicorp/time"

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.34.0
 	github.com/kr/pretty v0.3.1
-	github.com/vmware/go-vcloud-director/v3 v3.0.0-alpha.9
+	github.com/vmware/go-vcloud-director/v3 v3.0.0-alpha.10
 )
 
 require (
@@ -69,4 +69,4 @@ require (
 	google.golang.org/protobuf v1.34.0 // indirect
 )
 
-replace github.com/vmware/go-vcloud-director/v3 => github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211113049-cf60b8590d2e
+replace github.com/vmware/go-vcloud-director/v3 => github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211124338-dda667a5311a

go.sum

@@ -1,7 +1,7 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211113049-cf60b8590d2e h1:1fnLed+1cmP0RgtCxbxj2M8AP+zWRBdmGmHErfJ1Kvg=
-github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211113049-cf60b8590d2e/go.mod h1:68KHsVns52dsq/w5JQYzauaU/+NAi1FmCxhBrFc/VoQ=
+github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211124338-dda667a5311a h1:cAvmqPnap+GE5XLgGbVGBDf47misxjCATWmlYHtsdQk=
+github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211124338-dda667a5311a/go.mod h1:68KHsVns52dsq/w5JQYzauaU/+NAi1FmCxhBrFc/VoQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg=

vcd/resource_vcd_cse_kubernetes_cluster.go

@@ -33,8 +33,7 @@ func resourceVcdCseKubernetesCluster() *schema.Resource {
 			"cse_version": {
 				Type:         schema.TypeString,
 				Required:     true,
-				ForceNew:     true,
-				ValidateFunc: validation.StringInSlice([]string{"4.1.0", "4.1.1", "4.2.0", "4.2.1"}, false),
+				ValidateFunc: validation.StringInSlice([]string{"4.1.0", "4.1.1", "4.2.0", "4.2.1", "4.2.2", "4.2.3"}, false),
 				Description:  "The CSE version to use",
 				DiffSuppressFunc: func(k, oldValue, newValue string, d *schema.ResourceData) bool {
 					// This custom diff function allows to correctly compare versions.

vcd/resource_vcd_cse_kubernetes_cluster_test.go

@@ -33,6 +33,10 @@ func TestAccVcdCseKubernetesCluster(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("CSE versions 4.2.2 and 4.2.3 do not work with System Administrator")
+	}
+
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)
 	if err != nil {
 		t.Fatal(err)
@@ -597,6 +601,10 @@ func TestAccVcdCseKubernetesClusterCreationWithAutoscaler(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("test does not work with System Administrator")
+	}
+
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)
 	if err != nil {
 		t.Fatal(err)
@@ -746,6 +754,10 @@ func TestAccVcdCseKubernetesClusterFailure(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("test does not work with System Administrator")
+	}
+
 	vcdClient := createSystemTemporaryVCDConnection()
 
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)

website/docs/d/cse_kubernetes_cluster.html.markdown

@@ -18,6 +18,8 @@ Supports the following **Container Service Extension** versions:
 * 4.1.1 / 4.1.1a
 * 4.2.0
 * 4.2.1
+* 4.2.2
+* 4.2.3
 
 -> To install CSE in VMware Cloud Director, please follow [this guide](/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install)
 

website/docs/guides/container_service_extension_4_x_install.html.markdown

@@ -16,15 +16,21 @@ will allow tenant users to deploy **Tanzu Kubernetes Grid Multi-cloud (TKGm)** c
 To know more about CSE [4.2](https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/rn/vmware-cloud-director-container-service-extension-42-release-notes/index.html),
 you can visit [the documentation][cse_docs].
 
+This guide can be used to install the following **Container Service Extension** versions:
+
+* 4.2.0 (VCD Terraform Provider v3.12 or above)
+* 4.2.1 (VCD Terraform Provider v3.12 or above)
+* 4.2.2 (VCD Terraform Provider v3.14.1 or above)
+* 4.2.3 (VCD Terraform Provider v3.14.1 or above)
+
 ## Pre-requisites
 
 -> Please read also the pre-requisites section in the [CSE documentation][cse_docs].
 
 In order to complete the steps described in this guide, please be aware:
 
-* CSE 4.2 is supported from VCD v10.4.2 or above, as specified in the [Product Interoperability Matrix][product_matrix].
+* CSE 4.2.0 is supported from VCD 10.4.2 or above, but CSE 4.2.2 is supported on VCD 10.6.0, as specified in the [Product Interoperability Matrix][product_matrix].
   Please check that the target VCD appliance matches the criteria.
-* Terraform provider needs to be v3.12.0 or above.
 * Both CSE Server and the Bootstrap clusters require outbound Internet connectivity.
 * CSE 4.2 makes use of [ALB](/providers/vmware/vcd/latest/docs/guides/nsxt_alb) capabilities.
 
@@ -182,7 +188,7 @@ Then it will upload the required OVAs to them. The OVAs can be specified in `ter
 * `cse_ova_folder`: This will reference the path to the CSE OVA, as an absolute or relative path. It should **not** end with a trailing `/`.
 * `cse_ova_file`: This will reference the file name of the CSE OVA, like `VMware_Cloud_Director_Container_Service_Extension-4.2.1.ova`.
 
--> To download the required OVAs, please refer to the [CSE documentation][cse_docs]. 
+-> To download the required OVAs, please refer to the [CSE documentation][cse_docs].
 You can also check the [Product Interoperability Matrix][product_matrix] to confirm the appropriate version of TKGm.
 
 ~> Both CSE Server and TKGm OVAs are heavy. Please take into account that the upload process could take more than 30 minutes, depending
@@ -478,9 +484,29 @@ resource "vcd_global_role" "k8s_cluster_author" {
 }
 ```
 
-After applying the changes with `terraform apply`, you also need to update the CSE Server OVA to 4.2.1 and restart,
+After applying the changes with `terraform apply`, you also need to update the CSE Server OVA to 4.2.1 and redeploy,
 like it was done [in the previous section](#update-cse-server).
 
+## Upgrade from CSE 4.2.1 to 4.2.2 or 4.2.3
+
+In this case, you need to update the CSE Server Configuration, for example in 4.2.3:
+
+```hcl
+resource "vcd_rde" "vcdkeconfig_instance" {
+  # ...omitted
+  input_entity = templatefile(var.vcdkeconfig_template_filepath, {
+    # ...omitted
+    capvcd_version        = "1.3.2" # It was 1.3.0 in 4.2.1
+    cpi_version           = "1.6.1" # It was 1.6.0 in 4.2.1
+    csi_version           = "1.6.0"
+    rde_projector_version = "0.7.1" # It was 0.7.0 in 4.2.1
+  })
+}
+```
+
+After applying the changes with `terraform apply`, you *may* also need to update the CSE Server OVA to 4.2.2 (take into account that 
+CSE 4.2.3 reuses same OVA from 4.2.2) and redeploy, like it was done [in the previous section](#update-cse-server).
+
 ## Update CSE Server Configuration
 
 To make changes to the existing server configuration, you should be able to locate the [`vcd_rde`][rde] resource named `vcdkeconfig_instance`
@@ -516,7 +542,7 @@ This must be done as a 2-step operation.
 To upgrade the CSE Server appliance, first you need to upload a new CSE Server OVA to the CSE catalog and then replace
 the reference to the [vApp Template][catalog_vapp_template] in the CSE Server VM.
 
-In the [step 2 configuration][step2], you can find the `cse_ova` [vApp Template][catalog_vapp_template] and the 
+In the [step 2 configuration][step2], you can find the `cse_ova` [vApp Template][catalog_vapp_template] and the
 `cse_server_vm` [VM][vm] that were applied during the installation process.
 Then you can create a new `vcd_catalog_vapp_template` and modify `cse_server_vm` to reference it:
 
@@ -542,7 +568,7 @@ resource "vcd_vapp_vm" "cse_server_vm" {
 
 ## Working with Kubernetes clusters
 
-Please read the specific guide on that topic [here][cse_cluster_management_guide].
+Please use the `vcd_cse_kubernetes_cluster` resource that you can find [here][cse_cluster_resource].
 
 ## Uninstall CSE
 
@@ -554,7 +580,7 @@ Once all clusters are removed in the background by CSE Server, you may destroy t
 [api_token]: /providers/vmware/vcd/latest/docs/resources/api_token
 [catalog]: /providers/vmware/vcd/latest/docs/resources/catalog
 [catalog_vapp_template_ds]: /providers/vmware/vcd/latest/docs/data-sources/catalog_vapp_template
-[cse_cluster_management_guide]: /providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_cluster_management
+[cse_cluster_resource]: /providers/vmware/vcd/latest/docs/resources/cse_kubernetes_cluster
 [cse_docs]: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/index.html
 [edge_cluster]: /providers/vmware/vcd/latest/docs/data-sources/nsxt_edge_cluster
 [edge_gateway]: /providers/vmware/vcd/latest/docs/resources/nsxt_edgegateway
@@ -564,7 +590,7 @@ Once all clusters are removed in the background by CSE Server, you may destroy t
 [nsxt_tier0_router]: /providers/vmware/vcd/latest/docs/data-sources/nsxt_tier0_router
 [org]: /providers/vmware/vcd/latest/docs/resources/org
 [org_d]: /providers/vmware/vcd/latest/docs/data-sources/org
-[product_matrix]: https://interopmatrix.vmware.com/Interoperability?col=659,&row=0
+[product_matrix]: https://interopmatrix.broadcom.com/Interoperability
 [provider_gateway]: /providers/vmware/vcd/latest/docs/resources/external_network_v2
 [provider_vdc]: /providers/vmware/vcd/latest/docs/data-sources/provider_vdc
 [rights_bundle]: /providers/vmware/vcd/latest/docs/resources/rights_bundle
@@ -583,4 +609,4 @@ Once all clusters are removed in the background by CSE Server, you may destroy t
 [ui_plugin]: /providers/vmware/vcd/latest/docs/resources/ui_plugin
 [catalog_vapp_template]: /providers/vmware/vcd/latest/docs/resources/catalog_vapp_template
 [vdc]: /providers/vmware/vcd/latest/docs/resources/org_vdc
-[vm]: /providers/vmware/vcd/latest/docs/resources/vapp_vm
+[vm]: /providers/vmware/vcd/latest/docs/resources/vapp_vm