Merge main

Signed-off-by: Dainius Serplis <[email protected]>

.changes/v3.14.1/1357-improvements.md

@@ -0,0 +1,2 @@
+* Add support of CSE 4.2.2 and 4.2.3 versions by improving `vcd_cse_kubernetes cluster` and updating
+  the installation guide and examples [GH-1357]

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 Changes in progress for v4.0.0 are available at [.changes/v4.0.0](https://github.com/vmware/terraform-provider-vcd/tree/main/.changes/v4.0.0) until the release.
 
+## 3.14.1 (December 5, 2024)
+
+### IMPROVEMENTS
+* Add support of CSE 4.2.2 and 4.2.3 versions by improving `vcd_cse_kubernetes cluster` and updating
+  the installation guide and examples ([#1357](https://github.com/vmware/terraform-provider-vcd/pull/1357))
+
+### NOTES
+* Bump [`go-vcloud-director`](https://github.com/vmware/go-vcloud-director/tree/release/v2.x) to v2.26.1
+  (SDK this provider uses for low level access to the VCD) ([#1363](https://github.com/vmware/terraform-provider-vcd/pull/1363))
+
 ## 3.14.0 (September 17, 2024)
 
 ### FEATURES

PREVIOUS_VERSION

@@ -1 +1 @@
-v3.14.0
+v3.14.1

examples/container-service-extension/v4.2/install/step1/3.11-cse-install-1-provider-config.tf

@@ -11,12 +11,12 @@
 #   Other than that, this snippet should be applied as it is.
 # ------------------------------------------------------------------------------------------------------------
 
-# VCD Provider configuration. It must be at least v3.12.0 and configured with a System administrator account.
+# VCD Provider configuration. It must be v3.14.1 and configured with a System administrator account.
 terraform {
   required_providers {
     vcd = {
       source  = "vmware/vcd"
-      version = ">= 3.12"
+      version = "3.14.1"
     }
   }
 }

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.2

@@ -0,0 +1,61 @@
+# Change configuration to your needs and rename to 'terraform.tfvars'
+# For more details about the variables specified here, please read the guide first:
+# https://registry.terraform.io/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install
+
+# ------------------------------------------------
+# VCD Provider config
+# ------------------------------------------------
+
+vcd_url                = "https://vcd.my-awesome-corp.com"
+administrator_user     = "administrator"
+administrator_password = "change-me"
+administrator_org      = "System"
+insecure_login         = "false"
+
+# ------------------------------------------------
+# CSE Server Pre-requisites
+# ------------------------------------------------
+
+# This user will be created by the Terraform configuration, so you can
+# customise what its username and password will be.
+# This user will have an API token that must be consumed by the CSE Server.
+cse_admin_username = "cse_admin"
+cse_admin_password = "change-me"
+
+# ------------------------------------------------
+# CSE Server Settings
+# ------------------------------------------------
+
+# These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
+# To know more about the specific versions, please refer to the CSE documentation.
+# The values set here correspond to CSE 4.2.2:
+vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
+capvcd_version                = "1.3.0" # 4.2.2
+cpi_version                   = "1.6.0" # 4.2.2
+csi_version                   = "1.6.0" # 4.2.2
+rde_projector_version         = "0.7.1" # 4.2.2
+
+# Optional but recommended to avoid rate limiting when configuring the TKGm clusters.
+# Create this one in https://github.com/settings/tokens
+github_personal_access_token = ""
+
+# Node will be considered unhealthy and remediated if joining the cluster takes longer than this timeout (seconds)
+node_startup_timeout = "900"
+# A newly joined node will be considered unhealthy and remediated if it cannot host workloads for longer than this timeout (seconds)
+node_not_ready_timeout = "300"
+# A healthy node will be considered unhealthy and remediated if it is unreachable for longer than this timeout (seconds)
+node_unknown_timeout = "300"
+# Remediation will be suspended when the number of unhealthy nodes exceeds this percentage.
+# (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated)
+max_unhealthy_node_percentage = 100
+
+# URL from where TKG clusters will fetch container images
+container_registry_url = "projects.registry.vmware.com"
+
+# Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+k8s_cluster_certificates = []
+
+# Certificate(s) to allow clusters to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+bootstrap_vm_certificates = []

examples/container-service-extension/v4.2/install/step1/terraform.tfvars.example.4.2.3

@@ -0,0 +1,61 @@
+# Change configuration to your needs and rename to 'terraform.tfvars'
+# For more details about the variables specified here, please read the guide first:
+# https://registry.terraform.io/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install
+
+# ------------------------------------------------
+# VCD Provider config
+# ------------------------------------------------
+
+vcd_url                = "https://vcd.my-awesome-corp.com"
+administrator_user     = "administrator"
+administrator_password = "change-me"
+administrator_org      = "System"
+insecure_login         = "false"
+
+# ------------------------------------------------
+# CSE Server Pre-requisites
+# ------------------------------------------------
+
+# This user will be created by the Terraform configuration, so you can
+# customise what its username and password will be.
+# This user will have an API token that must be consumed by the CSE Server.
+cse_admin_username = "cse_admin"
+cse_admin_password = "change-me"
+
+# ------------------------------------------------
+# CSE Server Settings
+# ------------------------------------------------
+
+# These are required to create the Runtime Defined Entity that will contain the CSE Server configuration (vcdKeConfig)
+# To know more about the specific versions, please refer to the CSE documentation.
+# The values set here correspond to CSE 4.2.3:
+vcdkeconfig_template_filepath = "../../entities/vcdkeconfig.json.template"
+capvcd_version                = "1.3.2" # 4.2.3
+cpi_version                   = "1.6.1" # 4.2.3
+csi_version                   = "1.6.0" # 4.2.3
+rde_projector_version         = "0.7.1" # 4.2.3
+
+# Optional but recommended to avoid rate limiting when configuring the TKGm clusters.
+# Create this one in https://github.com/settings/tokens
+github_personal_access_token = ""
+
+# Node will be considered unhealthy and remediated if joining the cluster takes longer than this timeout (seconds)
+node_startup_timeout = "900"
+# A newly joined node will be considered unhealthy and remediated if it cannot host workloads for longer than this timeout (seconds)
+node_not_ready_timeout = "300"
+# A healthy node will be considered unhealthy and remediated if it is unreachable for longer than this timeout (seconds)
+node_unknown_timeout = "300"
+# Remediation will be suspended when the number of unhealthy nodes exceeds this percentage.
+# (100% means that unhealthy nodes will always be remediated, while 0% means that unhealthy nodes will never be remediated)
+max_unhealthy_node_percentage = 100
+
+# URL from where TKG clusters will fetch container images
+container_registry_url = "projects.registry.vmware.com"
+
+# Certificate(s) to allow the ephemeral VM (created during cluster creation) to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+k8s_cluster_certificates = []
+
+# Certificate(s) to allow clusters to authenticate with.
+# For example, when pulling images from a container registry. (Copy and paste .cert file contents)
+bootstrap_vm_certificates = []

examples/container-service-extension/v4.2/install/step2/3.11-cse-install-4-provider-config.tf

@@ -11,12 +11,12 @@
 #   Other than that, this snippet should be applied as it is.
 # ------------------------------------------------------------------------------------------------------------
 
-# VCD Provider configuration. It must be at least v3.12.0 and configured with a System administrator account.
+# VCD Provider configuration. It must be v3.14.1 and configured with a System administrator account.
 terraform {
   required_providers {
     vcd = {
       source  = "vmware/vcd"
-      version = ">= 3.12"
+      version = "3.14.1"
     }
     time = {
       source  = "hashicorp/time"

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.34.0
 	github.com/kr/pretty v0.3.1
-	github.com/vmware/go-vcloud-director/v3 v3.0.0-alpha.9
+	github.com/vmware/go-vcloud-director/v3 v3.0.0-alpha.11
 )
 
 require (
@@ -69,4 +69,4 @@ require (
 	google.golang.org/protobuf v1.34.0 // indirect
 )
 
-replace github.com/vmware/go-vcloud-director/v3 => github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241210095607-56f0630c06ae
+replace github.com/vmware/go-vcloud-director/v3 => github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211131211-033a21e4bb6b

go.sum

@@ -1,7 +1,7 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241210095607-56f0630c06ae h1:WYYdjlNfBK9x+mFmaRz/ZHNG+eaIqzua+4Q7pwZgZjo=
-github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241210095607-56f0630c06ae/go.mod h1:68KHsVns52dsq/w5JQYzauaU/+NAi1FmCxhBrFc/VoQ=
+github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211131211-033a21e4bb6b h1:bQx5TP0NtcjleEUyE0U1HKgIjZsjggGUjpHYR21MtjA=
+github.com/Didainius/go-vcloud-director/v3 v3.0.0-alpha.4.0.20241211131211-033a21e4bb6b/go.mod h1:68KHsVns52dsq/w5JQYzauaU/+NAi1FmCxhBrFc/VoQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/ProtonMail/go-crypto v1.1.0-alpha.2 h1:bkyFVUP+ROOARdgCiJzNQo2V2kiB97LyUpzH9P6Hrlg=

vcd/resource_vcd_cse_kubernetes_cluster.go

@@ -33,8 +33,7 @@ func resourceVcdCseKubernetesCluster() *schema.Resource {
 			"cse_version": {
 				Type:         schema.TypeString,
 				Required:     true,
-				ForceNew:     true,
-				ValidateFunc: validation.StringInSlice([]string{"4.1.0", "4.1.1", "4.2.0", "4.2.1"}, false),
+				ValidateFunc: validation.StringInSlice([]string{"4.1.0", "4.1.1", "4.2.0", "4.2.1", "4.2.2", "4.2.3"}, false),
 				Description:  "The CSE version to use",
 				DiffSuppressFunc: func(k, oldValue, newValue string, d *schema.ResourceData) bool {
 					// This custom diff function allows to correctly compare versions.

vcd/resource_vcd_cse_kubernetes_cluster_test.go

@@ -33,6 +33,10 @@ func TestAccVcdCseKubernetesCluster(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("CSE versions 4.2.2 and 4.2.3 do not work with System Administrator")
+	}
+
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)
 	if err != nil {
 		t.Fatal(err)
@@ -597,6 +601,10 @@ func TestAccVcdCseKubernetesClusterCreationWithAutoscaler(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("test does not work with System Administrator")
+	}
+
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)
 	if err != nil {
 		t.Fatal(err)
@@ -746,6 +754,10 @@ func TestAccVcdCseKubernetesClusterFailure(t *testing.T) {
 	preTestChecks(t)
 	requireCseConfig(t, testConfig)
 
+	if testConfig.Cse.Version == "4.2.2" || testConfig.Cse.Version == "4.2.3" {
+		t.Skip("test does not work with System Administrator")
+	}
+
 	vcdClient := createSystemTemporaryVCDConnection()
 
 	cseVersion, err := semver.NewVersion(testConfig.Cse.Version)

vcd/resource_vcd_tm_vcenter.go

@@ -233,7 +233,12 @@ func resourceVcdTmVcenterRead(ctx context.Context, d *schema.ResourceData, meta
 		getEntityFunc:  fakeGetById, // TODO: TM: remove this function
 		stateStoreFunc: setTmVcenterData,
 		readHooks: []outerEntityHook[*govcd.VCenter]{
-			sleepHookVcenter(1 * time.Minute),
+			// TODO: TM ensure that the vCenter listener state is "CONNECTED"  before triggering
+			// refresh as it will fail otherwise. At the moment it has a delay before it becomes
+			// CONNECTED after creation task succeeds. It should not be needed once vCenter creation
+			// task ensures that the listener is connected.
+			waitForListenerStatusConnected,
+
 			refreshVcenter(shouldRefresh),               // vCenter read can optionally trigger "refresh" operation
 			refreshVcenterPolicy(shouldRefreshPolicies), // vCenter read can optionally trigger "refresh policies" operation
 		},
@@ -279,7 +284,7 @@ func disableVcenter(v *govcd.VCenter) error {
 func refreshVcenter(execute bool) outerEntityHook[*govcd.VCenter] {
 	return func(v *govcd.VCenter) error {
 		if execute {
-			err := v.Refresh()
+			err := v.RefreshVcenter()
 			if err != nil {
 				return fmt.Errorf("error refreshing vCenter: %s", err)
 			}
@@ -302,6 +307,24 @@ func refreshVcenterPolicy(execute bool) outerEntityHook[*govcd.VCenter] {
 	}
 }
 
+// TODO: TM: should not be required because a successful vCenter creation task should work
+func waitForListenerStatusConnected(v *govcd.VCenter) error {
+	for c := 0; c < 20; c++ {
+		err := v.Refresh()
+		if err != nil {
+			return fmt.Errorf("error refreshing vCenter: %s", err)
+		}
+
+		if v.VSphereVCenter.ListenerState == "CONNECTED" {
+			return nil
+		}
+
+		time.Sleep(2 * time.Second)
+	}
+
+	return fmt.Errorf("failed waiting for listener state to become 'CONNECTED', got '%s'", v.VSphereVCenter.ListenerState)
+}
+
 // autoTrustHostCertificate can automatically add host certificate to trusted ones
 // * urlSchemaFieldName - Terraform schema field (TypeString) name that contains URL of entity
 // * trustSchemaFieldName - Terraform schema field (TypeBool) name that defines if the certificate should be trusted

website/docs/d/cse_kubernetes_cluster.html.markdown

@@ -18,6 +18,8 @@ Supports the following **Container Service Extension** versions:
 * 4.1.1 / 4.1.1a
 * 4.2.0
 * 4.2.1
+* 4.2.2
+* 4.2.3
 
 -> To install CSE in VMware Cloud Director, please follow [this guide](/providers/vmware/vcd/latest/docs/guides/container_service_extension_4_x_install)