feat: create virtual clusters from provision

internal/controller/kubefirst.go

@@ -7,21 +7,28 @@ See the LICENSE file for more details.
 package controller
 
 import (
+	"bytes"
+	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	awsext "github.com/kubefirst/kubefirst-api/extensions/aws"
 	"github.com/kubefirst/kubefirst-api/internal/secrets"
+	"github.com/kubefirst/kubefirst-api/pkg/types"
+	"github.com/kubefirst/runtime/pkg"
 	"github.com/kubefirst/runtime/pkg/k8s"
 	log "github.com/rs/zerolog/log"
 	v1secret "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
-func readKubefirstAPITokenFromSecret(clientset *kubernetes.Clientset) string {
+func ReadKubefirstAPITokenFromSecret(clientset *kubernetes.Clientset) string {
 	existingKubernetesSecret, err := k8s.ReadSecretV2(clientset, "kubefirst", "kubefirst-initial-secrets")
 	if err != nil || existingKubernetesSecret == nil {
 		log.Printf("Error reading existing Secret data: %s", err)
@@ -78,8 +85,76 @@ func (clctrl *ClusterController) ExportClusterRecord() error {
 
 	if err != nil {
 		clctrl.HandleError(err.Error())
-		return errors.New(fmt.Sprintf("unable to save secret to management cluster. %s", err))
+		return fmt.Errorf("unable to save secret to management cluster. %s", err)
 	}
 
 	return nil
 }
+
+// ExportClusterRecord will export cluster record to mgmt cluster
+func (clctrl *ClusterController) CreateVirtualClusters() error {
+	var fullDomainName string
+
+	if clctrl.SubdomainName != "" {
+		fullDomainName = fmt.Sprintf("%s.%s", clctrl.SubdomainName, clctrl.DomainName)
+	} else {
+		fullDomainName = clctrl.DomainName
+	}
+
+	consoleCloudUrl := fmt.Sprintf("https://kubefirst.%s", fullDomainName)
+
+	if strings.ToLower(os.Getenv("K1_LOCAL_DEBUG")) == "true" { //allow using local console running on port 3000
+		consoleCloudUrl = "http://localhost:3000"
+	}
+
+	err := pkg.IsAppAvailable(fmt.Sprintf("%s/api/proxyHealth", consoleCloudUrl), "kubefirst api")
+	if err != nil {
+		log.Error().Msgf("unable to wait for kubefirst console: %s", err)
+		clctrl.HandleError(err.Error())
+		return err
+	}
+
+	requestObject := types.ProxyRequest{
+		Url: fmt.Sprintf("/cluster/%s/vclusters", clctrl.ClusterName),
+	}
+
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	httpClient := http.Client{Transport: customTransport}
+
+	payload, err := json.Marshal(requestObject)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/proxy", consoleCloudUrl), bytes.NewReader(payload))
+	if err != nil {
+		log.Error().Msgf("unable to create default clusters: %s", err)
+		clctrl.HandleError(err.Error())
+		return err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Accept", "application/json")
+
+	res, err := httpClient.Do(req)
+	if err != nil {
+		log.Error().Msgf("unable to create default clusters: %s", err)
+		clctrl.HandleError(err.Error())
+		return err
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		log.Error().Msgf("unable to create default clusters: %s %s", err, body)
+		clctrl.HandleError(err.Error())
+		return err
+	}
+
+	log.Info().Msg("cluster creation complete")
+
+	return nil
+}

internal/router/api/v1/cluster.go

@@ -855,29 +855,41 @@ func PostResetClusterProgress(c *gin.Context) {
 }
 
 // PostCreateVcluster godoc
-// @Summary Create a Kubefirst cluster
-// @Description Create a Kubefirst cluster
+// @Summary Create default virtual clusters
+// @Description Create default virtual clusters
 // @Tags cluster
 // @Accept json
 // @Produce json
 // @Param	cluster_name	path	string	true	"Cluster name"
-// @Param	definition	body	types.ClusterDefinition	true	"Cluster create request in JSON format"
 // @Success 202 {object} types.JSONSuccessResponse
 // @Failure 400 {object} types.JSONFailureResponse
-// @Router /vcluster/:cluster_name [post]
+// @Router /cluster/:cluster_name/vclusters [post]
 // @Param Authorization header string true "API key" default(Bearer <API key>)
-// PostCreateVcluster handles a request to create a cluster
+// PostCreateVcluster handles a request to create default virtual cluster for the mgmt cluster
 func PostCreateVcluster(c *gin.Context) {
-	importedCluster, err := secrets.ImportClusterIfEmpty(true)
-	if err != nil {
-		log.Fatal().Msg(err.Error())
+	clusterName, param := c.Params.Get("cluster_name")
+	if !param {
+		c.JSON(http.StatusBadRequest, types.JSONFailureResponse{
+			Message: ":cluster_name not provided",
+		})
+		return
 	}
 
-	err = environments.CreateDefaultClusters(importedCluster)
+	kcfg := utils.GetKubernetesClient(clusterName)
+
+	cluster, err := secrets.GetCluster(kcfg.Clientset, clusterName)
 	if err != nil {
-		log.Info().Msgf("Error creating default environments %s", err.Error())
+		log.Fatal().Msg(err.Error())
 	}
+
+	go func() {
+		err = environments.CreateDefaultClusters(cluster)
+		if err != nil {
+			log.Fatal().Msgf("Error creating default environments %s", err.Error())
+		}
+	}()
+
 	c.JSON(http.StatusOK, types.JSONSuccessResponse{
-		Message: "created default cluster environments",
+		Message: "created default cluster environments enqueued",
 	})
 }

internal/router/router.go

@@ -53,7 +53,7 @@ func SetupRouter() *gin.Engine {
 		v1.POST("/cluster/:cluster_name", middleware.ValidateAPIKey(), router.PostCreateCluster)
 		v1.GET("/cluster/:cluster_name/export", middleware.ValidateAPIKey(), router.GetExportCluster)
 		v1.POST("/cluster/:cluster_name/reset_progress", middleware.ValidateAPIKey(), router.PostResetClusterProgress)
-		v1.POST("/vclusters", middleware.ValidateAPIKey(), router.PostCreateVcluster)
+		v1.POST("/cluster/:cluster_name/vclusters", middleware.ValidateAPIKey(), router.PostCreateVcluster)
 
 		// KubeConfig
 		v1.POST("/kubeconfig/:cloud_provider", middleware.ValidateAPIKey(), router.GetClusterKubeConfig)

main.go

@@ -40,7 +40,6 @@ func main() {
 
 	log.Info().Msg("checking for cluster import secret for management cluster")
 	// Import if needed
-	//TODO: SECRETS
 	importedCluster, err := secrets.ImportClusterIfEmpty(true)
 	if err != nil {
 		log.Fatal().Msg(err.Error())
@@ -68,19 +67,6 @@ func main() {
 				}
 			}()
 		}
-
-		// if importedCluster.CloudProvider != "k3d" {
-		// 	go func() {
-		// 		log.Info().Msgf("Waiting for services to be created  %s", importedCluster.ClusterName)
-		// 		time.Sleep(time.Second * 30)
-
-		// 		log.Info().Msgf("adding default environments for cluster %s", importedCluster.ClusterName)
-		// 		err := environments.CreateDefaultClusters(importedCluster)
-		// 		if err != nil {
-		// 			log.Info().Msgf("Error creating default environments %s", err.Error())
-		// 		}
-		// 	}()
-		// }
 	}
 
 	// Programmatically set swagger info

pkg/types/cluster.go

@@ -160,6 +160,10 @@ type ProxyImportRequest struct {
 	Url  string  `bson:"url" json:"url"`
 }
 
+type ProxyRequest struct {
+	Url string `bson:"url" json:"url"`
+}
+
 type Environment struct {
 	ID                primitive.ObjectID `bson:"_id" json:"_id"`
 	Name              string             `bson:"name" json:"name"`

providers/akamai/create.go

@@ -258,7 +258,34 @@ func CreateAkamaiCluster(definition *pkgtypes.ClusterDefinition) error {
 		return err
 	}
 
+	// Wait for last sync wave app transition to Running
+	log.Info().Msg("waiting for final sync wave Deployment to transition to Running")
+	argocdDeployment, err := k8s.ReturnDeploymentObject(
+		kcfg.Clientset,
+		"app.kubernetes.io/name",
+		"argocd-server",
+		"argocd",
+		3600,
+	)
+	if err != nil {
+		log.Error().Msgf("Error finding argocd Deployment: %s", err)
+		ctrl.HandleError(err.Error())
+		return err
+	}
+	_, err = k8s.WaitForDeploymentReady(kcfg.Clientset, argocdDeployment, 3600)
+	if err != nil {
+		log.Error().Msgf("Error waiting for argocd deployment to enter Ready state: %s", err)
+
+		ctrl.HandleError(err.Error())
+		return err
+	}
+
 	log.Info().Msg("cluster creation complete")
 
+	err = ctrl.CreateVirtualClusters()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

providers/aws/create.go

@@ -286,7 +286,34 @@ func CreateAWSCluster(definition *pkgtypes.ClusterDefinition) error {
 		return err
 	}
 
+	// Wait for last sync wave app transition to Running
+	log.Info().Msg("waiting for final sync wave Deployment to transition to Running")
+	argocdDeployment, err := k8s.ReturnDeploymentObject(
+		kcfg.Clientset,
+		"app.kubernetes.io/name",
+		"argocd-server",
+		"argocd",
+		3600,
+	)
+	if err != nil {
+		log.Error().Msgf("Error finding argocd Deployment: %s", err)
+		ctrl.HandleError(err.Error())
+		return err
+	}
+	_, err = k8s.WaitForDeploymentReady(kcfg.Clientset, argocdDeployment, 3600)
+	if err != nil {
+		log.Error().Msgf("Error waiting for argocd deployment to enter Ready state: %s", err)
+
+		ctrl.HandleError(err.Error())
+		return err
+	}
+
 	log.Info().Msg("cluster creation complete")
 
+	err = ctrl.CreateVirtualClusters()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

providers/civo/create.go

@@ -218,7 +218,6 @@ func CreateCivoCluster(definition *pkgtypes.ClusterDefinition) error {
 		ctrl.HandleError(err.Error())
 		return err
 	} else {
-
 		// Create default service entries
 		cl, _ := secrets.GetCluster(ctrl.KubernetesClient, ctrl.ClusterName)
 		err = services.AddDefaultServices(&cl)
@@ -280,7 +279,10 @@ func CreateCivoCluster(definition *pkgtypes.ClusterDefinition) error {
 
 	log.Info().Msg("cluster creation complete")
 
-	//! call api to create default clusters
+	err = ctrl.CreateVirtualClusters()
+	if err != nil {
+		return err
+	}
 
 	return nil
 }

providers/digitalocean/create.go

@@ -259,7 +259,34 @@ func CreateDigitaloceanCluster(definition *pkgtypes.ClusterDefinition) error {
 		return err
 	}
 
+	// Wait for last sync wave app transition to Running
+	log.Info().Msg("waiting for final sync wave Deployment to transition to Running")
+	argocdDeployment, err := k8s.ReturnDeploymentObject(
+		kcfg.Clientset,
+		"app.kubernetes.io/name",
+		"argocd-server",
+		"argocd",
+		3600,
+	)
+	if err != nil {
+		log.Error().Msgf("Error finding argocd Deployment: %s", err)
+		ctrl.HandleError(err.Error())
+		return err
+	}
+	_, err = k8s.WaitForDeploymentReady(kcfg.Clientset, argocdDeployment, 3600)
+	if err != nil {
+		log.Error().Msgf("Error waiting for argocd deployment to enter Ready state: %s", err)
+
+		ctrl.HandleError(err.Error())
+		return err
+	}
+
 	log.Info().Msg("cluster creation complete")
 
+	err = ctrl.CreateVirtualClusters()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }

providers/google/create.go

@@ -287,7 +287,34 @@ func CreateGoogleCluster(definition *pkgtypes.ClusterDefinition) error {
 		return err
 	}
 
+	// Wait for last sync wave app transition to Running
+	log.Info().Msg("waiting for final sync wave Deployment to transition to Running")
+	argocdDeployment, err := k8s.ReturnDeploymentObject(
+		kcfg.Clientset,
+		"app.kubernetes.io/name",
+		"argocd-server",
+		"argocd",
+		3600,
+	)
+	if err != nil {
+		log.Error().Msgf("Error finding argocd Deployment: %s", err)
+		ctrl.HandleError(err.Error())
+		return err
+	}
+	_, err = k8s.WaitForDeploymentReady(kcfg.Clientset, argocdDeployment, 3600)
+	if err != nil {
+		log.Error().Msgf("Error waiting for argocd deployment to enter Ready state: %s", err)
+
+		ctrl.HandleError(err.Error())
+		return err
+	}
+
 	log.Info().Msg("cluster creation complete")
 
+	err = ctrl.CreateVirtualClusters()
+	if err != nil {
+		return err
+	}
+
 	return nil
 }