fix(platform): ensure app addons (#2052)

api/platform/types.go

@@ -119,6 +119,8 @@ const (
 	HubAPIServerAnno = "tkestack.io/hub-api-server"
 	// cluster credential token
 	CredentialTokenAnno = "tkestack.io/credential-token"
+	// AnywhereApplicationAnno contains base64 application json data
+	AnywhereApplicationAnno = "tkestack.io/anywhere-application"
 )
 
 // KubeVendorType describe the kubernetes provider of the cluster

api/platform/v1/types.go

@@ -130,6 +130,8 @@ const (
 	HubAPIServerAnno = "tkestack.io/hub-api-server"
 	// cluster credential token
 	CredentialTokenAnno = "tkestack.io/credential-token"
+	// AnywhereApplicationAnno contains base64 application json data
+	AnywhereApplicationAnno = "tkestack.io/anywhere-application"
 )
 
 // KubeVendorType describe the kubernetes provider of the cluster

pkg/platform/provider/baremetal/cluster/create.go

@@ -51,6 +51,7 @@ import (
 	kubeaggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 	utilsnet "k8s.io/utils/net"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	application "tkestack.io/tke/api/application/v1"
 	platformv1 "tkestack.io/tke/api/platform/v1"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/constants"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/images"
@@ -74,9 +75,9 @@ import (
 	"tkestack.io/tke/pkg/platform/provider/util/mark"
 	v1 "tkestack.io/tke/pkg/platform/types/v1"
 	"tkestack.io/tke/pkg/util/apiclient"
-	"tkestack.io/tke/pkg/util/clusternet"
 	"tkestack.io/tke/pkg/util/cmdstring"
 	containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
+	"tkestack.io/tke/pkg/util/extenderapi"
 	"tkestack.io/tke/pkg/util/hosts"
 	"tkestack.io/tke/pkg/util/log"
 	"tkestack.io/tke/pkg/util/template"
@@ -1586,11 +1587,11 @@ func (p *Provider) EnsureAnywhereEdtion(ctx context.Context, c *v1.Cluster) erro
 	if err != nil {
 		return err
 	}
-	hubClient, err := clusternet.GetHubClient(config)
+	hubClient, err := extenderapi.GetExtenderClient(config)
 	if err != nil {
 		return err
 	}
-	current, err := clusternet.GetManagedCluster(hubClient, c.Name)
+	current, err := extenderapi.GetManagedCluster(hubClient, c.Name)
 	if err != nil {
 		return err
 	}
@@ -1634,22 +1635,22 @@ func (p *Provider) EnsureCheckAnywhereSubscription(ctx context.Context, c *v1.Cl
 	if err != nil {
 		return err
 	}
-	hubClient, err := clusternet.GetHubClient(config)
+	hubClient, err := extenderapi.GetExtenderClient(config)
 	if err != nil {
 		return err
 	}
-	mcls, err := clusternet.GetManagedCluster(hubClient, c.Name)
+	mcls, err := extenderapi.GetManagedCluster(hubClient, c.Name)
 	if err != nil {
 		return err
 	}
-	sub, err := clusternet.GetSubscription(hubClient, c.Annotations[platformv1.AnywhereSubscriptionNameAnno], c.Annotations[platformv1.AnywhereSubscriptionNamespaceAnno])
+	sub, err := extenderapi.GetSubscription(hubClient, c.Annotations[platformv1.AnywhereSubscriptionNameAnno], c.Annotations[platformv1.AnywhereSubscriptionNamespaceAnno])
 	if err != nil {
 		return err
 	}
 	for _, feed := range sub.Spec.Feeds {
 		wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
 			var helmrelease *appsv1alpha1.HelmRelease
-			helmrelease, err = clusternet.GetHelmRelease(hubClient, clusternet.GenerateHelmReleaseName(sub.Name, feed), mcls.Namespace)
+			helmrelease, err = extenderapi.GetHelmRelease(hubClient, extenderapi.GenerateHelmReleaseName(sub.Name, feed), mcls.Namespace)
 			if err != nil {
 				err = fmt.Errorf("get helmrelease %s failed: %v", feed.Name, err)
 				return false, nil
@@ -1668,6 +1669,38 @@ func (p *Provider) EnsureCheckAnywhereSubscription(ctx context.Context, c *v1.Cl
 
 }
 
+// Ensure anywhere addon applications
+func (p *Provider) EnsureAnywhereAddons(ctx context.Context, c *v1.Cluster) error {
+	config, err := c.RESTConfig()
+	if err != nil {
+		return err
+	}
+	extenderClient, err := extenderapi.GetExtenderClient(config)
+	if err != nil {
+		return err
+	}
+	if c.Annotations[platformv1.AnywhereApplicationAnno] != "" {
+		applicationJSON, err := base64.StdEncoding.DecodeString(c.Annotations[platformv1.AnywhereApplicationAnno])
+		if err != nil {
+			return fmt.Errorf("decode application JSON failed: %v", err)
+
+		}
+		applications := &application.AppList{}
+		err = json.Unmarshal(applicationJSON, applications)
+		if err != nil {
+			return fmt.Errorf("unmarshal application failed %v", err)
+		}
+
+		for _, app := range applications.Items {
+			err := extenderClient.Create(ctx, &app)
+			if err != nil && !apierrors.IsAlreadyExists(err) {
+				return fmt.Errorf("create application %+v failed: %v", app, err)
+			}
+		}
+	}
+	return nil
+}
+
 // update cluster to connect remote cluster apiserver
 func (p *Provider) EnsureModifyCluster(ctx context.Context, c *v1.Cluster) error {
 	var hubAPIServerURL *url.URL
@@ -1685,11 +1718,11 @@ func (p *Provider) EnsureModifyCluster(ctx context.Context, c *v1.Cluster) error
 	if err != nil {
 		return err
 	}
-	hubClient, err := clusternet.GetHubClient(config)
+	hubClient, err := extenderapi.GetExtenderClient(config)
 	if err != nil {
 		return err
 	}
-	currentManagerCluster, err := clusternet.GetManagedCluster(hubClient, c.Name)
+	currentManagerCluster, err := extenderapi.GetManagedCluster(hubClient, c.Name)
 	if err != nil {
 		return err
 	}
@@ -1714,11 +1747,11 @@ func (p *Provider) EnsureModifyClusterCredential(ctx context.Context, c *v1.Clus
 	if err != nil {
 		return err
 	}
-	hubClient, err := clusternet.GetHubClient(config)
+	hubClient, err := extenderapi.GetExtenderClient(config)
 	if err != nil {
 		return err
 	}
-	currentManagerCluster, err := clusternet.GetManagedCluster(hubClient, c.Name)
+	currentManagerCluster, err := extenderapi.GetManagedCluster(hubClient, c.Name)
 	if err != nil {
 		return err
 	}

pkg/util/clusternet/clusternet.go → pkg/util/extenderapi/extenderapi.go

@@ -1,15 +1,16 @@
-package clusternet
+package extenderapi
 
 import (
 	"context"
 	"errors"
 	"fmt"
-
 	appsv1alpha1 "github.com/clusternet/apis/apps/v1alpha1"
 	clustersv1beta1 "github.com/clusternet/apis/clusters/v1beta1"
-	"k8s.io/apimachinery/pkg/runtime"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	application "tkestack.io/tke/api/application/v1"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	runtimeutil "k8s.io/apimachinery/pkg/util/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"k8s.io/client-go/rest"
@@ -20,33 +21,33 @@ var (
 )
 
 func init() {
-	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-
-	utilruntime.Must(clustersv1beta1.AddToScheme(scheme))
-	utilruntime.Must(appsv1alpha1.AddToScheme(scheme))
+	runtimeutil.Must(clientgoscheme.AddToScheme(scheme))
+	runtimeutil.Must(application.AddToScheme(scheme))
+	runtimeutil.Must(clustersv1beta1.AddToScheme(scheme))
+	runtimeutil.Must(appsv1alpha1.AddToScheme(scheme))
 }
 
-func GetHubClient(config *rest.Config) (client.Client, error) {
+func GetExtenderClient(config *rest.Config) (client.Client, error) {
 	var err error
 
 	if config == nil {
-		return nil, errors.New("empty  hub restconfig file")
+		return nil, errors.New("kube restconfig file is empty")
 	}
 
 	config.ContentConfig.ContentType = "application/json"
 
 	if err != nil {
-		return nil, fmt.Errorf("fail to get hub cluster rest config ,err is %v", err)
+		return nil, fmt.Errorf("failed to get cluster rest config with err: %v", err)
 	}
 
-	clusternetClient, err := client.New(config, client.Options{
+	extenderClient, err := client.New(config, client.Options{
 		Scheme: scheme,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("fail to build a clusternet clien, error is %v", err)
+		return nil, fmt.Errorf("failed to create cluster client with error: %v", err)
 	}
 
-	return clusternetClient, nil
+	return extenderClient, nil
 }
 
 func GetManagedCluster(clientSet client.Client, name string) (*clustersv1beta1.ManagedCluster, error) {