diff --git a/cmd/tke-installer/app/config/config.go b/cmd/tke-installer/app/config/config.go index 9ea22617c..1d45cda67 100644 --- a/cmd/tke-installer/app/config/config.go +++ b/cmd/tke-installer/app/config/config.go @@ -19,11 +19,7 @@ package config import ( - "helm.sh/helm/v3/pkg/chartutil" - "k8s.io/apimachinery/pkg/util/wait" - applicationv1 "tkestack.io/tke/api/application/v1" "tkestack.io/tke/cmd/tke-installer/app/options" - helmaction "tkestack.io/tke/pkg/application/helm/action" clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster" "tkestack.io/tke/pkg/util/log" ) @@ -55,41 +51,6 @@ type Config struct { EnableCustomExpansion bool // CustomExpansionDir path to expansions. default `data/expansions` CustomExpansionDir string - ExpansionApps []ExpansionApp - PlatformApps []PlatformApp -} -type ExpansionApp struct { - Name string - Enable bool - Chart Chart -} - -type Chart struct { - Name string - TenantID string - ChartGroupName string - // install options - Version string - // install options - TargetCluster string - // install options - TargetNamespace string - // install options - // chartutil.ReadValues/ReadValuesFile - Values chartutil.Values -} - -type PlatformApp struct { - HelmInstallOptions helmaction.InstallOptions - LocalChartPath string - ConditionFunc wait.ConditionFunc - Enable bool - Installed bool - // rawValues: json format or yaml format - RawValues string - RawValuesType applicationv1.RawValuesType - // values: can specify multiple or separate values: key1=val1,key2=val2 - Values []string } // CreateConfigFromOptions creates a running configuration instance based diff --git a/cmd/tke-installer/app/installer/application.go b/cmd/tke-installer/app/installer/application.go index cdbed4d9e..1ee028194 100644 --- a/cmd/tke-installer/app/installer/application.go +++ b/cmd/tke-installer/app/installer/application.go @@ -21,13 +21,15 @@ package installer import ( "context" "fmt" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" applicationv1 "tkestack.io/tke/api/application/v1" - "tkestack.io/tke/cmd/tke-installer/app/config" "tkestack.io/tke/cmd/tke-installer/app/installer/constants" + "tkestack.io/tke/cmd/tke-installer/app/installer/images" + "tkestack.io/tke/cmd/tke-installer/app/installer/types" helmaction "tkestack.io/tke/pkg/application/helm/action" helmutil "tkestack.io/tke/pkg/application/helm/util" "tkestack.io/tke/pkg/util/apiclient" @@ -35,11 +37,11 @@ import ( func (t *TKE) completeExpansionApps() error { - if len(t.Config.ExpansionApps) == 0 { + if len(t.Para.Config.ExpansionApps) == 0 { return nil } - for _, expansionApp := range t.Config.ExpansionApps { + for _, expansionApp := range t.Para.Config.ExpansionApps { if !expansionApp.Enable { continue } @@ -54,7 +56,7 @@ func (t *TKE) completeExpansionApps() error { return nil } -func (t *TKE) completeChart(chart *config.Chart) error { +func (t *TKE) completeChart(chart *types.Chart) error { _, err := chart.Values.YAML() if err != nil { @@ -85,7 +87,7 @@ func (t *TKE) completeChart(chart *config.Chart) error { func (t *TKE) installApplications(ctx context.Context) error { - if len(t.Config.ExpansionApps) == 0 { + if len(t.Para.Config.ExpansionApps) == 0 { return nil } @@ -103,7 +105,7 @@ func (t *TKE) installApplications(ctx context.Context) error { return fmt.Errorf("list all applications failed %v", err) } - for _, expansionApp := range t.Config.ExpansionApps { + for _, expansionApp := range t.Para.Config.ExpansionApps { if !expansionApp.Enable { continue } @@ -121,7 +123,7 @@ func (t *TKE) installApplications(ctx context.Context) error { return nil } -func (t *TKE) applicationAlreadyInstalled(expansionApp config.ExpansionApp, installedApps []applicationv1.App) bool { +func (t *TKE) applicationAlreadyInstalled(expansionApp *types.ExpansionApp, installedApps []applicationv1.App) bool { for _, installedApp := range installedApps { // if there's an existed application with the same namespace+name, we consider it as already exists @@ -132,7 +134,7 @@ func (t *TKE) applicationAlreadyInstalled(expansionApp config.ExpansionApp, inst return false } -func (t *TKE) installApplication(ctx context.Context, expansionApp config.ExpansionApp) error { +func (t *TKE) installApplication(ctx context.Context, expansionApp *types.ExpansionApp) error { chart := expansionApp.Chart @@ -170,14 +172,14 @@ func (t *TKE) installApplication(ctx context.Context, expansionApp config.Expans return nil } func (t *TKE) initPlatformApps(ctx context.Context) error { - defaultPlatformApps := []config.PlatformApp{} + defaultPlatformApps := []*types.PlatformApp{} if t.Para.Config.Auth.TKEAuth != nil { authAPIOptions, err := t.getTKEAuthAPIOptions(ctx) if err != nil { return fmt.Errorf("get tke-auth-api options failed: %v", err) } - tkeAuth := config.PlatformApp{ - HelmInstallOptions: helmaction.InstallOptions{ + tkeAuth := &types.PlatformApp{ + HelmInstallOptions: &helmaction.InstallOptions{ Namespace: t.namespace, ReleaseName: "tke-auth", Values: map[string]interface{}{ @@ -207,8 +209,8 @@ func (t *TKE) initPlatformApps(ctx context.Context) error { if err != nil { return fmt.Errorf("get tke-platform-api options failed: %v", err) } - tkePlatform := config.PlatformApp{ - HelmInstallOptions: helmaction.InstallOptions{ + tkePlatform := &types.PlatformApp{ + HelmInstallOptions: &helmaction.InstallOptions{ Namespace: t.namespace, ReleaseName: "tke-platform", Values: map[string]interface{}{ @@ -233,53 +235,151 @@ func (t *TKE) initPlatformApps(ctx context.Context) error { }, } defaultPlatformApps = append(defaultPlatformApps, tkePlatform) - t.Config.PlatformApps = append(defaultPlatformApps, t.Config.PlatformApps...) + t.Para.Config.PlatformApps = append(defaultPlatformApps, t.Para.Config.PlatformApps...) + return nil +} + +func (t *TKE) preprocessPlatformApps(ctx context.Context) error { + + for _, platformApp := range t.Para.Config.PlatformApps { + if !platformApp.Enable || platformApp.Installed { + continue + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.CephRBDChartReleaseName) { + platformApp.ConditionFunc = func() (bool, error) { + provisionerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, platformApp.HelmInstallOptions.Namespace, "ceph-csi-rbd-provisioner") + if err != nil { + return false, nil + } + nodepluginOk, err := apiclient.CheckDaemonset(ctx, t.globalClient, platformApp.HelmInstallOptions.Namespace, "ceph-csi-rbd-nodeplugin") + if err != nil { + return false, nil + } + return provisionerOk && nodepluginOk, nil + } + platformApp.LocalChartPath = constants.ChartDirName + "ceph-csi-rbd/" + + if err := t.mergePlatformAppValues(platformApp); err != nil { + return err + } + values := platformApp.HelmInstallOptions.Values + if values["csiConfig"] == nil { + return fmt.Errorf("ceph-csi-rbd platformAPP csiConfig nil") + } + if values["secret"] == nil || len(values["secret"].(map[string]interface{})["userID"].(string)) == 0 || len(values["secret"].(map[string]interface{})["userKey"].(string)) == 0 { + return fmt.Errorf("ceph-csi-rbd platformAPP secret userID | userKey nil") + } + if values["storageClass"] == nil || len(values["storageClass"].(map[string]interface{})["clusterID"].(string)) == 0 { + return fmt.Errorf("ceph-csi-rbd platformAPP storageClass clusterID nil") + } + + values["images"] = map[string]interface{}{ + "enable": true, + "nodeplugin": map[string]interface{}{ + "registrar": images.Get().CsiNodeDriverRegistrar.FullName(), + "plugin": images.Get().CephCsi.FullName(), + }, + "provisioner": map[string]interface{}{ + "provisioner": images.Get().CsiNodeDriverRegistrar.FullName(), + "attacher": images.Get().CsiAttacher.FullName(), + "resizer": images.Get().CsiResizer.FullName(), + "snapshotter": images.Get().CsiSnapshotter.FullName(), + }, + } + if values["storageClass"] == nil { + values["storageClass"] = map[string]interface{}{ + "replicaCount": t.Config.Replicas, + } + } else { + values["storageClass"].(map[string]interface{})["replicaCount"] = t.Config.Replicas + } + values["namespace"] = platformApp.HelmInstallOptions.Namespace + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.NFSChartReleaseName) { + platformApp.ConditionFunc = func() (bool, error) { + provisionerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, platformApp.HelmInstallOptions.Namespace, "nfs-subdir-external-provisioner") + if err != nil { + return false, nil + } + return provisionerOk, nil + } + platformApp.LocalChartPath = constants.ChartDirName + "nfs-subdir-external-provisioner/" + + if err := t.mergePlatformAppValues(platformApp); err != nil { + return err + } + values := platformApp.HelmInstallOptions.Values + if values == nil || values["nfs"] == nil || len(values["nfs"].(map[string]interface{})["server"].(string)) == 0 || len(values["nfs"].(map[string]interface{})["path"].(string)) == 0 { + return fmt.Errorf("nfs-subdir-external-provisioner platformApp lack nfs.server or nfs.path") + } + values["replicaCount"] = t.Config.Replicas + values["namespace"] = platformApp.HelmInstallOptions.Namespace + if values["images"] == nil { + values["images"] = map[string]interface{}{ + "repository": images.Get().NFSProvisioner.FullName(), + } + } else { + values["images"].(map[string]interface{})["repository"] = images.Get().NFSProvisioner.FullName() + } + } + } + return nil +} + +func (t *TKE) mergePlatformAppValues(platformApp *types.PlatformApp) error { + if len(platformApp.RawValues) != 0 || len(platformApp.Values) != 0 { + values, err := helmutil.MergeValues(platformApp.Values, platformApp.RawValues, string(platformApp.RawValuesType)) + if err != nil { + return err + } + platformApp.HelmInstallOptions.Values = values + platformApp.RawValues = "" + platformApp.Values = nil + } return nil } func (t *TKE) installPlatformApps(ctx context.Context) error { - if len(t.Config.PlatformApps) == 0 { + if len(t.Para.Config.PlatformApps) == 0 { return nil } - for i, platformApp := range t.Config.PlatformApps { + for i, platformApp := range t.Para.Config.PlatformApps { if !platformApp.Enable || platformApp.Installed { continue } - t.log.Infof("Start instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) err := t.installPlatformApp(ctx, platformApp) if err != nil { t.log.Errorf("Install %s failed", platformApp.HelmInstallOptions.ReleaseName) + return err } - t.Config.PlatformApps[i].Installed = true - t.log.Infof("End instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) + t.Para.Config.PlatformApps[i].Installed = true } return nil } -func (t *TKE) installPlatformApp(ctx context.Context, platformApp config.PlatformApp) error { +func (t *TKE) installPlatformApp(ctx context.Context, platformApp *types.PlatformApp) error { platformApp.HelmInstallOptions.Timeout = 10 * time.Minute - if len(platformApp.RawValues) != 0 || len(platformApp.Values) != 0 { - values, err := helmutil.MergeValues(platformApp.Values, platformApp.RawValues, string(platformApp.RawValuesType)) - if err != nil { - return err - } - platformApp.HelmInstallOptions.Values = values + if err := t.mergePlatformAppValues(platformApp); err != nil { + return err } + // TODO currently only support local chart install if len(platformApp.LocalChartPath) != 0 { - if _, err := t.helmClient.InstallWithLocal(&platformApp.HelmInstallOptions, platformApp.LocalChartPath); err != nil { + t.log.Infof("Start instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) + if _, err := t.helmClient.InstallWithLocal(platformApp.HelmInstallOptions, platformApp.LocalChartPath); err != nil { uninstallOptions := helmaction.UninstallOptions{ Timeout: 10 * time.Minute, ReleaseName: platformApp.HelmInstallOptions.ReleaseName, Namespace: platformApp.HelmInstallOptions.Namespace, } - reponse, err := t.helmClient.Uninstall(&uninstallOptions) - if err != nil { - return fmt.Errorf("clean %s failed %v", reponse.Release.Name, err) + _, uninstallErr := t.helmClient.Uninstall(&uninstallOptions) + if uninstallErr != nil { + return fmt.Errorf("clean %s failed %v", platformApp.HelmInstallOptions.ReleaseName, uninstallErr) } return err } + t.log.Infof("End instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) } if platformApp.ConditionFunc != nil { err := wait.PollImmediate(5*time.Second, 10*time.Minute, platformApp.ConditionFunc) diff --git a/cmd/tke-installer/app/installer/application_test.go b/cmd/tke-installer/app/installer/application_test.go index 1d36ac2d1..a66a0ef51 100644 --- a/cmd/tke-installer/app/installer/application_test.go +++ b/cmd/tke-installer/app/installer/application_test.go @@ -9,7 +9,7 @@ import ( "helm.sh/helm/v3/pkg/chartutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" registryv1 "tkestack.io/tke/api/registry/v1" - "tkestack.io/tke/cmd/tke-installer/app/config" + "tkestack.io/tke/cmd/tke-installer/app/installer/types" helmaction "tkestack.io/tke/pkg/application/helm/action" applicationutil "tkestack.io/tke/pkg/application/util" "tkestack.io/tke/pkg/mesh/util/json" @@ -53,7 +53,7 @@ func TestTKE_installApplication(t *testing.T) { } name := "demo" - chart := &config.Chart{ + chart := &types.Chart{ Name: name, TenantID: "default", ChartGroupName: "public", @@ -62,7 +62,7 @@ func TestTKE_installApplication(t *testing.T) { TargetNamespace: "default", Values: chartutil.Values{}, } - expansionApp := config.ExpansionApp{ + expansionApp := &types.ExpansionApp{ Name: name, Enable: true, Chart: *chart, @@ -76,7 +76,7 @@ func TestTKE_installApplication(t *testing.T) { t.Fatal(err) } } - b, err := json.MarshalIndent([]config.ExpansionApp{ + b, err := json.MarshalIndent([]types.ExpansionApp{ { Name: name, Enable: true, diff --git a/cmd/tke-installer/app/installer/constants/constants.go b/cmd/tke-installer/app/installer/constants/constants.go index 08ec2a7cd..4a03b2d2d 100644 --- a/cmd/tke-installer/app/installer/constants/constants.go +++ b/cmd/tke-installer/app/installer/constants/constants.go @@ -104,3 +104,10 @@ const ( PathForDiskSpaceRequest = "/var/lib" ) + +const ( + CephRBDStorageClassName = "csi-rbd-sc" + CephRBDChartReleaseName = "ceph-csi-rbd" + NFSStorageClassName = "nfs-sc" + NFSChartReleaseName = "nfs-subdir-external-provisioner" +) diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go index 2e767b986..94d7c1514 100644 --- a/cmd/tke-installer/app/installer/images/images.go +++ b/cmd/tke-installer/app/installer/images/images.go @@ -36,6 +36,15 @@ type BaseComponents struct { TKERegistryController containerregistry.Image ProviderRes containerregistry.Image TKEGateway containerregistry.Image + + NFSProvisioner containerregistry.Image + + CsiNodeDriverRegistrar containerregistry.Image + CsiProvisioner containerregistry.Image + CsiAttacher containerregistry.Image + CsiResizer containerregistry.Image + CsiSnapshotter containerregistry.Image + CephCsi containerregistry.Image } type ExComponents struct { @@ -107,6 +116,15 @@ var baseComponents = BaseComponents{ TKERegistryController: containerregistry.Image{Name: "tke-registry-controller", Tag: Version}, ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-3"}, TKEGateway: containerregistry.Image{Name: "tke-gateway", Tag: Version}, + + NFSProvisioner: containerregistry.Image{Name: "nfs-subdir-external-provisioner", Tag: "v4.0.2"}, + + CsiNodeDriverRegistrar: containerregistry.Image{Name: "csi-node-driver-registrar", Tag: "v2.4.0"}, + CsiProvisioner: containerregistry.Image{Name: "csi-provisioner", Tag: "v3.1.0"}, + CsiAttacher: containerregistry.Image{Name: "csi-attacher", Tag: "v3.4.0"}, + CsiResizer: containerregistry.Image{Name: "csi-resizer", Tag: "v1.4.0"}, + CsiSnapshotter: containerregistry.Image{Name: "csi-snapshotter", Tag: "v4.2.0"}, + CephCsi: containerregistry.Image{Name: "cephcsi", Tag: "v3.6.0"}, } var components = Components{baseComponents, exComponents} diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go index 48b232942..c381cbe0d 100644 --- a/cmd/tke-installer/app/installer/installer.go +++ b/cmd/tke-installer/app/installer/installer.go @@ -283,6 +283,10 @@ func (t *TKE) initSteps() { Name: "Init Platform Applications", Func: t.initPlatformApps, }, + { + Name: "Preprocess Platform Applications", + Func: t.preprocessPlatformApps, + }, { Name: "Install Platform Applications", Func: t.installPlatformApps, @@ -292,12 +296,8 @@ func (t *TKE) initSteps() { if t.Para.Config.Registry.TKERegistry != nil { t.steps = append(t.steps, []types.Handler{ { - Name: "Install tke-registry-api", - Func: t.installTKERegistryAPI, - }, - { - Name: "Install tke-registry-controller", - Func: t.installTKERegistryController, + Name: "Install tke-registry chart", + Func: t.installTKERegistryChart, }, }...) } @@ -469,7 +469,7 @@ func (t *TKE) initSteps() { }...) } - if len(t.Config.ExpansionApps) > 0 { + if len(t.Para.Config.ExpansionApps) > 0 { t.steps = append(t.steps, []types.Handler{ { Name: "Install Applications", @@ -1216,7 +1216,11 @@ func (t *TKE) createGlobalCluster(ctx context.Context) error { } func (t *TKE) backup() error { - data, _ := json.MarshalIndent(t, "", " ") + data, err := json.MarshalIndent(t, "", " ") + if err != nil { + t.log.Infof("json marshal tke failed, err = %s", err.Error()) + return err + } return ioutil.WriteFile(constants.ClusterFile, data, 0777) } func (t *TKE) loadImages(ctx context.Context) error { @@ -2107,95 +2111,152 @@ func (t *TKE) installTKENotifyController(ctx context.Context) error { }) } -func (t *TKE) installTKERegistryAPI(ctx context.Context) error { - - node, err := apiclient.GetNodeByMachineIP(ctx, t.globalClient, t.servers[0]) +func (t *TKE) installTKERegistryChart(ctx context.Context) error { + registryAPIOptions, err := t.getTKERegistryAPIOptions(ctx) if err != nil { - return err + return fmt.Errorf("get tke-registry-api options failed: %v", err) + } + registryControllerOptions, err := t.getTKERegistryControllerOptions(ctx) + if err != nil { + return fmt.Errorf("get tke-registry-controller options failed: %v", err) + } + tkeRegistry := &types.PlatformApp{ + HelmInstallOptions: &helmaction.InstallOptions{ + Namespace: t.namespace, + ReleaseName: "tke-registry", + Values: map[string]interface{}{ + "api": registryAPIOptions, + "controller": registryControllerOptions, + }, + DependencyUpdate: false, + ChartPathOptions: helmaction.ChartPathOptions{}, + }, + LocalChartPath: constants.ChartDirName + "tke-registry/", + Enable: true, + ConditionFunc: func() (bool, error) { + apiOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-registry-api") + if err != nil { + return false, nil + } + controllerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-registry-controller") + if err != nil { + return false, nil + } + return apiOk && controllerOk, nil + }, } + return t.installPlatformApp(ctx, tkeRegistry) +} + +func (t *TKE) getTKERegistryAPIOptions(ctx context.Context) (map[string]interface{}, error) { options := map[string]interface{}{ - "Replicas": t.Config.Replicas, - "Image": images.Get().TKERegistryAPI.FullName(), - "NodeName": node.Name, - "AdminUsername": t.Para.Config.Registry.TKERegistry.Username, - "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password), - "EnableAuth": t.Para.Config.Auth.TKEAuth != nil, - "EnableBusiness": t.businessEnabled(), - "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain, - "EnableAudit": t.auditEnabled(), - "HarborEnabled": t.Para.Config.Registry.TKERegistry.HarborEnabled, - "HarborCAFile": t.Para.Config.Registry.TKERegistry.HarborCAFile, + "replicas": t.Config.Replicas, + "namespace": t.namespace, + "image": images.Get().TKERegistryAPI.FullName(), + "adminUsername": t.Para.Config.Registry.TKERegistry.Username, + "adminPassword": string(t.Para.Config.Registry.TKERegistry.Password), + "enableAuth": t.Para.Config.Auth.TKEAuth != nil, + "enableBusiness": t.businessEnabled(), + "domainSuffix": t.Para.Config.Registry.TKERegistry.Domain, + "enableAudit": t.auditEnabled(), + "harborEnabled": t.Para.Config.Registry.TKERegistry.HarborEnabled, + "harborCAFile": t.Para.Config.Registry.TKERegistry.HarborCAFile, } //check if s3 enabled storageConfig := t.Para.Config.Registry.TKERegistry.Storage s3Enabled := (storageConfig != nil && storageConfig.S3 != nil) - options["S3Enabled"] = s3Enabled + options["s3Enabled"] = s3Enabled if s3Enabled { - options["S3Storage"] = storageConfig.S3 + options["s3Storage"] = storageConfig.S3 } //or enable filesystem by default - options["FilesystemEnabled"] = !s3Enabled + options["filesystemEnabled"] = !s3Enabled + if options["filesystemEnabled"] == true { + useCephRbd, useNFS := false, false + for _, platformApp := range t.Para.Config.PlatformApps { + if !platformApp.Enable || !platformApp.Installed { + continue + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.CephRBDChartReleaseName) { + useCephRbd = true + options["cephRbd"] = true + options["cephRbdPVCName"] = "ceph-rbd-registry-pvc" + options["cephRbdStorageClassName"] = constants.CephRBDStorageClassName + break + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.NFSChartReleaseName) { + useNFS = true + options["nfs"] = true + options["nfsPVCName"] = "nfs-registry-pvc" + options["nfsStorageClassName"] = constants.NFSStorageClassName + break + } + } + if !(useCephRbd || useNFS) { + options["baremetalStorage"] = true + node, err := apiclient.GetNodeByMachineIP(ctx, t.globalClient, t.servers[0]) + if err != nil { + return nil, err + } + options["nodeName"] = node.Name + } + } if t.Para.Config.Auth.OIDCAuth != nil { - options["OIDCClientID"] = t.Para.Config.Auth.OIDCAuth.ClientID - options["OIDCIssuerURL"] = t.Para.Config.Auth.OIDCAuth.IssuerURL - options["UseOIDCCA"] = t.Para.Config.Auth.OIDCAuth.CACert != nil - } - err = apiclient.CreateResourceWithDir(ctx, t.globalClient, "manifests/tke-registry-api/*.yaml", options) - if err != nil { - return err + options["oIDCClientID"] = t.Para.Config.Auth.OIDCAuth.ClientID + options["oIDCIssuerURL"] = t.Para.Config.Auth.OIDCAuth.IssuerURL + options["useOIDCCA"] = t.Para.Config.Auth.OIDCAuth.CACert != nil } - return wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { - ok, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-registry-api") - if err != nil { - return false, nil - } - return ok, nil - }) + return options, nil } -func (t *TKE) installTKERegistryController(ctx context.Context) error { +func (t *TKE) getTKERegistryControllerOptions(ctx context.Context) (map[string]interface{}, error) { node, err := apiclient.GetNodeByMachineIP(ctx, t.globalClient, t.servers[0]) if err != nil { - return err + return nil, err } options := map[string]interface{}{ - "Replicas": t.Config.Replicas, - "Image": images.Get().TKERegistryController.FullName(), - "NodeName": node.Name, - "AdminUsername": t.Para.Config.Registry.TKERegistry.Username, - "AdminPassword": string(t.Para.Config.Registry.TKERegistry.Password), - "EnableAuth": t.Para.Config.Auth.TKEAuth != nil, - "EnableBusiness": t.businessEnabled(), - "DomainSuffix": t.Para.Config.Registry.TKERegistry.Domain, - "DefaultChartGroups": defaultChartGroupsStringConfig, + "replicas": t.Config.Replicas, + "image": images.Get().TKERegistryController.FullName(), + "nodeName": node.Name, + "adminUsername": t.Para.Config.Registry.TKERegistry.Username, + "adminPassword": string(t.Para.Config.Registry.TKERegistry.Password), + "enableAuth": t.Para.Config.Auth.TKEAuth != nil, + "enableBusiness": t.businessEnabled(), + "domainSuffix": t.Para.Config.Registry.TKERegistry.Domain, + "defaultChartGroups": defaultChartGroupsStringConfig, } //check if s3 enabled storageConfig := t.Para.Config.Registry.TKERegistry.Storage s3Enabled := (storageConfig != nil && storageConfig.S3 != nil) - options["S3Enabled"] = s3Enabled + options["s3Enabled"] = s3Enabled if s3Enabled { - options["S3Storage"] = storageConfig.S3 + options["s3Storage"] = storageConfig.S3 } //or enable filesystem by default - options["FilesystemEnabled"] = !s3Enabled - - err = apiclient.CreateResourceWithDir(ctx, t.globalClient, "manifests/tke-registry-controller/*.yaml", options) - if err != nil { - return err - } - - return wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { - ok, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-registry-controller") - if err != nil { - return false, nil + options["filesystemEnabled"] = !s3Enabled + if options["filesystemEnabled"] == true { + useCephRbd, useNFS := false, false + for _, platformApp := range t.Para.Config.PlatformApps { + if !platformApp.Enable || !platformApp.Installed { + continue + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.CephRBDChartReleaseName) { + useCephRbd = true + break + } + if strings.EqualFold(platformApp.HelmInstallOptions.ReleaseName, constants.NFSChartReleaseName) { + useNFS = true + break + } } - return ok, nil - }) + options["baremetalStorage"] = !(useCephRbd || useNFS) + } + return options, nil } func (t *TKE) installTKEApplicationAPI(ctx context.Context) error { diff --git a/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml b/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml index 855e3c15e..ff234bf0b 100644 --- a/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml +++ b/cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml @@ -85,7 +85,7 @@ spec: requests: cpu: 50m memory: 128Mi -{{- if .FilesystemEnabled }} +{{- if .baremetalStorage }} nodeName: {{ .NodeName }} {{- end }} volumes: @@ -97,15 +97,59 @@ spec: name: tke-registry-api {{- if .FilesystemEnabled }} - name: registry-data + {{- if .baremetalStorage }} hostPath: path: /var/lib/tke-registry-api type: DirectoryOrCreate + {{- end }} + {{- if .cephRbd }} + persistentVolumeClaim: + claimName: {{ .cephRbdPVCName }} + {{- end }} + {{- if .nfs }} + persistentVolumeClaim: + claimName: {{ .nfsPVCName }} + {{- end }} {{- end }} - name: tke-registry-api-secret secret: secretName: tke-registry-api --- +{{- if .cephRbd }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .cephRbdPVCName }} + namespace: {{ .Namespace }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 40Gi + storageClassName: {{ .cephRbdStorageClassName }} + +--- +{{- end }} +{{- if .nfs }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .nfsPVCName }} + namespace: {{ .Namespace }} +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 40Gi + storageClassName: {{ .nfsStorageClassName }} + +--- +{{- end }} apiVersion: v1 kind: Secret metadata: diff --git a/cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml b/cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml index ee9e5adad..9b20759be 100644 --- a/cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml +++ b/cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml @@ -54,7 +54,7 @@ spec: requests: cpu: 50m memory: 128Mi -{{- if .FilesystemEnabled }} +{{- if .baremetalStorage }} nodeName: {{ .NodeName }} {{- end }} volumes: diff --git a/cmd/tke-installer/app/installer/types/types.go b/cmd/tke-installer/app/installer/types/types.go index a01042535..9b68bc430 100644 --- a/cmd/tke-installer/app/installer/types/types.go +++ b/cmd/tke-installer/app/installer/types/types.go @@ -24,7 +24,11 @@ import ( "github.com/thoas/go-funk" + "helm.sh/helm/v3/pkg/chartutil" + "k8s.io/apimachinery/pkg/util/wait" + applicationv1 "tkestack.io/tke/api/application/v1" v1 "tkestack.io/tke/api/platform/v1" + helmaction "tkestack.io/tke/pkg/application/helm/action" ) type CreateClusterPara struct { @@ -41,18 +45,20 @@ func (c *CreateClusterPara) RegistryIP() string { // Config is the installer config type Config struct { - Basic *Basic `json:"basic"` - Auth Auth `json:"auth"` - Registry Registry `json:"registry"` - Business *Business `json:"business,omitempty"` - Monitor *Monitor `json:"monitor,omitempty"` - Logagent *Logagent `json:"logagent,omitempty"` - HA *HA `json:"ha,omitempty"` - Gateway *Gateway `json:"gateway,omitempty"` - Audit *Audit `json:"audit,omitempty"` - Application *Application `json:"application,omitempty"` - Mesh *Mesh `json:"mesh,omitempty"` - SkipSteps []string `json:"skipSteps,omitempty"` + Basic *Basic `json:"basic"` + Auth Auth `json:"auth"` + Registry Registry `json:"registry"` + Business *Business `json:"business,omitempty"` + Monitor *Monitor `json:"monitor,omitempty"` + Logagent *Logagent `json:"logagent,omitempty"` + HA *HA `json:"ha,omitempty"` + Gateway *Gateway `json:"gateway,omitempty"` + Audit *Audit `json:"audit,omitempty"` + Application *Application `json:"application,omitempty"` + Mesh *Mesh `json:"mesh,omitempty"` + SkipSteps []string `json:"skipSteps,omitempty"` + ExpansionApps []*ExpansionApp `json:"expansionApps,omitempty"` + PlatformApps []*PlatformApp `json:"platformApps,omitempty"` } type Basic struct { @@ -96,6 +102,40 @@ type ElasticSearch struct { Index string `json:"index"` } +type ExpansionApp struct { + Name string + Enable bool + Chart Chart +} + +type Chart struct { + Name string + TenantID string + ChartGroupName string + // install options + Version string + // install options + TargetCluster string + // install options + TargetNamespace string + // install options + // chartutil.ReadValues/ReadValuesFile + Values chartutil.Values +} + +type PlatformApp struct { + HelmInstallOptions *helmaction.InstallOptions + LocalChartPath string + ConditionFunc wait.ConditionFunc `json:"-"` + Enable bool + Installed bool + // rawValues: json format or yaml format + RawValues string + RawValuesType applicationv1.RawValuesType + // values: can specify multiple or separate values: key1=val1,key2=val2 + Values []string +} + func (r *Registry) Domain() string { if r.UserInputRegistry.Domain != "" { return r.UserInputRegistry.Domain