diff --git a/cmd/tke-installer/app/config/config.go b/cmd/tke-installer/app/config/config.go index 3456bbbfd0..f4d0a639c1 100644 --- a/cmd/tke-installer/app/config/config.go +++ b/cmd/tke-installer/app/config/config.go @@ -20,7 +20,9 @@ package config import ( "helm.sh/helm/v3/pkg/chartutil" + "k8s.io/apimachinery/pkg/util/wait" "tkestack.io/tke/cmd/tke-installer/app/options" + helmaction "tkestack.io/tke/pkg/application/helm/action" clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster" "tkestack.io/tke/pkg/util/log" ) @@ -52,9 +54,10 @@ type Config struct { EnableCustomExpansion bool // CustomExpansionDir path to expansions. default `data/expansions` CustomExpansionDir string + ExpansionApps []ExpansionApp PlatformApps []PlatformApp } -type PlatformApp struct { +type ExpansionApp struct { Name string Enable bool Chart Chart @@ -75,6 +78,14 @@ type Chart struct { Values chartutil.Values } +type PlatformApp struct { + HelmInstallOptions helmaction.InstallOptions + LocalChartPath string + ConditionFunc wait.ConditionFunc + Enable bool + Installed bool +} + // CreateConfigFromOptions creates a running configuration instance based // on a given TKE apiserver command line or configuration file option. func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config, error) { diff --git a/cmd/tke-installer/app/installer/application.go b/cmd/tke-installer/app/installer/application.go index 3019fbb257..d78b73bf9b 100644 --- a/cmd/tke-installer/app/installer/application.go +++ b/cmd/tke-installer/app/installer/application.go @@ -21,25 +21,30 @@ package installer import ( "context" "fmt" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" applicationv1 "tkestack.io/tke/api/application/v1" "tkestack.io/tke/cmd/tke-installer/app/config" "tkestack.io/tke/cmd/tke-installer/app/installer/constants" + helmaction "tkestack.io/tke/pkg/application/helm/action" + "tkestack.io/tke/pkg/util/apiclient" ) -func (t *TKE) completePlatformApps() error { +func (t *TKE) completeExpansionApps() error { - if len(t.Config.PlatformApps) == 0 { + if len(t.Config.ExpansionApps) == 0 { return nil } - for _, platformApp := range t.Config.PlatformApps { - if !platformApp.Enable { + for _, expansionApp := range t.Config.ExpansionApps { + if !expansionApp.Enable { continue } - err := t.completeChart(&platformApp.Chart) + err := t.completeChart(&expansionApp.Chart) if err != nil { - return fmt.Errorf("bad platform app config. %v, %v", platformApp.Name, err) + return fmt.Errorf("bad platform app config. %v, %v", expansionApp.Name, err) } } @@ -79,7 +84,7 @@ func (t *TKE) completeChart(chart *config.Chart) error { func (t *TKE) installApplications(ctx context.Context) error { - if len(t.Config.PlatformApps) == 0 { + if len(t.Config.ExpansionApps) == 0 { return nil } @@ -97,38 +102,38 @@ func (t *TKE) installApplications(ctx context.Context) error { return fmt.Errorf("list all applications failed %v", err) } - for _, platformApp := range t.Config.PlatformApps { - if !platformApp.Enable { + for _, expansionApp := range t.Config.ExpansionApps { + if !expansionApp.Enable { continue } - if t.applicationAlreadyInstalled(platformApp, apps.Items) { - t.log.Infof("application already exists. we don't override applications while installing. %v/%v", platformApp.Chart.TargetNamespace, platformApp.Chart.Name) + if t.applicationAlreadyInstalled(expansionApp, apps.Items) { + t.log.Infof("application already exists. we don't override applications while installing. %v/%v", expansionApp.Chart.TargetNamespace, expansionApp.Chart.Name) continue } - err := t.installApplication(ctx, platformApp) + err := t.installApplication(ctx, expansionApp) if err != nil { - return fmt.Errorf("install application failed. %v, %v", platformApp.Name, err) + return fmt.Errorf("install application failed. %v, %v", expansionApp.Name, err) } - t.log.Infof("finish application installation %v", platformApp.Name) + t.log.Infof("finish application installation %v", expansionApp.Name) } return nil } -func (t *TKE) applicationAlreadyInstalled(platformApp config.PlatformApp, installedApps []applicationv1.App) bool { +func (t *TKE) applicationAlreadyInstalled(expansionApp config.ExpansionApp, installedApps []applicationv1.App) bool { for _, installedApp := range installedApps { // if there's an existed application with the same namespace+name, we consider it as already exists - if platformApp.Name == installedApp.Spec.Name && platformApp.Chart.TargetNamespace == installedApp.Namespace { + if expansionApp.Name == installedApp.Spec.Name && expansionApp.Chart.TargetNamespace == installedApp.Namespace { return true } } return false } -func (t *TKE) installApplication(ctx context.Context, platformApp config.PlatformApp) error { +func (t *TKE) installApplication(ctx context.Context, expansionApp config.ExpansionApp) error { - chart := platformApp.Chart + chart := expansionApp.Chart rawValues, err := chart.Values.YAML() if err != nil { @@ -143,7 +148,7 @@ func (t *TKE) installApplication(ctx context.Context, platformApp config.Platfor Spec: applicationv1.AppSpec{ Type: constants.DefaultApplicationInstallDriverType, TenantID: chart.TenantID, - Name: platformApp.Name, + Name: expansionApp.Name, TargetCluster: chart.TargetCluster, Chart: applicationv1.Chart{ TenantID: chart.TenantID, @@ -163,3 +168,116 @@ func (t *TKE) installApplication(ctx context.Context, platformApp config.Platfor } return nil } +func (t *TKE) initPlatformApps(ctx context.Context) error { + defaultPlatformApps := []config.PlatformApp{} + if t.Para.Config.Auth.TKEAuth != nil { + authApiOptions, err := t.getTKEAuthAPIOptions(ctx) + if err != nil { + return fmt.Errorf("get tke-auth-api options failed: %v", err) + } + tkeAuth := config.PlatformApp{ + HelmInstallOptions: helmaction.InstallOptions{ + Namespace: t.namespace, + ReleaseName: "tke-auth", + Values: map[string]interface{}{ + "api": authApiOptions, + "controller": t.getTKEAuthControllerOptions(ctx), + }, + DependencyUpdate: false, + ChartPathOptions: helmaction.ChartPathOptions{}, + }, + LocalChartPath: constants.ChartDirName + "tke-auth/", + Enable: true, + ConditionFunc: func() (bool, error) { + apiOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-auth-api") + if err != nil { + return false, nil + } + controllerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-auth-controller") + if err != nil { + return false, nil + } + return apiOk && controllerOk, nil + }, + } + defaultPlatformApps = append(defaultPlatformApps, tkeAuth) + } + platformApiOptions, err := t.getTKEPlatformAPIOptions(ctx) + if err != nil { + return fmt.Errorf("get tke-platform-api options failed: %v", err) + } + tkePlatform := config.PlatformApp{ + HelmInstallOptions: helmaction.InstallOptions{ + Namespace: t.namespace, + ReleaseName: "tke-platform", + Values: map[string]interface{}{ + "api": platformApiOptions, + "controller": t.getTKEPlatformControllerOptions(ctx), + }, + DependencyUpdate: false, + ChartPathOptions: helmaction.ChartPathOptions{}, + }, + LocalChartPath: constants.ChartDirName + "tke-platform/", + Enable: true, + ConditionFunc: func() (bool, error) { + apiOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-platform-api") + if err != nil { + return false, nil + } + controllerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-platform-controller") + if err != nil { + return false, nil + } + return apiOk && controllerOk, nil + }, + } + defaultPlatformApps = append(defaultPlatformApps, tkePlatform) + t.Config.PlatformApps = append(defaultPlatformApps, t.Config.PlatformApps...) + return nil +} + +func (t *TKE) installPlatformApps(ctx context.Context) error { + + if len(t.Config.PlatformApps) == 0 { + return nil + } + for i, platformApp := range t.Config.PlatformApps { + if !platformApp.Enable || platformApp.Installed { + continue + } + t.log.Infof("Start instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) + err := t.installPlatformApp(ctx, platformApp) + if err != nil { + t.log.Errorf("Install %s failed", platformApp.HelmInstallOptions.ReleaseName) + } + t.Config.PlatformApps[i].Installed = true + t.log.Infof("End instal platform app %s in %s namespace", platformApp.HelmInstallOptions.ReleaseName, platformApp.HelmInstallOptions.Namespace) + } + + return nil +} + +func (t *TKE) installPlatformApp(ctx context.Context, platformApp config.PlatformApp) error { + platformApp.HelmInstallOptions.Timeout = 10 * time.Minute + if len(platformApp.LocalChartPath) != 0 { + if _, err := t.helmClient.InstallWithLocal(&platformApp.HelmInstallOptions, platformApp.LocalChartPath); err != nil { + uninstallOptions := helmaction.UninstallOptions{ + Timeout: 10 * time.Minute, + ReleaseName: platformApp.HelmInstallOptions.ReleaseName, + Namespace: platformApp.HelmInstallOptions.Namespace, + } + reponse, err := t.helmClient.Uninstall(&uninstallOptions) + if err != nil { + return fmt.Errorf("clean %s failed %v", reponse.Release.Name, err) + } + return err + } + } + if platformApp.ConditionFunc != nil { + err := wait.PollImmediate(5*time.Second, 10*time.Minute, platformApp.ConditionFunc) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/tke-installer/app/installer/application_test.go b/cmd/tke-installer/app/installer/application_test.go index 80e7788ce2..1d36ac2d15 100644 --- a/cmd/tke-installer/app/installer/application_test.go +++ b/cmd/tke-installer/app/installer/application_test.go @@ -3,10 +3,11 @@ package installer import ( "context" "fmt" - "helm.sh/helm/v3/pkg/chartutil" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "os" "testing" + + "helm.sh/helm/v3/pkg/chartutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" registryv1 "tkestack.io/tke/api/registry/v1" "tkestack.io/tke/cmd/tke-installer/app/config" helmaction "tkestack.io/tke/pkg/application/helm/action" @@ -61,21 +62,21 @@ func TestTKE_installApplication(t *testing.T) { TargetNamespace: "default", Values: chartutil.Values{}, } - platformApp := config.PlatformApp{ + expansionApp := config.ExpansionApp{ Name: name, Enable: true, Chart: *chart, } - if tke.applicationAlreadyInstalled(platformApp, apps.Items) { + if tke.applicationAlreadyInstalled(expansionApp, apps.Items) { t.Log("already installed") } else { - err = tke.installApplication(context.Background(), platformApp) + err = tke.installApplication(context.Background(), expansionApp) if err != nil { t.Fatal(err) } } - b, err := json.MarshalIndent([]config.PlatformApp{ + b, err := json.MarshalIndent([]config.ExpansionApp{ { Name: name, Enable: true, diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go index a69edee483..48b2329428 100644 --- a/cmd/tke-installer/app/installer/installer.go +++ b/cmd/tke-installer/app/installer/installer.go @@ -278,19 +278,14 @@ func (t *TKE) initSteps() { }, }...) - if t.Para.Config.Auth.TKEAuth != nil { - t.steps = append(t.steps, []types.Handler{ - { - Name: "Install tke-auth chart", - Func: t.installTKEAuthChart, - }, - }...) - } - t.steps = append(t.steps, []types.Handler{ { - Name: "Install tke-platform chart", - Func: t.installTKEPlatformChart, + Name: "Init Platform Applications", + Func: t.initPlatformApps, + }, + { + Name: "Install Platform Applications", + Func: t.installPlatformApps, }, }...) @@ -474,7 +469,7 @@ func (t *TKE) initSteps() { }...) } - if len(t.Config.PlatformApps) > 0 { + if len(t.Config.ExpansionApps) > 0 { t.steps = append(t.steps, []types.Handler{ { Name: "Install Applications", @@ -668,7 +663,7 @@ func (t *TKE) prepare() apierrors.APIStatus { } } - err = t.completePlatformApps() + err = t.completeExpansionApps() if err != nil { return apierrors.NewInternalError(err) } @@ -1693,52 +1688,6 @@ func (t *TKE) installETCD(ctx context.Context) error { return apiclient.CreateResourceWithDir(ctx, t.globalClient, "manifests/etcd/*.yaml", nil) } -func (t *TKE) installTKEAuthChart(ctx context.Context) error { - apiOptions, err := t.getTKEAuthAPIOptions(ctx) - if err != nil { - return err - } - values := map[string]interface{}{ - "api": apiOptions, - "controller": t.getTKEAuthControllerOptions(ctx), - } - chartPathOptions := &helmaction.ChartPathOptions{} - installOptions := &helmaction.InstallOptions{ - Namespace: t.namespace, - ReleaseName: "tke-auth", - DependencyUpdate: false, - Values: values, - Timeout: 10 * time.Minute, - ChartPathOptions: *chartPathOptions, - } - - chartFilePath := constants.ChartDirName + "tke-auth/" - if _, err := t.helmClient.InstallWithLocal(installOptions, chartFilePath); err != nil { - uninstallOptions := helmaction.UninstallOptions{ - Timeout: 10 * time.Minute, - ReleaseName: "tke-auth", - Namespace: t.namespace, - } - reponse, err := t.helmClient.Uninstall(&uninstallOptions) - if err != nil { - return fmt.Errorf("%s uninstall fail, err = %s", reponse.Release.Name, err.Error()) - } - return err - } - - return wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { - apiOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-auth-api") - if err != nil { - return false, nil - } - controllerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-auth-controller") - if err != nil { - return false, nil - } - return apiOk && controllerOk, nil - }) -} - func (t *TKE) getTKEAuthAPIOptions(ctx context.Context) (map[string]interface{}, error) { redirectHosts := t.servers redirectHosts = append(redirectHosts, "tke-gateway") @@ -1814,52 +1763,6 @@ func (t *TKE) installTKEAudit(ctx context.Context) error { }) } -func (t *TKE) installTKEPlatformChart(ctx context.Context) error { - apiOptions, err := t.getTKEPlatformAPIOptions(ctx) - if err != nil { - return err - } - values := map[string]interface{}{ - "api": apiOptions, - "controller": t.getTKEPlatformControllerOptions(ctx), - } - chartPathOptions := &helmaction.ChartPathOptions{} - installOptions := &helmaction.InstallOptions{ - Namespace: t.namespace, - ReleaseName: "tke-platform", - DependencyUpdate: false, - Values: values, - Timeout: 10 * time.Minute, - ChartPathOptions: *chartPathOptions, - } - - chartFilePath := constants.ChartDirName + "tke-platform/" - if _, err := t.helmClient.InstallWithLocal(installOptions, chartFilePath); err != nil { - uninstallOptions := helmaction.UninstallOptions{ - Timeout: 10 * time.Minute, - ReleaseName: "tke-platform", - Namespace: t.namespace, - } - reponse, err := t.helmClient.Uninstall(&uninstallOptions) - if err != nil { - return fmt.Errorf("%s uninstall fail, err = %s", reponse.Release.Name, err.Error()) - } - return err - } - - return wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { - apiOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-platform-api") - if err != nil { - return false, nil - } - controllerOk, err := apiclient.CheckDeployment(ctx, t.globalClient, t.namespace, "tke-platform-controller") - if err != nil { - return false, nil - } - return apiOk && controllerOk, nil - }) -} - func (t *TKE) getTKEPlatformAPIOptions(ctx context.Context) (map[string]interface{}, error) { cacrt, err := ioutil.ReadFile(constants.DataDir + "ca.crt") if err != nil {