diff --git a/api/platform/types.go b/api/platform/types.go index 9a0808038..56673c15b 100644 --- a/api/platform/types.go +++ b/api/platform/types.go @@ -123,6 +123,8 @@ const ( AnywhereApplicationAnno = "tkestack.io/anywhere-application" // AnywhereValidateAnno is exist, the cluster will always return validate result AnywhereValidateAnno = "tkestack.io/anywhere-validate" + // LocationBasedImagePrefixAnno is exist, the cluster will use it as k8s images prefix + LocationBaseImagePrefixAnno = "tkestack.io/location-based-image-prefix" ) // KubeVendorType describe the kubernetes provider of the cluster diff --git a/api/platform/v1/types.go b/api/platform/v1/types.go index df76a958c..881f46fd4 100644 --- a/api/platform/v1/types.go +++ b/api/platform/v1/types.go @@ -134,6 +134,8 @@ const ( AnywhereApplicationAnno = "tkestack.io/anywhere-application" // AnywhereValidateAnno is exist, the cluster will always return validate result AnywhereValidateAnno = "tkestack.io/anywhere-validate" + // LocationBasedImagePrefixAnno is exist, the cluster will use it as k8s images prefix + LocationBasedImagePrefixAnno = "tkestack.io/location-based-image-prefix" ) // KubeVendorType describe the kubernetes provider of the cluster diff --git a/build/docker/tools/provider-res/Makefile b/build/docker/tools/provider-res/Makefile index 4bd0b5bc0..f4fe1a4d5 100644 --- a/build/docker/tools/provider-res/Makefile +++ b/build/docker/tools/provider-res/Makefile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations under the License. IMAGE = provider-res -VERSION = v1.21.4-4 +VERSION = v1.21.4-5 REGISTRY_PREFIX ?= tkestack COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) diff --git a/build/docker/tools/tke-installer/release.sh b/build/docker/tools/tke-installer/release.sh index 834c5aa40..a066b5efa 100755 --- a/build/docker/tools/tke-installer/release.sh +++ b/build/docker/tools/tke-installer/release.sh @@ -23,7 +23,7 @@ set -o pipefail REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack} BUILDER=${BUILDER:-default} VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')} -PROVIDER_RES_VERSION=v1.21.4-4 +PROVIDER_RES_VERSION=v1.21.4-5 K8S_VERSION=${PROVIDER_RES_VERSION%-*} DOCKER_VERSION=19.03.14 CONTAINERD_VERSION=1.5.4 diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go index 49545da1c..d6cf97a63 100644 --- a/cmd/tke-installer/app/installer/images/images.go +++ b/cmd/tke-installer/app/installer/images/images.go @@ -117,7 +117,7 @@ var baseComponents = BaseComponents{ TKEPlatformController: containerregistry.Image{Name: "tke-platform-controller", Tag: Version}, TKERegistryAPI: containerregistry.Image{Name: "tke-registry-api", Tag: Version}, TKERegistryController: containerregistry.Image{Name: "tke-registry-controller", Tag: Version}, - ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-4"}, + ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-5"}, TKEGateway: containerregistry.Image{Name: "tke-gateway", Tag: Version}, NginxIngress: containerregistry.Image{Name: "ingress-nginx-controller", Tag: "v1.1.3"}, diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go index f25697f71..ba7640420 100644 --- a/pkg/platform/provider/baremetal/cluster/create.go +++ b/pkg/platform/provider/baremetal/cluster/create.go @@ -511,14 +511,23 @@ func (p *Provider) EnsureContainerRuntime(ctx context.Context, c *v1.Cluster) er return p.EnsureContainerd(ctx, c) } +func (p *Provider) getImagePrefix(c *v1.Cluster) string { + if anno, ok := c.Annotations[platformv1.LocationBasedImagePrefixAnno]; ok { + return anno + } + return containerregistryutil.GetPrefix() +} + func (p *Provider) EnsureContainerd(ctx context.Context, c *v1.Cluster) error { insecureRegistries := []string{p.Config.Registry.Domain} if c.Spec.TenantID != "" { insecureRegistries = append(insecureRegistries, c.Spec.TenantID+"."+p.Config.Registry.Domain) } + prefix := p.getImagePrefix(c) option := &containerd.Option{ InsecureRegistries: insecureRegistries, - SandboxImage: images.Get().Pause.FullName(), + SandboxImage: path.Join(prefix, images.Get().Pause.BaseName()), + RegistryMirror: prefix, } for _, machine := range c.Spec.Machines { machineSSH, err := machine.SSH() @@ -838,7 +847,11 @@ func (p *Provider) EnsureKubeadmInitPhaseKubeletStart(ctx context.Context, c *v1 phase += fmt.Sprintf(" --node-name=%s", c.Spec.Machines[0].IP) } } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), phase) + err = kubeadm.WriteInitConfig(machineSSH, p.getKubeadmInitConfig(c)) + if err != nil { + return err + } + return kubeadm.Init(machineSSH, phase) } func (p *Provider) EnsureKubeadmInitPhaseCerts(ctx context.Context, c *v1.Cluster) error { @@ -849,7 +862,7 @@ func (p *Provider) EnsureKubeadmInitPhaseCerts(ctx context.Context, c *v1.Cluste if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "certs all") + return kubeadm.Init(machineSSH, "certs all") } func (p *Provider) EnsureKubeadmInitPhaseKubeConfig(ctx context.Context, c *v1.Cluster) error { @@ -860,7 +873,7 @@ func (p *Provider) EnsureKubeadmInitPhaseKubeConfig(ctx context.Context, c *v1.C if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "kubeconfig all") + return kubeadm.Init(machineSSH, "kubeconfig all") } func (p *Provider) EnsureKubeadmInitPhaseControlPlane(ctx context.Context, c *v1.Cluster) error { @@ -871,7 +884,7 @@ func (p *Provider) EnsureKubeadmInitPhaseControlPlane(ctx context.Context, c *v1 if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "control-plane all") + return kubeadm.Init(machineSSH, "control-plane all") } func (p *Provider) EnsureKubeadmInitPhaseETCD(ctx context.Context, c *v1.Cluster) error { @@ -882,7 +895,7 @@ func (p *Provider) EnsureKubeadmInitPhaseETCD(ctx context.Context, c *v1.Cluster if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "etcd local") + return kubeadm.Init(machineSSH, "etcd local") } func (p *Provider) EnsureKubeadmInitPhaseUploadConfig(ctx context.Context, c *v1.Cluster) error { @@ -893,7 +906,7 @@ func (p *Provider) EnsureKubeadmInitPhaseUploadConfig(ctx context.Context, c *v1 if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "upload-config all ") + return kubeadm.Init(machineSSH, "upload-config all ") } func (p *Provider) EnsureKubeadmInitPhaseUploadCerts(ctx context.Context, c *v1.Cluster) error { @@ -901,7 +914,7 @@ func (p *Provider) EnsureKubeadmInitPhaseUploadCerts(ctx context.Context, c *v1. if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "upload-certs --upload-certs") + return kubeadm.Init(machineSSH, "upload-certs --upload-certs") } func (p *Provider) EnsureKubeadmInitPhaseBootstrapToken(ctx context.Context, c *v1.Cluster) error { @@ -912,7 +925,7 @@ func (p *Provider) EnsureKubeadmInitPhaseBootstrapToken(ctx context.Context, c * if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "bootstrap-token") + return kubeadm.Init(machineSSH, "bootstrap-token") } func (p *Provider) EnsureKubeadmInitPhaseAddon(ctx context.Context, c *v1.Cluster) error { @@ -923,7 +936,7 @@ func (p *Provider) EnsureKubeadmInitPhaseAddon(ctx context.Context, c *v1.Cluste if err != nil { return err } - return kubeadm.Init(machineSSH, p.getKubeadmInitConfig(c), "addon all") + return kubeadm.Init(machineSSH, "addon all") } func (p *Provider) EnsureGalaxy(ctx context.Context, c *v1.Cluster) error { diff --git a/pkg/platform/provider/baremetal/cluster/kubeadm.go b/pkg/platform/provider/baremetal/cluster/kubeadm.go index 501a00525..9a8aabb6e 100644 --- a/pkg/platform/provider/baremetal/cluster/kubeadm.go +++ b/pkg/platform/provider/baremetal/cluster/kubeadm.go @@ -21,6 +21,7 @@ package cluster import ( "fmt" "net" + "path" "github.com/imdario/mergo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,7 +35,6 @@ import ( "tkestack.io/tke/pkg/platform/provider/baremetal/phases/kubeadm" v1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/apiclient" - "tkestack.io/tke/pkg/util/containerregistry" "tkestack.io/tke/pkg/util/json" "tkestack.io/tke/pkg/util/version" ) @@ -184,7 +184,7 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste DNS: kubeadmv1beta2.DNS{ Type: kubeadmv1beta2.CoreDNS, }, - ImageRepository: containerregistry.GetPrefix(), + ImageRepository: p.getImagePrefix(c), ClusterName: c.Name, FeatureGates: map[string]bool{ "IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack}, @@ -311,7 +311,7 @@ func (p *Provider) getSchedulerExtraArgs(c *v1.Cluster) map[string]string { func (p *Provider) getKubeletExtraArgs(c *v1.Cluster) map[string]string { args := map[string]string{ - "pod-infra-container-image": images.Get().Pause.FullName(), + "pod-infra-container-image": path.Join(p.getImagePrefix(c), images.Get().Pause.BaseName()), } utilruntime.Must(mergo.Merge(&args, c.Spec.KubeletExtraArgs)) diff --git a/pkg/platform/provider/baremetal/cluster/update.go b/pkg/platform/provider/baremetal/cluster/update.go index 275da6e4e..27b4aac58 100644 --- a/pkg/platform/provider/baremetal/cluster/update.go +++ b/pkg/platform/provider/baremetal/cluster/update.go @@ -119,7 +119,12 @@ func (p *Provider) EnsureAPIServerCert(ctx context.Context, c *v1.Cluster) error preActions = append(preActions, fmt.Sprintf("rm -f %s", file)) } - err = kubeadm.Init(s, kubeadmConfig, "certs apiserver", preActions...) + err = kubeadm.WriteInitConfig(s, kubeadmConfig) + if err != nil { + return errors.Wrap(err, machine.IP) + } + + err = kubeadm.Init(s, "certs apiserver", preActions...) if err != nil { return errors.Wrap(err, machine.IP) } diff --git a/pkg/platform/provider/baremetal/conf/containerd/config.toml b/pkg/platform/provider/baremetal/conf/containerd/config.toml index 515de2236..c1a71be3a 100644 --- a/pkg/platform/provider/baremetal/conf/containerd/config.toml +++ b/pkg/platform/provider/baremetal/conf/containerd/config.toml @@ -42,5 +42,5 @@ state = "/run/containerd" insecure_skip_verify=true {{end}} [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."ccr.ccs.tencentyun.com"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{.RegistryMirror}}"] endpoint = ["http://mirrors.tke.com"] diff --git a/pkg/platform/provider/baremetal/config/config.go b/pkg/platform/provider/baremetal/config/config.go index a850a2da1..31e9cfdf8 100644 --- a/pkg/platform/provider/baremetal/config/config.go +++ b/pkg/platform/provider/baremetal/config/config.go @@ -57,6 +57,7 @@ type Config struct { Scheduler Scheduler `yaml:"scheduler"` AuthzWebhook AuthzWebhook `yaml:"authzWebhook"` Business Business `yaml:"business"` + SupportOSList []string `yaml:"supportOSList"` } func (c *Config) Save(filename string) error { diff --git a/pkg/platform/provider/baremetal/machine/create.go b/pkg/platform/provider/baremetal/machine/create.go index 9b6d9b510..baf59f471 100644 --- a/pkg/platform/provider/baremetal/machine/create.go +++ b/pkg/platform/provider/baremetal/machine/create.go @@ -49,6 +49,7 @@ import ( typesv1 "tkestack.io/tke/pkg/platform/types/v1" "tkestack.io/tke/pkg/util/apiclient" "tkestack.io/tke/pkg/util/cmdstring" + containerregistryutil "tkestack.io/tke/pkg/util/containerregistry" "tkestack.io/tke/pkg/util/hosts" ) @@ -338,6 +339,7 @@ func (p *Provider) EnsureContainerd(ctx context.Context, machine *platformv1.Mac InsecureRegistries: insecureRegistries, IsGPU: gpu.IsEnable(machine.Spec.Labels), SandboxImage: images.Get().Pause.FullName(), + RegistryMirror: containerregistryutil.GetPrefix(), } err = containerd.Install(machineSSH, option) if err != nil { diff --git a/pkg/platform/provider/baremetal/phases/containerd/containerd.go b/pkg/platform/provider/baremetal/phases/containerd/containerd.go index 419cc6c97..465ec4042 100644 --- a/pkg/platform/provider/baremetal/phases/containerd/containerd.go +++ b/pkg/platform/provider/baremetal/phases/containerd/containerd.go @@ -37,6 +37,7 @@ type Option struct { IsGPU bool Root string SandboxImage string + RegistryMirror string } const ( diff --git a/pkg/platform/provider/baremetal/phases/kubeadm/kubeadm.go b/pkg/platform/provider/baremetal/phases/kubeadm/kubeadm.go index 4d9df411a..eb900f989 100644 --- a/pkg/platform/provider/baremetal/phases/kubeadm/kubeadm.go +++ b/pkg/platform/provider/baremetal/phases/kubeadm/kubeadm.go @@ -102,16 +102,15 @@ func Install(s ssh.Interface, option *Option) error { return nil } -func Init(s ssh.Interface, kubeadmConfig *InitConfig, phase string, preActions ...string) error { +func WriteInitConfig(s ssh.Interface, kubeadmConfig *InitConfig) error { configData, err := kubeadmConfig.Marshal() if err != nil { return err } - err = s.WriteFile(bytes.NewReader(configData), constants.KubeadmConfigFileName) - if err != nil { - return err - } + return s.WriteFile(bytes.NewReader(configData), constants.KubeadmConfigFileName) +} +func Init(s ssh.Interface, phase string, preActions ...string) error { cmd, err := template.ParseString(initCmd, map[string]interface{}{ "Phase": phase, "Config": constants.KubeadmConfigFileName, diff --git a/pkg/platform/provider/baremetal/validation/cluster.go b/pkg/platform/provider/baremetal/validation/cluster.go index beb55b015..b4f9e6c0e 100644 --- a/pkg/platform/provider/baremetal/validation/cluster.go +++ b/pkg/platform/provider/baremetal/validation/cluster.go @@ -542,6 +542,7 @@ func ValidateCIDRs(cls *platform.Cluster, specPath *field.Path) field.ErrorList result.Name = AnywhereValidateItemHostNetOverlapping result.Description = "Verify Node IP(s) and CIDR Config" result.ErrorList = allErrs + result.Checked = true return field.ErrorList{result.ToFieldError()}