diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go index 8182ff4821..ebe4015d82 100644 --- a/pkg/platform/provider/baremetal/cluster/create.go +++ b/pkg/platform/provider/baremetal/cluster/create.go @@ -161,17 +161,17 @@ func (p *Provider) EnsurePreflight(ctx context.Context, c *v1.Cluster) error { } func (p *Provider) EnsureRegistryHosts(ctx context.Context, c *v1.Cluster) error { - if !p.config.Registry.NeedSetHosts() { + if !p.Config.Registry.NeedSetHosts() { return nil } machines := map[bool][]platformv1.ClusterMachine{ true: c.Spec.ScalingMachines, false: c.Spec.Machines}[len(c.Spec.ScalingMachines) > 0] domains := []string{ - p.config.Registry.Domain, + p.Config.Registry.Domain, } if c.Spec.TenantID != "" { - domains = append(domains, c.Spec.TenantID+"."+p.config.Registry.Domain) + domains = append(domains, c.Spec.TenantID+"."+p.Config.Registry.Domain) } for _, machine := range machines { machineSSH, err := machine.SSH() @@ -181,7 +181,7 @@ func (p *Provider) EnsureRegistryHosts(ctx context.Context, c *v1.Cluster) error for _, one := range domains { remoteHosts := hosts.RemoteHosts{Host: one, SSH: machineSSH} - err := remoteHosts.Set(p.config.Registry.IP) + err := remoteHosts.Set(p.Config.Registry.IP) if err != nil { return errors.Wrap(err, machine.IP) } @@ -499,9 +499,9 @@ func (p *Provider) EnsureContainerRuntime(ctx context.Context, c *v1.Cluster) er } func (p *Provider) EnsureContainerd(ctx context.Context, c *v1.Cluster) error { - insecureRegistries := []string{p.config.Registry.Domain} - if p.config.Registry.NeedSetHosts() && c.Spec.TenantID != "" { - insecureRegistries = append(insecureRegistries, c.Spec.TenantID+"."+p.config.Registry.Domain) + insecureRegistries := []string{p.Config.Registry.Domain} + if p.Config.Registry.NeedSetHosts() && c.Spec.TenantID != "" { + insecureRegistries = append(insecureRegistries, c.Spec.TenantID+"."+p.Config.Registry.Domain) } option := &containerd.Option{ InsecureRegistries: insecureRegistries, @@ -527,15 +527,15 @@ func (p *Provider) EnsureDocker(ctx context.Context, c *v1.Cluster) error { machines := map[bool][]platformv1.ClusterMachine{ true: c.Spec.ScalingMachines, false: c.Spec.Machines}[len(c.Spec.ScalingMachines) > 0] - insecureRegistries := fmt.Sprintf(`"%s"`, p.config.Registry.Domain) - if p.config.Registry.NeedSetHosts() && c.Spec.TenantID != "" { - insecureRegistries = fmt.Sprintf(`%s,"%s"`, insecureRegistries, c.Spec.TenantID+"."+p.config.Registry.Domain) + insecureRegistries := fmt.Sprintf(`"%s"`, p.Config.Registry.Domain) + if p.Config.Registry.NeedSetHosts() && c.Spec.TenantID != "" { + insecureRegistries = fmt.Sprintf(`%s,"%s"`, insecureRegistries, c.Spec.TenantID+"."+p.Config.Registry.Domain) } extraArgs := c.Spec.DockerExtraArgs - utilruntime.Must(mergo.Merge(&extraArgs, p.config.Docker.ExtraArgs)) + utilruntime.Must(mergo.Merge(&extraArgs, p.Config.Docker.ExtraArgs)) option := &docker.Option{ InsecureRegistries: insecureRegistries, - RegistryDomain: p.config.Registry.Domain, + RegistryDomain: p.Config.Registry.Domain, ExtraArgs: extraArgs, } for _, machine := range machines { @@ -558,7 +558,7 @@ func (p *Provider) EnsureKubernetesImages(ctx context.Context, c *v1.Cluster) er machines := map[bool][]platformv1.ClusterMachine{ true: c.Spec.ScalingMachines, false: c.Spec.Machines}[len(c.Spec.ScalingMachines) > 0] - option := &image.Option{Version: c.Spec.Version, RegistryDomain: p.config.Registry.Domain, KubeImages: images.KubecomponetNames} + option := &image.Option{Version: c.Spec.Version, RegistryDomain: p.Config.Registry.Domain, KubeImages: images.KubecomponetNames} for _, machine := range machines { machineSSH, err := machine.SSH() if err != nil { @@ -724,7 +724,7 @@ func (p *Provider) EnsureAuthzWebhook(ctx context.Context, c *v1.Cluster) error if isGlobalCluster { authzEndpoint, _ = c.AuthzWebhookBuiltinEndpoint() } else { - authzEndpoint = p.config.AuthzWebhook.Endpoint + authzEndpoint = p.Config.AuthzWebhook.Endpoint } } option := authzwebhook.Option{ @@ -758,7 +758,7 @@ func (p *Provider) EnsurePrepareForControlplane(ctx context.Context, c *v1.Clust return errors.Wrap(err, "parse schedulerPolicyConfig error") } auditWebhookConfig, err := template.ParseString(auditWebhookConfig, map[string]interface{}{ - "AuditBackendAddress": p.config.Audit.Address, + "AuditBackendAddress": p.Config.Audit.Address, "ClusterName": c.Name, }) if err != nil { @@ -788,7 +788,7 @@ func (p *Provider) EnsurePrepareForControlplane(ctx context.Context, c *v1.Clust } } - if p.config.AuditEnabled() { + if p.Config.AuditEnabled() { if len(auditPolicyData) != 0 { err = machineSSH.WriteFile(bytes.NewReader(auditPolicyData), constants.KubernetesAuditPolicyConfigFile) if err != nil { diff --git a/pkg/platform/provider/baremetal/cluster/kubeadm.go b/pkg/platform/provider/baremetal/cluster/kubeadm.go index a469bf7292..0dca9affa7 100644 --- a/pkg/platform/provider/baremetal/cluster/kubeadm.go +++ b/pkg/platform/provider/baremetal/cluster/kubeadm.go @@ -183,7 +183,7 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste DNS: kubeadmv1beta2.DNS{ Type: kubeadmv1beta2.CoreDNS, }, - ImageRepository: p.config.Registry.Prefix, + ImageRepository: p.Config.Registry.Prefix, ClusterName: c.Name, FeatureGates: map[string]bool{ "IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack}, @@ -246,7 +246,7 @@ func (p *Provider) getAPIServerExtraArgs(c *v1.Cluster) map[string]string { args := map[string]string{ "token-auth-file": constants.TokenFile, } - if p.config.AuditEnabled() { + if p.Config.AuditEnabled() { args["audit-policy-file"] = constants.KubernetesAuditPolicyConfigFile args["audit-webhook-config-file"] = constants.KubernetesAuditWebhookConfigFile } @@ -259,7 +259,7 @@ func (p *Provider) getAPIServerExtraArgs(c *v1.Cluster) map[string]string { } utilruntime.Must(mergo.Merge(&args, c.Spec.APIServerExtraArgs)) - utilruntime.Must(mergo.Merge(&args, p.config.APIServer.ExtraArgs)) + utilruntime.Must(mergo.Merge(&args, p.Config.APIServer.ExtraArgs)) return args } @@ -287,7 +287,7 @@ func (p *Provider) getControllerManagerExtraArgs(c *v1.Cluster) map[string]strin } utilruntime.Must(mergo.Merge(&args, c.Spec.ControllerManagerExtraArgs)) - utilruntime.Must(mergo.Merge(&args, p.config.ControllerManager.ExtraArgs)) + utilruntime.Must(mergo.Merge(&args, p.Config.ControllerManager.ExtraArgs)) return args } @@ -303,7 +303,7 @@ func (p *Provider) getSchedulerExtraArgs(c *v1.Cluster) map[string]string { } utilruntime.Must(mergo.Merge(&args, c.Spec.SchedulerExtraArgs)) - utilruntime.Must(mergo.Merge(&args, p.config.Scheduler.ExtraArgs)) + utilruntime.Must(mergo.Merge(&args, p.Config.Scheduler.ExtraArgs)) return args } @@ -314,7 +314,7 @@ func (p *Provider) getKubeletExtraArgs(c *v1.Cluster) map[string]string { } utilruntime.Must(mergo.Merge(&args, c.Spec.KubeletExtraArgs)) - utilruntime.Must(mergo.Merge(&args, p.config.Kubelet.ExtraArgs)) + utilruntime.Must(mergo.Merge(&args, p.Config.Kubelet.ExtraArgs)) return args } diff --git a/pkg/platform/provider/baremetal/cluster/provider.go b/pkg/platform/provider/baremetal/cluster/provider.go index a97aac256e..bb3d11d1bf 100644 --- a/pkg/platform/provider/baremetal/cluster/provider.go +++ b/pkg/platform/provider/baremetal/cluster/provider.go @@ -56,7 +56,7 @@ func RegisterProvider() { type Provider struct { *clusterprovider.DelegateProvider - config *config.Config + Config *config.Config } var _ clusterprovider.Provider = &Provider{} @@ -164,7 +164,7 @@ func NewProvider() (*Provider, error) { if err != nil { return nil, err } - p.config = cfg + p.Config = cfg containerregistry.Init(cfg.Registry.Domain, cfg.Registry.Namespace) @@ -215,14 +215,14 @@ func (p *Provider) PreCreate(cluster *types.Cluster) error { } } - if p.config.AuditEnabled() { + if p.Config.AuditEnabled() { if !cluster.AuthzWebhookEnabled() { cluster.Spec.Features.AuthzWebhookAddr = &platform.AuthzWebhookAddr{Builtin: &platform. BuiltinAuthzWebhookAddr{}} } } - if p.config.BusinessEnabled() { + if p.Config.BusinessEnabled() { if !cluster.AuthzWebhookEnabled() { cluster.Spec.Features.AuthzWebhookAddr = &platform.AuthzWebhookAddr{Builtin: &platform. BuiltinAuthzWebhookAddr{}} @@ -244,8 +244,8 @@ func (p *Provider) PreCreate(cluster *types.Cluster) error { if !cluster.Spec.Features.EnableMetricsServer { cluster.Spec.Features.SkipConditions = append(cluster.Spec.Features.SkipConditions, "EnsureMetricsServer") } - if p.config.Feature.SkipConditions != nil { - cluster.Spec.Features.SkipConditions = append(cluster.Spec.Features.SkipConditions, p.config.Feature.SkipConditions...) + if p.Config.Feature.SkipConditions != nil { + cluster.Spec.Features.SkipConditions = append(cluster.Spec.Features.SkipConditions, p.Config.Feature.SkipConditions...) } if cluster.Spec.Etcd == nil { diff --git a/pkg/platform/provider/edge/cluster/create.go b/pkg/platform/provider/edge/cluster/create.go new file mode 100644 index 0000000000..815a5475ed --- /dev/null +++ b/pkg/platform/provider/edge/cluster/create.go @@ -0,0 +1,388 @@ +package cluster + +import ( + "bytes" + "context" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "github.com/superedge/edgeadm/pkg/edgeadm/cmd" + "io" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kuberuntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientsetscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "os" + "strings" + platformv1 "tkestack.io/tke/api/platform/v1" + v1 "tkestack.io/tke/pkg/platform/types/v1" + + superedgecommon "github.com/superedge/edgeadm/pkg/edgeadm/common" + "github.com/superedge/edgeadm/pkg/edgeadm/constant" + "github.com/superedge/edgeadm/pkg/edgeadm/steps" + "github.com/superedge/edgeadm/pkg/util" + "io/ioutil" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "tkestack.io/tke/pkg/platform/provider/baremetal/constants" + "tkestack.io/tke/pkg/util/log" +) + +const ( + SuperEdgeRepo = "superedge.tencentcloudcr.com/superedge" + EgressYaml = ` +apiVersion: apiserver.k8s.io/v1beta1 +kind: EgressSelectorConfiguration +egressSelections: +- name: cluster + connection: + proxyProtocol: HTTPConnect + transport: + tcp: + url: https://tunnel-cloud.edge-system.svc.cluster.local:8000 + tlsConfig: + caBundle: /etc/kubernetes/pki/ca.crt + clientCert: /etc/kubernetes/pki/tunnel-anp-client.crt + clientKey: /etc/kubernetes/pki/tunnel-anp-client.key +` +) +const ( + EdgeImageRepository = "superedge.io/edgeImageResository" + EdgeVersion = "superedge.io/edge-version" + EdgeVirtualAddr = "superedge.io/edge-virtual-addr" +) + +func (p *Provider) EnsureEdgeFlannel(ctx context.Context, c *v1.Cluster) error { + edgeConf := &cmd.EdgeadmConfig{} + edgeConf.ManifestsDir = "" + + cfg := &kubeadmapi.InitConfiguration{} + cfg.ImageRepository = c.Annotations[EdgeImageRepository] + cfg.Networking.PodSubnet = c.Spec.ClusterCIDR + + clientSet, err := c.Clientset() + if err != nil { + return err + } + // Deploy edge flannel + return steps.EnsureFlannelAddon(cfg, edgeConf, clientSet) +} + +func (p *Provider) EnsurePrepareEgdeCluster(ctx context.Context, c *v1.Cluster) error { + // prepare node delay domain + nodeDelayDomain := "" + nodeDelayDomains := []string{constants.APIServerHostName} + for _, domain := range nodeDelayDomains { + nodeDelayDomain += fmt.Sprintf("%s\n", domain) + } + + // prepare node hosts config + nodeDomains := []string{ + p.bconfig.Registry.Domain, + c.Cluster.Spec.TenantID + "." + p.bconfig.Registry.Domain, + } + hostsConfig := "" + for _, one := range nodeDomains { + hostsConfig += fmt.Sprintf("%s %s\n", p.bconfig.Registry.IP, one) + } + + // prepare insecure registry config + insecureRegistries := "" + for _, registrie := range nodeDomains { + insecureRegistries += fmt.Sprintf("%s\n", registrie) + } + + // create edge-info configMap + edgeInfoCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constant.EdgeCertCM, + }, + Data: map[string]string{ + constant.EdgeNodeHostConfig: hostsConfig, + constant.EdgeNodeDelayDomain: nodeDelayDomain, + constant.InsecureRegistries: insecureRegistries, + }, + } + clientSet, err := c.Clientset() + if err != nil { + return err + } + if err := superedgecommon.EnsureEdgeSystemNamespace(clientSet); err != nil { + return err + } + alreayEdgeInfoCM, err := clientSet.CoreV1().ConfigMaps(constant.NamespaceEdgeSystem). + Get(context.TODO(), constant.EdgeCertCM, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + cm, err := clientSet.CoreV1().ConfigMaps( + constant.NamespaceEdgeSystem).Create(context.TODO(), edgeInfoCM, metav1.CreateOptions{}) + if err != nil { + return err + } + log.Infof("Create success configMap: %v", constant.EdgeNodeHostConfig, util.ToJson(cm)) + return nil + } + return err + } + + alreayEdgeInfoCM.Data[constant.EdgeNodeHostConfig] = hostsConfig + alreayEdgeInfoCM.Data[constant.EdgeNodeHostConfig] = hostsConfig + alreayEdgeInfoCM.Data[constant.EdgeNodeHostConfig] = hostsConfig + if _, err := clientSet.CoreV1().ConfigMaps(constant.NamespaceEdgeSystem). + Update(context.TODO(), alreayEdgeInfoCM, metav1.UpdateOptions{}); err != nil { + return err + } + + return nil +} + +func (p *Provider) EnsureApplyEdgeApps(ctx context.Context, c *v1.Cluster) error { + // get kube-apiserver ip + apiserverIP := c.Spec.Machines[0].IP + if len(c.Spec.PublicAlternativeNames) > 0 { + apiserverIP = c.Spec.PublicAlternativeNames[0] + } + if c.Spec.Features.HA != nil { + if c.Spec.Features.HA.TKEHA != nil { + apiserverIP = c.Spec.Features.HA.TKEHA.VIP + } + if c.Spec.Features.HA.ThirdPartyHA != nil { + apiserverIP = c.Spec.Features.HA.ThirdPartyHA.VIP + } + } + + os.MkdirAll(fmt.Sprintf("/tmp/%s", c.Name), os.ModePerm) + // create edge cluster car key cart + caKeyFile := fmt.Sprintf("/tmp/%s/ca.key", c.Name) + err := ioutil.WriteFile(caKeyFile, c.ClusterCredential.CAKey, 0644) + if err != nil { + return err + } + + caCertFile := fmt.Sprintf("/tmp/%s/ca.crt", c.Name) + err = ioutil.WriteFile(caCertFile, c.ClusterCredential.CACert, 0644) + if err != nil { + return err + } + + certSANs := []string{apiserverIP} + for _, machine := range c.Spec.Machines { + certSANs = append(certSANs, machine.IP) + } + if len(c.Spec.PublicAlternativeNames) > 0 { + certSANs = append(certSANs, c.Spec.PublicAlternativeNames...) + } + + // deploy superedge edge cluster apps + clientset, err := c.Clientset() + if err != nil { + return err + } + edgeConf := &cmd.EdgeadmConfig{} + edgeConf.ManifestsDir = "" + edgeConf.TunnelCloudToken = util.GetRandToken(32) + edgeConf.Version = c.Annotations[EdgeVersion] + edgeConf.EdgeImageRepository = c.Annotations[EdgeImageRepository] + + virtualAddr, ok := c.Annotations[EdgeVirtualAddr] + if ok { + edgeConf.EdgeVirtualAddr = virtualAddr + } else { + edgeConf.EdgeVirtualAddr = constant.DefaultEdgeVirtualAddr + } + + cfg := &kubeadmapi.InitConfiguration{} + cfg.APIServer.CertSANs = certSANs + cfg.CertificatesDir = fmt.Sprintf("/tmp/%s/", c.Name) + cfg.ControlPlaneEndpoint = apiserverIP + cfg.NodeRegistration.Name = c.Spec.Machines[0].IP + version, err := kubeadmutil.KubernetesReleaseVersion(strings.Split(c.Spec.Version, "-")[0]) + if err != nil { + klog.Errorf("Failed to get k8s version, cluster: %s, error: %v", c.Name, err) + return err + } + cfg.KubernetesVersion = version + + err = steps.EnsureServiceGroupAddon(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install ServiceGroup, cluster: %s, error: %v", c.Name, err) + return err + } + + steps.EnsureTunnelAddon(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install ServiceGroup, cluster: %s, error: %v", c.Name, err) + return err + } + + steps.EnsureEdgeHealthAddon(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install EdgeHealth, cluster: %s, error: %v", c.Name, err) + return err + } + + steps.EnsureEdgeCorednsAddon(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install EdgeCoredns, cluster: %s, error: %v", c.Name, err) + return err + } + + steps.EnsureNodePrepare(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install NodePrepar, cluster: %s, error: %v", c.Name, err) + return err + } + + steps.EnsureEdgeKubeConfig(cfg, edgeConf, clientset) + if err != nil { + klog.Errorf("Failed to install EdgeKubeConfig, cluster: %s, error: %v", c.Name, err) + return err + } + return nil +} + +func (p *Provider) EnsureKubeadmConfig(ctx context.Context, c *v1.Cluster) error { + client, err := c.Clientset() + if err != nil { + klog.Errorf("Failed to get clientSet, cluster: %s, error: %v", c.Name, err) + return err + } + cm, err := client.CoreV1().ConfigMaps(constant.NamespaceKubeSystem).Get(ctx, kubeadmconstants.KubeadmConfigConfigMap, metav1.GetOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + klog.Errorf("Failed to get configMap: %s, cluster: %s, error: %v", kubeadmconstants.KubeadmConfigConfigMap, c.Name, err) + return err + } + clusterConfig, ok := cm.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] + if !ok { + return fmt.Errorf("Fialed to get %s, cluster: %s ", kubeadmconstants.ClusterConfigurationConfigMapKey, c.Name) + } + + f := bytes.NewBuffer([]byte(clusterConfig)) + d := yamlutil.NewYAMLOrJSONDecoder(f, 4096) + ext := kuberuntime.RawExtension{} + if err := d.Decode(&ext); err != nil { + if err != io.EOF { + return err + } + } + obj, _, err := unstructured.UnstructuredJSONScheme.Decode(ext.Raw, nil, nil) + if err != nil { + return err + } + unstructuredMap, err := kuberuntime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return err + } + unstructuredObj := &unstructured.Unstructured{Object: unstructuredMap} + edgeRepo, ok := c.Annotations[EdgeImageRepository] + if !ok { + edgeRepo = EdgeImageRepository + } + unstructured.SetNestedField(unstructuredObj.Object, edgeRepo, "edgeImageRepository") + + clusterConfigByte, err := kubeadmutil.MarshalToYamlForCodecs(obj, schema.GroupVersion{ + Group: strings.Split(unstructuredObj.GetAPIVersion(), "/")[0], + Version: strings.Split(unstructuredObj.GetAPIVersion(), "/")[1], + }, kubeadmscheme.Codecs) + if err != nil { + return err + } + cm.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] = string(clusterConfigByte) + cm.ResourceVersion = "" + + err = apiclient.CreateOrMutateConfigMap(client, cm, func(cm *corev1.ConfigMap) error { + // Upgrade will call to UploadConfiguration with a modified KubernetesVersion reflecting the new + // Kubernetes version. In that case, the mutation path will take place. + cm.Data[kubeadmconstants.ClusterConfigurationConfigMapKey] = string(clusterConfigByte) + return nil + }) + if err != nil { + return err + } + return err +} + +func (p *Provider) EnsureEgressSelector(ctx context.Context, c *v1.Cluster) error { + crt, err := tls.X509KeyPair(c.ClusterCredential.CACert, c.ClusterCredential.CAKey) + if err != nil { + return err + } + certs, err := x509.ParseCertificates(crt.Certificate[0]) + if err != nil { + return err + } + crt.Leaf = certs[0] + + clientCrt, clientKey, err := util.GenerateClientCertAndKey(crt.Leaf, crt.PrivateKey.(*rsa.PrivateKey), "tunnel-anp") + if err != nil { + return err + } + keydata := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", //tobe replaced + Bytes: x509.MarshalPKCS1PrivateKey(clientKey), + }) + crtdata := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", //tobe replaced + Bytes: clientCrt.Raw, + }) + machines := map[bool][]platformv1.ClusterMachine{ + true: c.Spec.ScalingMachines, + false: c.Spec.Machines}[len(c.Spec.ScalingMachines) > 0] + for _, machine := range machines { + machineSSH, err := machine.SSH() + if err != nil { + return err + } + err = machineSSH.WriteFile(bytes.NewReader(crtdata), "/etc/kubernetes/pki/tunnel-anp-client.crt") + if err != nil { + return err + } + + err = machineSSH.WriteFile(bytes.NewReader(keydata), "/etc/kubernetes/pki/tunnel-anp-client.key") + if err != nil { + return err + } + + err = machineSSH.WriteFile(bytes.NewReader([]byte(EgressYaml)), "/etc/kubernetes/egress-selector-configuration.yaml") + if err != nil { + return err + } + pod, err := machineSSH.ReadFile("/etc/kubernetes/manifests/kube-apiserver.yaml") + if err != nil { + return err + } + apiserver := &corev1.Pod{} + _, _, err = clientsetscheme.Codecs.UniversalDeserializer().Decode(pod, nil, apiserver) + if err != nil { + return err + } + for k, v := range apiserver.Spec.Containers { + if v.Name == "kube-apiserver" { + v.Command = append(v.Command, "--egress-selector-config-file=/etc/kubernetes/egress-selector-configuration.yaml") + apiserver.Spec.Containers[k] = v + } + } + apiserver.Spec.DNSPolicy = corev1.DNSClusterFirstWithHostNet + serialized, err := kubeadmutil.MarshalToYaml(apiserver, corev1.SchemeGroupVersion) + if err != nil { + return err + } + + err = machineSSH.WriteFile(bytes.NewReader(serialized), "/etc/kubernetes/manifests/kube-apiserver.yaml") + if err != nil { + return err + } + + } + return nil +} diff --git a/pkg/platform/provider/edge/conf/create_edge_cluster.yml b/pkg/platform/provider/edge/conf/create_edge_cluster.yml new file mode 100644 index 0000000000..bf6f7e93f9 --- /dev/null +++ b/pkg/platform/provider/edge/conf/create_edge_cluster.yml @@ -0,0 +1,35 @@ +apiVersion: platform.tkestack.io/v1 +kind: Cluster +metadata: + annotations: + superedge.io/edgeImageResository: superedge.tencentcloudcr.com/superedge + superedge.io/edge-version: v0.8.0 + superedge.io/edge-virtual-addr: 169.254.20.11 + name: attlee-superedge +spec: + displayName: attlee-superedge-demo + etcd: + local: + dataDir: "" + serverCertSANs: + - etcd + - etcd.kube-system + features: + containerRuntime: docker + skipConditions: + - EnsureCilium + machines: + - ip: 10.0.200.98 + password: PasswordBase64 + port: 22 + username: root + networkDevice: eth0 + clusterCIDR: 192.168.0.0/16 + properties: + maxClusterServiceNum: 256 + maxNodePodNum: 256 + publicAlternativeNames: + - 106.52.199.103 + tenantID: default + type: Edge + version: 1.20.6-tke.2