From 4ba2d4192249593731d83cda443ba4019926150b Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 29 Sep 2022 09:52:05 +0800 Subject: [PATCH] feat(platform): validate storage info (#2093) --- .../provider/baremetal/validation/cluster.go | 143 +++++++++++++++++- .../baremetal/validation/constants.go | 1 + .../baremetal/validation/overridevalue.go | 61 ++++++++ pkg/util/ssh/os.go | 12 +- 4 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 pkg/platform/provider/baremetal/validation/overridevalue.go diff --git a/pkg/platform/provider/baremetal/validation/cluster.go b/pkg/platform/provider/baremetal/validation/cluster.go index ec18de7fb..ae739282a 100644 --- a/pkg/platform/provider/baremetal/validation/cluster.go +++ b/pkg/platform/provider/baremetal/validation/cluster.go @@ -22,10 +22,13 @@ import ( "context" "encoding/base64" "fmt" + appsv1alpha1 "github.com/clusternet/apis/apps/v1alpha1" "math" "net" + "strconv" "strings" "time" + "tkestack.io/tke/pkg/mesh/util/json" k8serror "k8s.io/apimachinery/pkg/api/errors" apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" @@ -125,6 +128,7 @@ func ValidatClusterSpec(platformClient platformv1client.PlatformV1Interface, clu allErrs = append(allErrs, ValidateClusterSpecVersion(platformClient, clusterName, cls.Spec.Version, fldPath.Child("version"), phase)...) allErrs = append(allErrs, ValidateCIDRs(cls, fldPath)...) allErrs = append(allErrs, ValidateClusterProperty(&cls.Spec, fldPath.Child("properties"))...) + allErrs = append(allErrs, ValidateStorage(cls, fldPath)...) if validateMachine { allErrs = append(allErrs, ValidateClusterMachines(cls, fldPath.Child("machines"))...) } @@ -281,7 +285,7 @@ func ValidateClusterMachines(cls *platform.Cluster, fldPath *field.Path) field.E selinuxResult.Name = AnywhereValidateItemSelinux selinuxResult.Description = "Verify Selinux" - selinuxResult.ErrorList = firewallErrs + selinuxResult.ErrorList = selinuxErrs allErrs = append(allErrs, proxyResult.ToFieldError(), @@ -350,7 +354,7 @@ func ValidateOSVersion(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList { func ValidateReservePorts(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList { allErrs := field.ErrorList{} for i, one := range sshs { - err := ssh.ReservePorts(one, reservePorts) + err := ssh.ReservePorts(one, "127.0.0.1", reservePorts) if err != nil { allErrs = append(allErrs, field.Invalid(fldPath.Index(i), one.Host, err.Error())) } @@ -358,6 +362,141 @@ func ValidateReservePorts(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList return allErrs } +func ValidateStorage(cls *platform.Cluster, fld *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + storageResult := TKEValidateResult{} + allErrs = append(allErrs, ValidateNFS(cls, fld)...) + allErrs = append(allErrs, ValidateCephFS(cls, fld)...) + + storageResult.Checked = true + if _, ok := cls.Annotations[platform.AnywhereValidateAnno]; ok { + storageResult.Name = AnywhereValidateItemStorage + storageResult.Description = "Validate Storage Info" + storageResult.ErrorList = allErrs + } + return allErrs +} + +func ValidateNFS(cls *platform.Cluster, fld *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if cls.Annotations[platformv1.AnywhereLocalizationsAnno] == "" { + return nil + } + + localizationsJSON, err := base64.StdEncoding.DecodeString(cls.Annotations[platformv1.AnywhereLocalizationsAnno]) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "decode error")) + return allErrs + } + var storageInfo *StorageInfo + storageInfo, err = GetStorageInfo(localizationsJSON) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, storageInfo, err.Error())) + return allErrs + } + if storageInfo == nil || !storageInfo.EnableNFS { + return nil + } + + machine, err := cls.Spec.Machines[0].SSH() + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "ssh machine failed")) + } + nfsServer := storageInfo.Nfs.Server + nfsPath := storageInfo.Nfs.Path + err = ssh.CheckNFS(machine, nfsServer, nfsPath) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "check nfs failed")) + } + + return allErrs +} + +func ValidateCephFS(cls *platform.Cluster, fld *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if cls.Annotations[platformv1.AnywhereLocalizationsAnno] == "" { + return nil + } + + localizationsJSON, err := base64.StdEncoding.DecodeString(cls.Annotations[platformv1.AnywhereLocalizationsAnno]) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "decode error")) + return allErrs + } + + var storageInfo *StorageInfo + storageInfo, err = GetStorageInfo(localizationsJSON) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, storageInfo, err.Error())) + return allErrs + } + if storageInfo == nil || !storageInfo.EnableCephfs { + return nil + } + + for _, host := range storageInfo.CsiConfig.Monitors { + arr := strings.SplitN(host, ":", 2) + if len(arr) != 2 { + allErrs = append(allErrs, field.Invalid(fld, host, "invalid host format")) + continue + } + + ip, port := arr[0], arr[1] + p, err := strconv.Atoi(port) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "convert port error")) + continue + } + + machine, err := cls.Spec.Machines[0].SSH() + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "ssh machine failed")) + } + + err = ssh.ReservePorts(machine, ip, []int{p}) + if err != nil { + allErrs = append(allErrs, field.Invalid(fld, err, "invalid port")) + } + } + return allErrs +} + +func GetStorageInfo(annoData []byte) (*StorageInfo, error) { + localizations := new(appsv1alpha1.LocalizationList) + err := json.Unmarshal(annoData, localizations) + if err != nil { + return nil, err + } + + res := &StorageInfo{} + + for _, item := range localizations.Items { + if item.ObjectMeta.Name != "tke-storage" { + continue + } + n := len(item.Spec.Overrides) + if n == 0 { + return nil, nil + } + value := &StorageOverrideValue{} + err = json.Unmarshal([]byte(item.Spec.Overrides[n-1].Value), value) + if err != nil { + return nil, err + } + res.EnableNFS = value.Global.EnableNFS + res.Nfs.Server = value.NfsSubdirExternalProvisioner.Nfs.Server + res.Nfs.Path = value.NfsSubdirExternalProvisioner.Nfs.Path + res.EnableCephfs = value.Global.EnableCephFS + for _, cfg := range value.CephCsiCephfs.CsiConfig { + res.CsiConfig.Monitors = append(res.CsiConfig.Monitors, cfg.Monitors...) + } + return res, nil + } + return nil, nil +} + func ValidateFirewall(fldPath *field.Path, sshs []*ssh.SSH) field.ErrorList { allErrs := field.ErrorList{} for i, one := range sshs { diff --git a/pkg/platform/provider/baremetal/validation/constants.go b/pkg/platform/provider/baremetal/validation/constants.go index 349a4f258..fc22b1379 100644 --- a/pkg/platform/provider/baremetal/validation/constants.go +++ b/pkg/platform/provider/baremetal/validation/constants.go @@ -29,6 +29,7 @@ const ( AnywhereValidateItemHostNetOverlapping = "HostNetOverlapping" AnywhereValidateItemFirewall = "Firewall" AnywhereValidateItemSelinux = "Selinux" + AnywhereValidateItemStorage = "Storage" ) const ( diff --git a/pkg/platform/provider/baremetal/validation/overridevalue.go b/pkg/platform/provider/baremetal/validation/overridevalue.go new file mode 100644 index 000000000..134f66c45 --- /dev/null +++ b/pkg/platform/provider/baremetal/validation/overridevalue.go @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package validation + +// StorageOverrideValue contains all the storage info used to validate from localization OverrideConfig +type StorageOverrideValue struct { + Global StorageGlobalCfg `json:"global"` + NfsSubdirExternalProvisioner NfsSubdirExternalProvisioner `json:"nfs-subdir-external-provisioner"` + CephCsiCephfs CephCsiCephfs `json:"ceph-csi-cephfs"` +} +type StorageGlobalCfg struct { + EnableNFS bool `json:"enableNFS"` + EnableCephFS bool `json:"enableCephFS"` +} +type Nfs struct { + Server string `json:"server"` + Path string `json:"path"` +} +type NfsSubdirExternalProvisioner struct { + Nfs Nfs `json:"nfs"` +} +type CsiConfig struct { + ClusterID string `json:"clusterID"` + Monitors []string `json:"monitors"` +} +type StorageClass struct { + ClusterID string `json:"clusterID"` + FsName string `json:"fsName"` +} +type CephCsiSecret struct { + AdminID string `json:"adminID"` + AdminKey string `json:"adminKey"` +} +type CephCsiCephfs struct { + CsiConfig []CsiConfig `json:"csiConfig"` + StorageClass StorageClass `json:"storageClass"` + Secret CephCsiSecret `json:"secret"` +} + +type StorageInfo struct { + EnableNFS bool + EnableCephfs bool + Nfs Nfs + CsiConfig CsiConfig +} diff --git a/pkg/util/ssh/os.go b/pkg/util/ssh/os.go index eba12b871..5d26662ae 100644 --- a/pkg/util/ssh/os.go +++ b/pkg/util/ssh/os.go @@ -156,10 +156,10 @@ func OSVersion(s Interface) (os string, err error) { return id + version, nil } -func ReservePorts(s Interface, ports []int) error { +func ReservePorts(s Interface, ip string, ports []int) error { var cmd, errMessage string for _, port := range ports { - cmd += fmt.Sprintf(`bash -c "/dev/null; echo $?; `, port) + cmd += fmt.Sprintf(`bash -c "/dev/null; echo $?; `, ip, port) } out, _, _, _ := s.Exec(cmd) out = strings.TrimSuffix(out, "\n") @@ -217,3 +217,11 @@ func SelinuxEnabled(s Interface) (enabled bool, err error) { res := strings.TrimSpace(stdout) return res == "0", nil } + +func CheckNFS(s Interface, server string, path string) (err error) { + _, stderr, exit, err := s.Execf("mkdir -p /tmp/nfs/&& mount -t nfs %s:%s /tmp/nfs/&& umount /tmp/nfs/&& rm -rf /tmp/nfs/", server, path) + if exit != 0 || err != nil { + return fmt.Errorf("check nfs failed:exit %d:stderr %s:error %s", exit, stderr, err) + } + return nil +}