feat: add searcher to scheduler cluster (#462)

* feat: add searcher

Signed-off-by: Gaius <[email protected]>

Makefile

@@ -16,7 +16,7 @@ PROJECT_NAME := "d7y.io/dragonfly/v2"
 DFGET_NAME := "dfget"
 VERSION := "2.0.0"
 PKG := "$(PROJECT_NAME)"
-PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/manager/\)' | grep -v '\(/test/\)')
+PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/ | grep -v '\(/test/\)')
 GIT_COMMIT := $(shell git rev-parse --verify HEAD --short=7)
 GIT_COMMIT_LONG := $(shell git rev-parse --verify HEAD)
 DFGET_ARCHIVE_PREFIX := "$(DFGET_NAME)_$(GIT_COMMIT)"

go.mod

@@ -59,6 +59,7 @@ require (
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 	golang.org/x/tools v0.1.4 // indirect
+	gonum.org/v1/gonum v0.9.3
 	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
 	google.golang.org/grpc v1.36.0
 	google.golang.org/protobuf v1.26.0

internal/dfplugin/dfplugin.go

@@ -34,6 +34,7 @@ const (
 
 	// PluginMetaKeyType indicates the type of a plugin, currently support: resource
 	PluginMetaKeyType = "type"
+
 	// PluginMetaKeyName indicates the name of a plugin
 	PluginMetaKeyName = "name"
 )
@@ -42,6 +43,7 @@ type PluginType string
 
 const (
 	PluginTypeResource = PluginType("resource")
+	PluginTypeManager  = PluginType("manager")
 )
 
 type PluginInitFunc func(option map[string]string) (plugin interface{}, meta map[string]string, err error)

manager/model/scheduler_cluster.go

@@ -10,6 +10,7 @@ type SchedulerCluster struct {
 	BIO             string            `gorm:"column:bio;size:1024" json:"bio"`
 	Config          datatypes.JSONMap `gorm:"column:config;not null" json:"config"`
 	ClientConfig    datatypes.JSONMap `gorm:"column:client_config;not null" json:"client_config"`
+	Scopes          datatypes.JSONMap `gorm:"column:scopes" json:"scopes"`
 	IsDefault       bool              `gorm:"column:is_default;not null;default:false" json:"is_default"`
 	CDNClusters     []CDNCluster      `gorm:"many2many:cdn_cluster_scheduler_cluster;" json:"-"`
 	Schedulers      []Scheduler       `json:"-"`

manager/searcher/plugin.go

@@ -0,0 +1,39 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package searcher
+
+import (
+	"errors"
+
+	"d7y.io/dragonfly/v2/internal/dfplugin"
+)
+
+const (
+	pluginName = "searcher"
+)
+
+func LoadPlugin() (Searcher, error) {
+	client, _, err := dfplugin.Load(dfplugin.PluginTypeManager, pluginName, map[string]string{})
+	if err != nil {
+		return nil, err
+	}
+
+	if rc, ok := client.(Searcher); ok {
+		return rc, err
+	}
+	return nil, errors.New("invalid client, not a ResourceClient")
+}

manager/searcher/plugin_test.go

@@ -0,0 +1,72 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package searcher
+
+import (
+	"os"
+	"os/exec"
+	"path"
+	"testing"
+
+	testifyassert "github.com/stretchr/testify/assert"
+)
+
+func TestLoadPlugin(t *testing.T) {
+	assert := testifyassert.New(t)
+	defer func() {
+		os.Remove("./testdata/d7y-manager-plugin-searcher.so")
+		os.Remove("./testdata/test")
+	}()
+
+	var (
+		cmd    *exec.Cmd
+		output []byte
+		wd     string
+		err    error
+	)
+
+	// build plugin
+	cmd = exec.Command("go", "build", "-buildmode=plugin", "-o=./testdata/d7y-manager-plugin-searcher.so", "testdata/plugin/searcher.go")
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+
+	// build test binary
+	cmd = exec.Command("go", "build", "-o=./testdata/test", "testdata/main.go")
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+
+	wd, err = os.Getwd()
+	assert.Nil(err)
+	wd = path.Join(wd, "testdata")
+
+	// execute test binary
+	cmd = exec.Command("./testdata/test", "-plugin-dir", wd)
+	output, err = cmd.CombinedOutput()
+	assert.Nil(err)
+	if err != nil {
+		t.Fatalf(string(output))
+		return
+	}
+}

manager/searcher/search.go

@@ -0,0 +1,123 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package searcher
+
+import (
+	"sort"
+
+	"d7y.io/dragonfly/v2/manager/model"
+	"github.com/mitchellh/mapstructure"
+	"gonum.org/v1/gonum/stat"
+)
+
+const (
+	conditionSecurityDomain = "security_domain"
+	conditionLocation       = "location"
+	conditionIDC            = "idc"
+)
+
+const (
+	conditionLocationWeight = 0.7
+	conditionIDCWeight      = 0.3
+)
+
+type Scopes struct {
+	Location []string `mapstructure:"location"`
+	IDC      []string `mapstructure:"idc"`
+}
+
+type Searcher interface {
+	FindSchedulerCluster([]model.SchedulerCluster, map[string]string) (model.SchedulerCluster, bool)
+}
+
+type searcher struct{}
+
+func New() Searcher {
+	s, err := LoadPlugin()
+	if err != nil {
+		return &searcher{}
+	}
+
+	return s
+}
+
+func (s *searcher) FindSchedulerCluster(schedulerClusters []model.SchedulerCluster, conditions map[string]string) (model.SchedulerCluster, bool) {
+	if len(schedulerClusters) <= 0 || len(conditions) <= 0 {
+		return model.SchedulerCluster{}, false
+	}
+
+	// If there are security domain conditions, match clusters of the same security domain.
+	// If the security domain condition does not exist, it matches clusters that does not have a security domain.
+	// Then use clusters sets to score according to scopes.
+	securityDomain := conditions[conditionSecurityDomain]
+	var clusters []model.SchedulerCluster
+	for _, v := range schedulerClusters {
+		if v.SecurityGroup.Domain == securityDomain {
+			clusters = append(clusters, v)
+		}
+	}
+
+	switch len(clusters) {
+	case 0:
+		return model.SchedulerCluster{}, false
+	case 1:
+		return clusters[0], true
+	default:
+		var maxMean float64 = 0
+		cluster := clusters[0]
+		for _, v := range clusters {
+			mean := calculateSchedulerClusterMean(conditions, v.Scopes)
+			if mean > maxMean {
+				maxMean = mean
+				cluster = v
+			}
+		}
+		return cluster, true
+	}
+}
+
+func calculateSchedulerClusterMean(conditions map[string]string, rawScopes map[string]interface{}) float64 {
+	var scopes Scopes
+	if err := mapstructure.Decode(rawScopes, &scopes); err != nil {
+		return 0
+	}
+
+	location := conditions[conditionLocation]
+	lx := calculateConditionScore(location, scopes.Location)
+
+	idc := conditions[conditionIDC]
+	ix := calculateConditionScore(idc, scopes.IDC)
+
+	return stat.Mean([]float64{lx, ix}, []float64{conditionLocationWeight, conditionIDCWeight})
+}
+
+func calculateConditionScore(value string, scope []string) float64 {
+	if value == "" {
+		return 0
+	}
+
+	if len(scope) <= 0 {
+		return 0
+	}
+
+	i := sort.SearchStrings(scope, value)
+	if i < 0 {
+		return 0
+	}
+
+	return 1
+}