rate limit filter

configs/api_config.yaml

@@ -79,4 +79,34 @@ pluginsGroup:
       - name: "trace"
         version: "0.0.1"
         priority: 1000
-        externalLookupName: "ExternalPluginTrace"
+        externalLookupName: "ExternalPluginTrace"
+rateLimit:
+  resources:
+    - name: test-dubbo
+      items:
+        #Exact
+        - matchStrategy: 0
+          pattern: "/api/v1/test-dubbo/user"
+        #Regex
+        - matchStrategy: 1
+          pattern: "/api/v1/test-dubbo/user/*"
+    - name: test-http
+      items:
+        #Exact
+        - matchStrategy: 0
+          pattern: "/api/v1/http/foo"
+        - matchStrategy: 0
+          pattern: "/api/v1/http/bar"
+        #Regex
+        - matchStrategy: 1
+          pattern: "/api/v1/http/foo/*"
+        - matchStrategy: 1
+          pattern: "/api/v1/http/bar/*"
+
+  rules:
+    #qps sample At most 100 requests can be passed in 1000ms, so qps is 100
+    - resource: "test-dubbo"
+      enable: true
+      flowRule:
+        threshold: 100
+        statintervalinms: 1000

docs/index.md

@@ -11,6 +11,7 @@
 * [dubbo](sample/dubbo.md)
 * [http](sample/http.md)
 * [mock](sample/mock.md)
+* [plugins](../samples/plugins/index.md)
 
 ## [User Guide](user/README.md)
 

go.mod

@@ -3,24 +3,29 @@ module github.com/apache/dubbo-go-pixiu
 go 1.14
 
 require (
-	github.com/apache/dubbo-go v1.5.5
-	github.com/apache/dubbo-go-hessian2 v1.7.0
+	github.com/alibaba/sentinel-golang v1.0.2
+	github.com/apache/dubbo-go v1.5.7-rc1
+	github.com/apache/dubbo-go-hessian2 v1.9.2
 	github.com/coreos/etcd v3.3.25+incompatible
-	github.com/dubbogo/dubbo-go-pixiu-filter v0.1.4-0.20210427062645-0bec837d429e
-	github.com/dubbogo/go-zookeeper v1.0.2
+	github.com/dubbogo/dubbo-go-pixiu-filter v0.1.4-0.20210613012702-8488bf80772c
+	github.com/dubbogo/go-zookeeper v1.0.3
+	github.com/dubbogo/gost v1.11.7
 	github.com/emirpasic/gods v1.12.0
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/goinggo/mapstructure v0.0.0-20140717182941-194205d9b4a9
 	github.com/hashicorp/consul/api v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/shirou/gopsutil v3.21.3+incompatible // indirect
 	github.com/spf13/cast v1.3.1
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/urfave/cli v1.22.4
 	go.uber.org/zap v1.16.0
-	google.golang.org/grpc v1.26.0
+	google.golang.org/grpc v1.33.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
-replace github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 => github.com/envoyproxy/go-control-plane v0.8.0
+replace (
+	github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane v0.8.0
+	google.golang.org/grpc v1.33.1 => google.golang.org/grpc v1.26.0
+)

pkg/common/constant/key.go

@@ -29,6 +29,7 @@ const (
 	RecoveryFilter           = "dgp.filters.recovery"
 	ResponseFilter           = "dgp.filters.response"
 	AccessLogFilter          = "dgp.filters.access_log"
+	RateLimitFilter          = "dgp.filters.rate_limit"
 )
 
 const (

pkg/filter/plugins/plugins.go

@@ -40,6 +40,11 @@ var (
 	errEmptyPluginConfig = errors.New("Empty plugin config")
 )
 
+func Init(groups []config.PluginsGroup, filePath string, resources []config.Resource) {
+	InitPluginsGroup(groups, filePath)
+	InitAPIURLWithFilterChain(resources)
+}
+
 // FilterChain include Pre & Post filters
 type FilterChain struct {
 	Pre  context.FilterChain

pkg/filter/ratelimit/init.go

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ratelimit
+
+import (
+	sentinel "github.com/alibaba/sentinel-golang/api"
+	sc "github.com/alibaba/sentinel-golang/core/config"
+	"github.com/alibaba/sentinel-golang/core/flow"
+	"github.com/alibaba/sentinel-golang/logging"
+	"github.com/dubbogo/dubbo-go-pixiu-filter/pkg/api/config/ratelimit"
+	"github.com/pkg/errors"
+)
+
+import (
+	"github.com/apache/dubbo-go-pixiu/pkg/filter/ratelimit/matcher"
+	"github.com/apache/dubbo-go-pixiu/pkg/logger"
+)
+
+func rateLimitInit(c *ratelimit.Config) error {
+	sentinelConf := sc.NewDefaultConfig()
+	if len(c.LogPath) > 0 {
+		sentinelConf.Sentinel.Log.Dir = c.LogPath
+	}
+	_ = logging.ResetGlobalLogger(getWrappedLogger())
+
+	if err := sentinel.InitWithConfig(sentinelConf); err != nil {
+		return errors.Wrap(err, "rate limit init fail")
+	}
+	matcher.Init()
+
+	loadApiResources(c.Resources)
+	loadRules(c.Rules)
+	return nil
+}
+
+// OnUpdate update api & rule
+func OnUpdate(c ratelimit.Config) {
+	loadApiResources(c.Resources)
+	loadRules(c.Rules)
+}
+
+// loadRules
+func loadRules(rules []ratelimit.Rule) {
+	var enableRules []*flow.Rule
+	for _, v := range rules {
+		if v.Enable {
+			enableRules = append(enableRules, &v.FlowRule)
+		}
+	}
+
+	if _, err := flow.LoadRules(enableRules); err != nil {
+		logger.Warnf("rate limit load rules err: %v", err)
+	}
+}
+
+// loadApiResources
+func loadApiResources(apis []ratelimit.Resource) {
+	matcher.Load(apis)
+}

pkg/filter/ratelimit/init_test.go

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ratelimit
+
+import "testing"
+
+func TestInit(t *testing.T) {
+	c := GetMockedRateLimitConfig()
+	err := rateLimitInit(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestOnUpdate(t *testing.T) {
+	config := GetMockedRateLimitConfig()
+	OnUpdate(*config)
+}

pkg/filter/ratelimit/logger_warpper.go

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ratelimit
+
+import "github.com/apache/dubbo-go-pixiu/pkg/logger"
+
+// loggerWrapper
+type loggerWrapper struct {
+}
+
+// getWrappedLogger
+func getWrappedLogger() loggerWrapper {
+	return loggerWrapper{}
+}
+
+func (l loggerWrapper) Debug(msg string, keysAndValues ...interface{}) {
+	logger.Debugf(msg, keysAndValues)
+}
+
+func (l loggerWrapper) Info(msg string, keysAndValues ...interface{}) {
+	logger.Infof(msg, keysAndValues)
+}
+
+func (l loggerWrapper) Warn(msg string, keysAndValues ...interface{}) {
+	logger.Warnf(msg, keysAndValues)
+}
+
+func (l loggerWrapper) Error(err error, msg string, keysAndValues ...interface{}) {
+	logger.Warnf(msg, err, keysAndValues)
+}
+
+// InfoEnabled todo logger should implements this method
+func (l loggerWrapper) InfoEnabled() bool {
+	return true
+}
+
+func (l loggerWrapper) ErrorEnabled() bool {
+	return true
+}
+
+func (l loggerWrapper) DebugEnabled() bool {
+	return true
+}
+
+func (l loggerWrapper) WarnEnabled() bool {
+	return true
+}

pkg/filter/ratelimit/matcher/exact.go

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package matcher
+
+import (
+	"github.com/dubbogo/dubbo-go-pixiu-filter/pkg/api/config/ratelimit"
+)
+
+import (
+	"sync"
+)
+
+type Exact struct {
+	apiNames map[string]string
+
+	mu sync.RWMutex
+}
+
+func (p *Exact) load(apis []ratelimit.Resource) {
+	m := map[string]string{}
+
+	for _, api := range apis {
+		apiName := api.Name
+		for _, item := range api.Items {
+			if item.MatchStrategy == ratelimit.EXACT {
+				m[item.Pattern] = apiName
+			}
+		}
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.apiNames = m
+}
+
+func (p *Exact) match(path string) (string, bool) {
+	p.mu.RLock()
+	defer p.mu.RUnlock()
+
+	resourceName, ok := p.apiNames[path]
+	return resourceName, ok
+}

pkg/filter/ratelimit/matcher/matcher.go

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package matcher
+
+import (
+	"github.com/dubbogo/dubbo-go-pixiu-filter/pkg/api/config/ratelimit"
+)
+
+var _matcher *Matcher
+
+// Init matcher
+func Init() {
+	_matcher = NewMatcher()
+}
+
+//PathMatcher according the url path find APIResource name
+type PathMatcher interface {
+	load(apis []ratelimit.Resource)
+
+	match(path string) (string, bool)
+}
+
+type Matcher struct {
+	matchers []PathMatcher
+}
+
+func NewMatcher() *Matcher {
+	return &Matcher{
+		matchers: []PathMatcher{
+			&Exact{},
+			&Regex{},
+		},
+	}
+}
+
+// Load load api resource for matchers
+func Load(apis []ratelimit.Resource) {
+	for _, v := range _matcher.matchers {
+		v.load(apis)
+	}
+}
+
+// Match match resource via url path
+func Match(path string) (string, bool) {
+	for _, m := range _matcher.matchers {
+		if res, ok := m.match(path); ok {
+			return res, ok
+		}
+	}
+	return "", false
+}