diff --git a/go.mod b/go.mod index 73164cdfaf..1576540141 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,10 @@ require ( github.com/joomcode/errorx v1.0.1 github.com/oleiade/reflections v1.0.1 github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12 - github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 + github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c - github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd + github.com/pingcap/log v0.0.0-20200511115504-543df19646ad github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd - github.com/pkg/errors v0.9.1 github.com/rs/cors v1.7.0 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 @@ -38,9 +37,9 @@ require ( github.com/thoas/go-funk v0.7.0 github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 - go.uber.org/atomic v1.5.0 + go.uber.org/atomic v1.6.0 go.uber.org/fx v1.10.0 - go.uber.org/zap v1.13.0 + go.uber.org/zap v1.15.0 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 diff --git a/go.sum b/go.sum index c5da22c815..8733587968 100644 --- a/go.sum +++ b/go.sum @@ -225,13 +225,15 @@ github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JH github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011 h1:58naV4XMEqm0hl9LcYo6cZoGBGiLtefMQMF/vo3XLgQ= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d h1:TH18wFO5Nq/zUQuWu9ms2urgZnLP69XJYiI2JZAkUGc= +github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c h1:wO9VvZezAU4ZPZj8+P5uWfsT/ppuABjJPmHNrpCQnlc= github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= -github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM= -github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/log v0.0.0-20200511115504-543df19646ad h1:SveG82rmu/GFxYanffxsSF503SiQV+2JLnWEiGiF+Tc= +github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd h1:k7CIHMFVKjHsda3PKkiN4zv++NEnexlUwiJEhryWpG0= github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -335,6 +337,8 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/dig v1.8.0 h1:1rR6hnL/bu1EVcjnRDN5kx1vbIjEJDTGhSQ2B3ddpcI= go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/fx v1.10.0 h1:S2K/H8oNied0Je/mLKdWzEWKZfv9jtxSDm8CnwK+5Fg= @@ -345,14 +349,16 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/pkg/keyvisual/decorator/decorator.go b/pkg/keyvisual/decorator/decorator.go index 683f134415..71bb717a65 100644 --- a/pkg/keyvisual/decorator/decorator.go +++ b/pkg/keyvisual/decorator/decorator.go @@ -27,41 +27,49 @@ type LabelKey struct { } // LabelStrategy requires cross-border determination and key decoration scheme. +// It supports dynamic reload configuration and generation of an actuator. type LabelStrategy interface { ReloadConfig(cfg *config.KeyVisualConfig) + NewLabeler() Labeler +} + +// Labeler is an executor of LabelStrategy, and its functions should not be called concurrently. +type Labeler interface { // CrossBorder determines whether two keys not belong to the same logical range. CrossBorder(startKey, endKey string) bool - // Label returns the Label information of the key. - // Note: When the key is "", need to use LabelGlobalStart or LabelGlobalEnd. - Label(key string) LabelKey - LabelGlobalStart() LabelKey - LabelGlobalEnd() LabelKey + // Label returns the Label information of the keys. + Label(keys []string) []LabelKey } // NaiveLabelStrategy is one of the simplest LabelStrategy. -type NaiveLabelStrategy struct{} - -func (s NaiveLabelStrategy) ReloadConfig(cfg *config.KeyVisualConfig) { +func NaiveLabelStrategy() LabelStrategy { + return naiveLabelStrategy{} } -// CrossBorder always returns false. So NaiveLabelStrategy believes that there are no cross-border situations. -func (s NaiveLabelStrategy) CrossBorder(startKey, endKey string) bool { - return false -} +type naiveLabelStrategy struct{} -// Label only decodes the key. -func (s NaiveLabelStrategy) Label(key string) LabelKey { - str := hex.EncodeToString([]byte(key)) - return LabelKey{ - Key: str, - Labels: []string{str}, - } +type naiveLabeler struct{} + +func (s naiveLabelStrategy) ReloadConfig(cfg *config.KeyVisualConfig) {} + +func (s naiveLabelStrategy) NewLabeler() Labeler { + return naiveLabeler{} } -func (s NaiveLabelStrategy) LabelGlobalStart() LabelKey { - return s.Label("") +// CrossBorder always returns false. So naiveLabelStrategy believes that there are no cross-border situations. +func (e naiveLabeler) CrossBorder(startKey, endKey string) bool { + return false } -func (s NaiveLabelStrategy) LabelGlobalEnd() LabelKey { - return s.Label("") +// Label only encodes the keys. +func (e naiveLabeler) Label(keys []string) []LabelKey { + labelKeys := make([]LabelKey, len(keys)) + for i, key := range keys { + str := hex.EncodeToString([]byte(key)) + labelKeys[i] = LabelKey{ + Key: str, + Labels: []string{str}, + } + } + return labelKeys } diff --git a/pkg/keyvisual/decorator/separator.go b/pkg/keyvisual/decorator/separator.go index 7b5063db71..3a74943dc5 100644 --- a/pkg/keyvisual/decorator/separator.go +++ b/pkg/keyvisual/decorator/separator.go @@ -1,3 +1,16 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package decorator import ( @@ -10,44 +23,52 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/config" ) -// NaiveLabelStrategy is one of the simplest LabelStrategy. -type separatorLabelStrategy struct { - Separator atomic.Value -} - +// SeparatorLabelStrategy implements the LabelStrategy interface. It obtains label information after splitting the key. func SeparatorLabelStrategy(cfg *config.KeyVisualConfig) LabelStrategy { s := &separatorLabelStrategy{} s.Separator.Store(cfg.PolicyKVSeparator) return s } +type separatorLabelStrategy struct { + Separator atomic.Value +} + +type separatorLabeler struct { + Separator string +} + // ReloadConfig reset separator func (s *separatorLabelStrategy) ReloadConfig(cfg *config.KeyVisualConfig) { s.Separator.Store(cfg.PolicyKVSeparator) log.Debug("Reload config", zap.String("separator", cfg.PolicyKVSeparator)) } +func (s *separatorLabelStrategy) NewLabeler() Labeler { + return &separatorLabeler{ + Separator: s.Separator.Load().(string), + } +} + // CrossBorder is temporarily not considering cross-border logic -func (s *separatorLabelStrategy) CrossBorder(startKey, endKey string) bool { +func (e *separatorLabeler) CrossBorder(startKey, endKey string) bool { return false } // Label uses separator to split key -func (s *separatorLabelStrategy) Label(key string) (label LabelKey) { - label.Key = key - separator := s.Separator.Load().(string) - if separator == "" { - label.Labels = []string{key} - return +func (e *separatorLabeler) Label(keys []string) []LabelKey { + labelKeys := make([]LabelKey, len(keys)) + for i, key := range keys { + var labels []string + if e.Separator == "" { + labels = []string{key} + } else { + labels = strings.Split(key, e.Separator) + } + labelKeys[i] = LabelKey{ + Key: key, + Labels: labels, + } } - label.Labels = strings.Split(key, separator) - return -} - -func (s *separatorLabelStrategy) LabelGlobalStart() LabelKey { - return s.Label("") -} - -func (s *separatorLabelStrategy) LabelGlobalEnd() LabelKey { - return s.Label("") + return labelKeys } diff --git a/pkg/keyvisual/decorator/tidb.go b/pkg/keyvisual/decorator/tidb.go index 1f7a182215..ba0a953829 100644 --- a/pkg/keyvisual/decorator/tidb.go +++ b/pkg/keyvisual/decorator/tidb.go @@ -29,27 +29,9 @@ import ( "github.com/pingcap/tidb-dashboard/pkg/tidb/model" ) -type tableDetail struct { - Name string - DB string - ID int64 - Indices map[int64]string -} - -type tidbLabelStrategy struct { - Config *config.Config - EtcdClient *clientv3.Client - - TableMap sync.Map - tidbClient *tidb.Client - SchemaVersion int64 - TidbAddress []string -} - -// TiDBLabelStrategy implements the LabelStrategy interface. Get Label Information from TiDB. -func TiDBLabelStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, cfg *config.Config, etcdClient *clientv3.Client, tidbClient *tidb.Client) LabelStrategy { +// TiDBLabelStrategy implements the LabelStrategy interface. It obtains Label Information from TiDB. +func TiDBLabelStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, etcdClient *clientv3.Client, tidbClient *tidb.Client) LabelStrategy { s := &tidbLabelStrategy{ - Config: cfg, EtcdClient: etcdClient, tidbClient: tidbClient, SchemaVersion: -1, @@ -69,9 +51,30 @@ func TiDBLabelStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, cfg *config.Config, return s } -func (s *tidbLabelStrategy) ReloadConfig(cfg *config.KeyVisualConfig) { +type tableDetail struct { + Name string + DB string + ID int64 + Indices map[int64]string +} + +type tidbLabelStrategy struct { + Config *config.Config + EtcdClient *clientv3.Client + + TableMap sync.Map + tidbClient *tidb.Client + SchemaVersion int64 + TidbAddress []string +} + +type tidbLabeler struct { + TableMap *sync.Map + Buffer model.KeyInfoBuffer } +func (s *tidbLabelStrategy) ReloadConfig(cfg *config.KeyVisualConfig) {} + func (s *tidbLabelStrategy) Background(ctx context.Context) { ticker := time.NewTicker(time.Minute) defer ticker.Stop() @@ -85,43 +88,78 @@ func (s *tidbLabelStrategy) Background(ctx context.Context) { } } +func (s *tidbLabelStrategy) NewLabeler() Labeler { + return &tidbLabeler{ + TableMap: &s.TableMap, + } +} + // CrossBorder does not allow cross tables or cross indexes within a table. -func (s *tidbLabelStrategy) CrossBorder(startKey, endKey string) bool { - startBytes, endBytes := model.Key(region.Bytes(startKey)), model.Key(region.Bytes(endKey)) - startIsMeta, startTableID := startBytes.MetaOrTable() - endIsMeta, endTableID := endBytes.MetaOrTable() +func (e *tidbLabeler) CrossBorder(startKey, endKey string) bool { + startInfo, _ := e.Buffer.DecodeKey(region.Bytes(startKey)) + startIsMeta, startTableID := startInfo.MetaOrTable() + startIndex := startInfo.IndexInfo() + + endInfo, _ := e.Buffer.DecodeKey(region.Bytes(endKey)) + endIsMeta, endTableID := endInfo.MetaOrTable() + endIndex := endInfo.IndexInfo() + if startIsMeta || endIsMeta { return startIsMeta != endIsMeta } if startTableID != endTableID { return true } - startIndex := startBytes.IndexID() - endIndex := endBytes.IndexID() return startIndex != endIndex } // Label will parse the ID information of the table and index. -func (s *tidbLabelStrategy) Label(key string) (label LabelKey) { +func (e *tidbLabeler) Label(keys []string) []LabelKey { + labelKeys := make([]LabelKey, len(keys)) + for i, key := range keys { + labelKeys[i] = e.label(key) + } + + if keys[0] == "" { + labelKeys[0] = globalStart + } + endIndex := len(keys) - 1 + if keys[endIndex] == "" { + labelKeys[endIndex] = globalEnd + } + + return labelKeys +} + +func (e *tidbLabeler) label(key string) (label LabelKey) { keyBytes := region.Bytes(key) label.Key = hex.EncodeToString(keyBytes) - decodeKey := model.Key(keyBytes) - isMeta, TableID := decodeKey.MetaOrTable() + keyInfo, _ := e.Buffer.DecodeKey(keyBytes) + + isMeta, tableID := keyInfo.MetaOrTable() if isMeta { label.Labels = append(label.Labels, "meta") - } else if v, ok := s.TableMap.Load(TableID); ok { - detail := v.(*tableDetail) + return + } + + var detail *tableDetail + if v, ok := e.TableMap.Load(tableID); ok { + detail = v.(*tableDetail) label.Labels = append(label.Labels, detail.DB, detail.Name) - if rowID := decodeKey.RowID(); rowID != 0 { - label.Labels = append(label.Labels, fmt.Sprintf("row_%d", rowID)) - } else if indexID := decodeKey.IndexID(); indexID != 0 { - label.Labels = append(label.Labels, detail.Indices[indexID]) - } } else { - label.Labels = append(label.Labels, fmt.Sprintf("table_%d", TableID)) - if rowID := decodeKey.RowID(); rowID != 0 { - label.Labels = append(label.Labels, fmt.Sprintf("row_%d", rowID)) - } else if indexID := decodeKey.IndexID(); indexID != 0 { + label.Labels = append(label.Labels, fmt.Sprintf("table_%d", tableID)) + } + + if isCommonHandle, rowID := keyInfo.RowInfo(); isCommonHandle { + label.Labels = append(label.Labels, "row") + } else if rowID != 0 { + label.Labels = append(label.Labels, fmt.Sprintf("row_%d", rowID)) + } else if indexID := keyInfo.IndexInfo(); indexID != 0 { + if detail == nil { + label.Labels = append(label.Labels, fmt.Sprintf("index_%d", indexID)) + } else if name, ok := detail.Indices[indexID]; ok { + label.Labels = append(label.Labels, name) + } else { label.Labels = append(label.Labels, fmt.Sprintf("index_%d", indexID)) } } @@ -137,11 +175,3 @@ var globalEnd = LabelKey{ Key: "", Labels: []string{}, } - -func (s *tidbLabelStrategy) LabelGlobalStart() LabelKey { - return globalStart -} - -func (s *tidbLabelStrategy) LabelGlobalEnd() LabelKey { - return globalEnd -} diff --git a/pkg/keyvisual/matrix/average.go b/pkg/keyvisual/matrix/average.go index bed5b9e774..1ee1113371 100644 --- a/pkg/keyvisual/matrix/average.go +++ b/pkg/keyvisual/matrix/average.go @@ -13,29 +13,20 @@ package matrix -import ( - "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" -) - -type averageHelper struct { +// AverageSplitStrategy adopts the strategy of equal distribution when buckets are split. +func AverageSplitStrategy() SplitStrategy { + return averageSplitStrategy{} } -type averageStrategy struct { - decorator.LabelStrategy -} +type averageSplitStrategy struct{} -// AverageStrategy adopts the strategy of equal distribution when buckets are split. -func AverageStrategy(label decorator.LabelStrategy) Strategy { - return averageStrategy{ - LabelStrategy: label, - } -} +type averageSplitter struct{} -func (averageStrategy) GenerateHelper(chunks []chunk, compactKeys []string) interface{} { - return averageHelper{} +func (averageSplitStrategy) NewSplitter(chunks []chunk, compactKeys []string) Splitter { + return averageSplitter{} } -func (averageStrategy) Split(dst, src chunk, tag splitTag, axesIndex int, helper interface{}) { +func (averageSplitter) Split(dst, src chunk, tag splitTag, axesIndex int) { CheckPartOf(dst.Keys, src.Keys) if len(dst.Keys) == len(src.Keys) { diff --git a/pkg/keyvisual/matrix/axis.go b/pkg/keyvisual/matrix/axis.go index 379ea97fc7..a8c4896857 100644 --- a/pkg/keyvisual/matrix/axis.go +++ b/pkg/keyvisual/matrix/axis.go @@ -13,6 +13,10 @@ package matrix +import ( + "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" +) + // Axis stores consecutive buckets. Each bucket has StartKey, EndKey, and some statistics. The EndKey of each bucket is // the StartKey of its next bucket. The actual data structure is stored in columns. Therefore satisfies: // len(Keys) == len(ValuesList[i]) + 1. In particular, ValuesList[0] is the base column. @@ -77,13 +81,13 @@ func (axis *Axis) Range(startKey string, endKey string) Axis { // Focus uses the base column as the chunk for the Focus operation to obtain the partitioning scheme, and uses this to // reduce other columns. -func (axis *Axis) Focus(strategy Strategy, threshold uint64, ratio int, target int) Axis { +func (axis *Axis) Focus(labeler decorator.Labeler, threshold uint64, ratio int, target int) Axis { if target >= len(axis.Keys)-1 { return *axis } baseChunk := createChunk(axis.Keys, axis.ValuesList[0]) - newChunk := baseChunk.Focus(strategy, threshold, ratio, target, MergeColdLogicalRange) + newChunk := baseChunk.Focus(labeler, threshold, ratio, target, MergeColdLogicalRange) valuesListLen := len(axis.ValuesList) newValuesList := make([][]uint64, valuesListLen) newValuesList[0] = newChunk.Values @@ -96,13 +100,13 @@ func (axis *Axis) Focus(strategy Strategy, threshold uint64, ratio int, target i // Divide uses the base column as the chunk for the Divide operation to obtain the partitioning scheme, and uses this to // reduce other columns. -func (axis *Axis) Divide(strategy Strategy, target int) Axis { +func (axis *Axis) Divide(labeler decorator.Labeler, target int) Axis { if target >= len(axis.Keys)-1 { return *axis } baseChunk := createChunk(axis.Keys, axis.ValuesList[0]) - newChunk := baseChunk.Divide(strategy, target, MergeColdLogicalRange) + newChunk := baseChunk.Divide(labeler, target, MergeColdLogicalRange) valuesListLen := len(axis.ValuesList) newValuesList := make([][]uint64, valuesListLen) newValuesList[0] = newChunk.Values @@ -216,7 +220,7 @@ func (c *chunk) GetFocusRows(threshold uint64) (count int) { // Given a `threshold`, merge the rows with less traffic, // and merge the most `ratio` rows at a time. // `target` is the estimated final number of rows. -func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int, mode FocusMode) chunk { +func (c *chunk) Focus(labeler decorator.Labeler, threshold uint64, ratio int, target int, mode FocusMode) chunk { newKeys := make([]string, 0, target) newValues := make([]uint64, 0, target) newKeys = append(newKeys, c.Keys[0]) @@ -236,7 +240,7 @@ func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int if value >= threshold || bucketSum >= threshold || i-start >= ratio || - strategy.CrossBorder(c.Keys[start], c.Keys[i]) { + labeler.CrossBorder(c.Keys[start], c.Keys[i]) { generateBucket(i) } bucketSum += value @@ -245,12 +249,12 @@ func (c *chunk) Focus(strategy Strategy, threshold uint64, ratio int, target int newChunk := createChunk(newKeys, newValues) if mode == MergeColdLogicalRange && len(newValues) >= target { - newChunk = newChunk.MergeColdLogicalRange(strategy, threshold, target) + newChunk = newChunk.MergeColdLogicalRange(labeler, threshold, target) } return newChunk } -func (c *chunk) MergeColdLogicalRange(strategy Strategy, threshold uint64, target int) chunk { +func (c *chunk) MergeColdLogicalRange(labeler decorator.Labeler, threshold uint64, target int) chunk { threshold /= 4 // TODO: This var can be adjusted newKeys := make([]string, 0, target) @@ -291,7 +295,7 @@ func (c *chunk) MergeColdLogicalRange(strategy Strategy, threshold uint64, targe } for i := range c.Values { - if strategy.CrossBorder(c.Keys[i], c.Keys[i+1]) { + if labeler.CrossBorder(c.Keys[i], c.Keys[i+1]) { generateRange(i + 1) } } @@ -302,7 +306,7 @@ func (c *chunk) MergeColdLogicalRange(strategy Strategy, threshold uint64, targe } // Divide uses binary search to find a suitable threshold, which can reduce the number of buckets of Axis to near the target. -func (c *chunk) Divide(strategy Strategy, target int, mode FocusMode) chunk { +func (c *chunk) Divide(labeler decorator.Labeler, target int, mode FocusMode) chunk { if target >= len(c.Values) { return *c } @@ -326,5 +330,5 @@ func (c *chunk) Divide(strategy Strategy, target int, mode FocusMode) chunk { threshold := lowerThreshold focusRows := c.GetFocusRows(threshold) ratio := len(c.Values)/(target-focusRows) + 1 - return c.Focus(strategy, threshold, ratio, target, mode) + return c.Focus(labeler, threshold, ratio, target, mode) } diff --git a/pkg/keyvisual/matrix/distance.go b/pkg/keyvisual/matrix/distance.go index 76b13e4c60..0021570cf9 100644 --- a/pkg/keyvisual/matrix/distance.go +++ b/pkg/keyvisual/matrix/distance.go @@ -21,39 +21,18 @@ import ( "sync" "go.uber.org/fx" - - "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" ) // TODO: // * Multiplexing data between requests // * Limit memory usage -type distanceHelper struct { - Scale [][]float64 -} - -type distanceStrategy struct { - decorator.LabelStrategy - - SplitRatio float64 - SplitLevel int - SplitCount int - - SplitRatioPow []float64 - - ScaleWorkerCh chan *scaleTask -} - -// DistanceStrategy adopts the strategy that the closer the split time is to the current time, the more traffic is -// allocated, when buckets are split. -func DistanceStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, label decorator.LabelStrategy, ratio float64, level int, count int) Strategy { +func DistanceSplitStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, ratio float64, level int, count int) SplitStrategy { pow := make([]float64, level) for i := range pow { pow[i] = math.Pow(ratio, float64(i)) } - s := &distanceStrategy{ - LabelStrategy: label, + s := &distanceSplitStrategy{ SplitRatio: ratio, SplitLevel: level, SplitCount: count, @@ -74,7 +53,21 @@ func DistanceStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, label decorator.Label return s } -func (s *distanceStrategy) GenerateHelper(chunks []chunk, compactKeys []string) interface{} { +type distanceSplitStrategy struct { + SplitRatio float64 + SplitLevel int + SplitCount int + + SplitRatioPow []float64 + + ScaleWorkerCh chan *scaleTask +} + +type distanceSplitter struct { + Scale [][]float64 +} + +func (s *distanceSplitStrategy) NewSplitter(chunks []chunk, compactKeys []string) Splitter { axesLen := len(chunks) keysLen := len(compactKeys) @@ -100,12 +93,12 @@ func (s *distanceStrategy) GenerateHelper(chunks []chunk, compactKeys []string) updateRightDis(dis[i], dis[i+1], chunks[i].Keys, compactKeys) } - return distanceHelper{ + return &distanceSplitter{ Scale: s.GenerateScale(chunks, compactKeys, dis), } } -func (s *distanceStrategy) Split(dst, src chunk, tag splitTag, axesIndex int, helper interface{}) { +func (e *distanceSplitter) Split(dst, src chunk, tag splitTag, axesIndex int) { CheckPartOf(dst.Keys, src.Keys) if len(dst.Keys) == len(src.Keys) { @@ -127,7 +120,6 @@ func (s *distanceStrategy) Split(dst, src chunk, tag splitTag, axesIndex int, he start++ } end := start + 1 - scale := helper.(distanceHelper).Scale switch tag { case splitTo: @@ -137,7 +129,7 @@ func (s *distanceStrategy) Split(dst, src chunk, tag splitTag, axesIndex int, he } value := src.Values[i] for ; start < end; start++ { - dst.Values[start] = uint64(float64(value) * scale[axesIndex][start]) + dst.Values[start] = uint64(float64(value) * e.Scale[axesIndex][start]) } end++ } @@ -148,7 +140,7 @@ func (s *distanceStrategy) Split(dst, src chunk, tag splitTag, axesIndex int, he } value := src.Values[i] for ; start < end; start++ { - dst.Values[start] += uint64(float64(value) * scale[axesIndex][start]) + dst.Values[start] += uint64(float64(value) * e.Scale[axesIndex][start]) } end++ } @@ -175,7 +167,7 @@ type scaleTask struct { Scale *[]float64 } -func (s *distanceStrategy) StartWorkers(ctx context.Context, wg *sync.WaitGroup) { +func (s *distanceSplitStrategy) StartWorkers(ctx context.Context, wg *sync.WaitGroup) { s.ScaleWorkerCh = make(chan *scaleTask, workerCount*100) wg.Add(workerCount) for i := 0; i < workerCount; i++ { @@ -186,11 +178,11 @@ func (s *distanceStrategy) StartWorkers(ctx context.Context, wg *sync.WaitGroup) } } -func (s *distanceStrategy) StopWorkers() { +func (s *distanceSplitStrategy) StopWorkers() { close(s.ScaleWorkerCh) } -func (s *distanceStrategy) GenerateScale(chunks []chunk, compactKeys []string, dis [][]int) [][]float64 { +func (s *distanceSplitStrategy) GenerateScale(chunks []chunk, compactKeys []string, dis [][]int) [][]float64 { var wg sync.WaitGroup axesLen := len(chunks) scale := make([][]float64, axesLen) @@ -208,7 +200,7 @@ func (s *distanceStrategy) GenerateScale(chunks []chunk, compactKeys []string, d return scale } -func (s *distanceStrategy) GenerateScaleColumnWork(ctx context.Context, ch <-chan *scaleTask) { +func (s *distanceSplitStrategy) GenerateScaleColumnWork(ctx context.Context, ch <-chan *scaleTask) { var maxDis int // Each split interval needs to be sorted after copying to tempDis var tempDis []int diff --git a/pkg/keyvisual/matrix/distance_test.go b/pkg/keyvisual/matrix/distance_test.go index b9cfc3cd20..da46c1c3ce 100644 --- a/pkg/keyvisual/matrix/distance_test.go +++ b/pkg/keyvisual/matrix/distance_test.go @@ -25,8 +25,6 @@ import ( . "github.com/pingcap/check" "go.uber.org/fx" - - "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" ) var _ = Suite(&testDistanceSuite{}) @@ -80,21 +78,22 @@ func BenchmarkGenerateScale(b *testing.B) { keymap.SaveKeys(compactKeys) keymap.SaveKeys(data.Keys) - var strategy *distanceStrategy + var strategy SplitStrategy wg := &sync.WaitGroup{} app := fx.New( - fx.Provide(func(lc fx.Lifecycle) Strategy { - return DistanceStrategy(lc, wg, decorator.NaiveLabelStrategy{}, 1.0/math.Phi, 15, 50).(*distanceStrategy) + fx.Provide(func(lc fx.Lifecycle) SplitStrategy { + return DistanceSplitStrategy(lc, wg, 1.0/math.Phi, 15, 50) }), fx.Populate(&strategy), ) _ = app.Start(context.Background()) + s := strategy.(*distanceSplitStrategy) b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() rollbackDis() b.StartTimer() - _ = strategy.GenerateScale(chunks, compactKeys, dis) + _ = s.GenerateScale(chunks, compactKeys, dis) } b.StopTimer() _ = app.Stop(context.Background()) diff --git a/pkg/keyvisual/matrix/interface.go b/pkg/keyvisual/matrix/interface.go index ba0a2f43bc..9a66dc3485 100644 --- a/pkg/keyvisual/matrix/interface.go +++ b/pkg/keyvisual/matrix/interface.go @@ -24,15 +24,18 @@ const ( splitAdd // Add to original value after split ) -// splitStrategy is an allocation scheme. GenerateHelper is used to generate a helper for a Plane. Split uses this -// helper to split a chunk of columns. -type splitStrategy interface { - GenerateHelper(chunks []chunk, compactKeys []string) interface{} - Split(dst, src chunk, tag splitTag, axesIndex int, helper interface{}) +// SplitStrategy is an allocation scheme. It is used to generate a Splitter for a plane to split a chunk of columns. +type SplitStrategy interface { + NewSplitter(chunks []chunk, compactKeys []string) Splitter +} + +type Splitter interface { + // Split a chunk of columns. + Split(dst, src chunk, tag splitTag, axesIndex int) } // Strategy is part of the customizable strategy in Matrix generation. -type Strategy interface { - splitStrategy +type Strategy struct { decorator.LabelStrategy + SplitStrategy } diff --git a/pkg/keyvisual/matrix/matrix.go b/pkg/keyvisual/matrix/matrix.go index dcae85fc61..89e6ca9886 100644 --- a/pkg/keyvisual/matrix/matrix.go +++ b/pkg/keyvisual/matrix/matrix.go @@ -29,22 +29,10 @@ type Matrix struct { } // CreateMatrix uses the specified times and keys to build an initial matrix with no data. -func CreateMatrix(strategy Strategy, times []time.Time, keys []string, valuesListLen int) Matrix { +func CreateMatrix(labeler decorator.Labeler, times []time.Time, keys []string, valuesListLen int) Matrix { dataMap := make(map[string][][]uint64, valuesListLen) // collect label keys - keyAxis := make([]decorator.LabelKey, len(keys)) - for i, key := range keys { - keyAxis[i] = strategy.Label(key) - } - - if keys[0] == "" { - keyAxis[0] = strategy.LabelGlobalStart() - } - endIndex := len(keys) - 1 - if keys[endIndex] == "" { - keyAxis[endIndex] = strategy.LabelGlobalEnd() - } - + keyAxis := labeler.Label(keys) // collect unix times timeAxis := make([]int64, len(times)) for i, t := range times { diff --git a/pkg/keyvisual/matrix/plane.go b/pkg/keyvisual/matrix/plane.go index 3e3f2e91cb..7da589767b 100644 --- a/pkg/keyvisual/matrix/plane.go +++ b/pkg/keyvisual/matrix/plane.go @@ -43,12 +43,12 @@ func CreateEmptyPlane(startTime, endTime time.Time, startKey, endKey string, val } // Compact compacts Plane into an axis. -func (plane *Plane) Compact(strategy Strategy) Axis { +func (plane *Plane) Compact(strategy SplitStrategy) Axis { chunks := make([]chunk, len(plane.Axes)) for i, axis := range plane.Axes { chunks[i] = createChunk(axis.Keys, axis.ValuesList[0]) } - compactChunk, helper := compact(strategy, chunks) + compactChunk, splitter := compact(strategy, chunks) valuesListLen := len(plane.Axes[0].ValuesList) valuesList := make([][]uint64, valuesListLen) valuesList[0] = compactChunk.Values @@ -56,7 +56,7 @@ func (plane *Plane) Compact(strategy Strategy) Axis { compactChunk.SetZeroValues() for i, axis := range plane.Axes { chunks[i].SetValues(axis.ValuesList[j]) - strategy.Split(compactChunk, chunks[i], splitAdd, i, helper) + splitter.Split(compactChunk, chunks[i], splitAdd, i) } valuesList[j] = compactChunk.Values } @@ -64,7 +64,7 @@ func (plane *Plane) Compact(strategy Strategy) Axis { } // Pixel pixelates Plane into a matrix with a number of rows close to the target. -func (plane *Plane) Pixel(strategy Strategy, target int, displayTags []string) Matrix { +func (plane *Plane) Pixel(strategy *Strategy, target int, displayTags []string) Matrix { valuesListLen := len(plane.Axes[0].ValuesList) if valuesListLen != len(displayTags) { panic("the length of displayTags and valuesList should be equal") @@ -74,9 +74,10 @@ func (plane *Plane) Pixel(strategy Strategy, target int, displayTags []string) M for i, axis := range plane.Axes { chunks[i] = createChunk(axis.Keys, axis.ValuesList[0]) } - compactChunk, helper := compact(strategy, chunks) - baseKeys := compactChunk.Divide(strategy, target, NotMergeLogicalRange).Keys - matrix := CreateMatrix(strategy, plane.Times, baseKeys, valuesListLen) + compactChunk, splitter := compact(strategy, chunks) + labeler := strategy.NewLabeler() + baseKeys := compactChunk.Divide(labeler, target, NotMergeLogicalRange).Keys + matrix := CreateMatrix(labeler, plane.Times, baseKeys, valuesListLen) var wg sync.WaitGroup var mutex sync.Mutex @@ -86,7 +87,7 @@ func (plane *Plane) Pixel(strategy Strategy, target int, displayTags []string) M goCompactChunk := createZeroChunk(compactChunk.Keys) for i, axis := range plane.Axes { goCompactChunk.Clear() - strategy.Split(goCompactChunk, createChunk(chunks[i].Keys, axis.ValuesList[j]), splitTo, i, helper) + splitter.Split(goCompactChunk, createChunk(chunks[i].Keys, axis.ValuesList[j]), splitTo, i) data[i] = goCompactChunk.Reduce(baseKeys).Values } mutex.Lock() @@ -103,7 +104,7 @@ func (plane *Plane) Pixel(strategy Strategy, target int, displayTags []string) M return matrix } -func compact(strategy Strategy, chunks []chunk) (compactChunk chunk, helper interface{}) { +func compact(strategy SplitStrategy, chunks []chunk) (compactChunk chunk, splitter Splitter) { // get compact chunk keys keySet := make(map[string]struct{}) unlimitedEnd := false @@ -128,9 +129,9 @@ func compact(strategy Strategy, chunks []chunk) (compactChunk chunk, helper inte } compactChunk = createZeroChunk(compactKeys) - helper = strategy.GenerateHelper(chunks, compactChunk.Keys) + splitter = strategy.NewSplitter(chunks, compactChunk.Keys) for i, c := range chunks { - strategy.Split(compactChunk, c, splitAdd, i, helper) + splitter.Split(compactChunk, c, splitAdd, i) } return } diff --git a/pkg/keyvisual/service.go b/pkg/keyvisual/service.go index 5fa17abcb0..b08d9f15b5 100644 --- a/pkg/keyvisual/service.go +++ b/pkg/keyvisual/service.go @@ -82,7 +82,7 @@ type Service struct { tidbClient *tidb.Client stat *storage.Stat - strategy matrix.Strategy + strategy *matrix.Strategy labelStrategy decorator.LabelStrategy } @@ -164,14 +164,13 @@ func (s *Service) Start(ctx context.Context) error { func (s *Service) newLabelStrategy( lc fx.Lifecycle, wg *sync.WaitGroup, - cfg *config.Config, etcdClient *clientv3.Client, tidbClient *tidb.Client, ) decorator.LabelStrategy { switch s.keyVisualCfg.Policy { case config.KeyVisualDBPolicy: log.Debug("New LabelStrategy", zap.String("policy", s.keyVisualCfg.Policy)) - return decorator.TiDBLabelStrategy(lc, wg, cfg, etcdClient, tidbClient) + return decorator.TiDBLabelStrategy(lc, wg, etcdClient, tidbClient) case config.KeyVisualKVPolicy: log.Debug("New LabelStrategy", zap.String("policy", s.keyVisualCfg.Policy), zap.String("separator", s.keyVisualCfg.PolicyKVSeparator)) @@ -318,11 +317,26 @@ func newWaitGroup(lc fx.Lifecycle) *sync.WaitGroup { return wg } -func newStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, labelStrategy decorator.LabelStrategy) matrix.Strategy { - return matrix.DistanceStrategy(lc, wg, labelStrategy, distanceStrategyRatio, distanceStrategyLevel, distanceStrategyCount) +func newStrategy(lc fx.Lifecycle, wg *sync.WaitGroup, labelStrategy decorator.LabelStrategy) *matrix.Strategy { + return &matrix.Strategy{ + LabelStrategy: labelStrategy, + SplitStrategy: matrix.DistanceSplitStrategy( + lc, wg, + distanceStrategyRatio, + distanceStrategyLevel, + distanceStrategyCount, + ), + } } -func newStat(lc fx.Lifecycle, wg *sync.WaitGroup, etcdClient *clientv3.Client, db *dbstore.DB, in input.StatInput, strategy matrix.Strategy) *storage.Stat { +func newStat( + lc fx.Lifecycle, + wg *sync.WaitGroup, + etcdClient *clientv3.Client, + db *dbstore.DB, + in input.StatInput, + strategy *matrix.Strategy, +) *storage.Stat { stat := storage.NewStat(lc, wg, db, defaultStatConfig, strategy, in.GetStartTime()) lc.Append(fx.Hook{ diff --git a/pkg/keyvisual/storage/model.go b/pkg/keyvisual/storage/model.go index 3e10965929..08442ff7a7 100644 --- a/pkg/keyvisual/storage/model.go +++ b/pkg/keyvisual/storage/model.go @@ -1,3 +1,16 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package storage import ( diff --git a/pkg/keyvisual/storage/model_test.go b/pkg/keyvisual/storage/model_test.go index 64a53a5730..1c3a60681d 100644 --- a/pkg/keyvisual/storage/model_test.go +++ b/pkg/keyvisual/storage/model_test.go @@ -1,3 +1,16 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package storage import ( diff --git a/pkg/keyvisual/storage/region.go b/pkg/keyvisual/storage/region.go index f25b160bcf..7f0cede362 100644 --- a/pkg/keyvisual/storage/region.go +++ b/pkg/keyvisual/storage/region.go @@ -17,6 +17,7 @@ import ( "github.com/pingcap/log" "go.uber.org/zap" + "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" "github.com/pingcap/tidb-dashboard/pkg/keyvisual/matrix" "github.com/pingcap/tidb-dashboard/pkg/keyvisual/region" ) @@ -27,11 +28,11 @@ const ( // preRatioTarget = 512 preTarget = 3072 - dirtyWrittenBytes = 1 << 32 + dirtyWrittenBytes uint64 = 1 << 32 ) // CreateStorageAxis converts the RegionsInfo to a StorageAxis. -func CreateStorageAxis(regions region.RegionsInfo, strategy matrix.Strategy) matrix.Axis { +func CreateStorageAxis(regions region.RegionsInfo, labeler decorator.Labeler) matrix.Axis { regionsLen := regions.Len() if regionsLen <= 0 { panic("At least one RegionInfo") @@ -46,15 +47,15 @@ func CreateStorageAxis(regions region.RegionsInfo, strategy matrix.Strategy) mat preAxis := matrix.CreateAxis(keys, valuesList) wash(&preAxis) - axis := IntoStorageAxis(preAxis, strategy) + axis := IntoStorageAxis(preAxis, labeler) log.Debug("New StorageAxis", zap.Int("region length", regionsLen), zap.Int("focus keys length", len(axis.Keys))) return axis } // IntoStorageAxis converts ResponseAxis to StorageAxis. -func IntoStorageAxis(responseAxis matrix.Axis, strategy matrix.Strategy) matrix.Axis { +func IntoStorageAxis(responseAxis matrix.Axis, labeler decorator.Labeler) matrix.Axis { // axis := preAxis.Focus(strategy, preThreshold, len(keys)/preRatioTarget, preTarget) - axis := responseAxis.Divide(strategy, preTarget) + axis := responseAxis.Divide(labeler, preTarget) var storageValuesList [][]uint64 storageValuesList = append(storageValuesList, axis.ValuesList[1:]...) return matrix.CreateAxis(axis.Keys, storageValuesList) diff --git a/pkg/keyvisual/storage/stat.go b/pkg/keyvisual/storage/stat.go index a50cdce612..c73690f988 100644 --- a/pkg/keyvisual/storage/stat.go +++ b/pkg/keyvisual/storage/stat.go @@ -23,6 +23,7 @@ import ( "go.uber.org/fx" "github.com/pingcap/tidb-dashboard/pkg/dbstore" + "github.com/pingcap/tidb-dashboard/pkg/keyvisual/decorator" "github.com/pingcap/tidb-dashboard/pkg/keyvisual/matrix" "github.com/pingcap/tidb-dashboard/pkg/keyvisual/region" ) @@ -49,31 +50,37 @@ type layerStat struct { Db *dbstore.DB // Hierarchical mechanism - Strategy matrix.Strategy - Ratio int - Next *layerStat + SplitStrategy matrix.SplitStrategy + Ratio int + Next *layerStat } -func newLayerStat(layerNum uint8, conf LayerConfig, strategy matrix.Strategy, startTime time.Time, db *dbstore.DB) *layerStat { +func newLayerStat( + layerNum uint8, + conf LayerConfig, + splitStrategy matrix.SplitStrategy, + startTime time.Time, + db *dbstore.DB, +) *layerStat { return &layerStat{ - StartTime: startTime, - EndTime: startTime, - RingAxes: make([]matrix.Axis, conf.Len), - RingTimes: make([]time.Time, conf.Len), - LayerNum: layerNum, - Head: 0, - Tail: 0, - Empty: true, - Len: conf.Len, - Db: db, - Strategy: strategy, - Ratio: conf.Ratio, - Next: nil, + StartTime: startTime, + EndTime: startTime, + RingAxes: make([]matrix.Axis, conf.Len), + RingTimes: make([]time.Time, conf.Len), + LayerNum: layerNum, + Head: 0, + Tail: 0, + Empty: true, + Len: conf.Len, + Db: db, + SplitStrategy: splitStrategy, + Ratio: conf.Ratio, + Next: nil, } } // Reduce merges ratio axes and append to next layerStat -func (s *layerStat) Reduce() { +func (s *layerStat) Reduce(labeler decorator.Labeler) { if s.Ratio == 0 || s.Next == nil { _ = s.DeleteFirstAxisFromDb() @@ -98,17 +105,17 @@ func (s *layerStat) Reduce() { } plane := matrix.CreatePlane(times, axes) - newAxis := plane.Compact(s.Strategy) + newAxis := plane.Compact(s.SplitStrategy) newAxis = IntoResponseAxis(newAxis, region.Integration) - newAxis = IntoStorageAxis(newAxis, s.Strategy) + newAxis = IntoStorageAxis(newAxis, labeler) newAxis.Shrink(uint64(s.Ratio)) - s.Next.Append(newAxis, s.StartTime) + s.Next.Append(newAxis, s.StartTime, labeler) } // Append appends a key axis to layerStat. -func (s *layerStat) Append(axis matrix.Axis, endTime time.Time) { +func (s *layerStat) Append(axis matrix.Axis, endTime time.Time, labeler decorator.Labeler) { if s.Head == s.Tail && !s.Empty { - s.Reduce() + s.Reduce(labeler) } _ = s.InsertLastAxisToDb(axis, endTime) @@ -181,13 +188,20 @@ type Stat struct { layers []*layerStat keyMap matrix.KeyMap - strategy matrix.Strategy + strategy *matrix.Strategy db *dbstore.DB } // NewStat generates a Stat based on the configuration. -func NewStat(lc fx.Lifecycle, wg *sync.WaitGroup, db *dbstore.DB, cfg StatConfig, strategy matrix.Strategy, startTime time.Time) *Stat { +func NewStat( + lc fx.Lifecycle, + wg *sync.WaitGroup, + db *dbstore.DB, + cfg StatConfig, + strategy *matrix.Strategy, + startTime time.Time, +) *Stat { layers := make([]*layerStat, len(cfg.LayersConfig)) for i, c := range cfg.LayersConfig { layers[i] = newLayerStat(uint8(i), c, strategy, startTime, db) @@ -253,7 +267,8 @@ func (s *Stat) Append(regions region.RegionsInfo, endTime time.Time) { if regions.Len() == 0 { return } - axis := CreateStorageAxis(regions, s.strategy) + labeler := s.strategy.NewLabeler() + axis := CreateStorageAxis(regions, labeler) s.keyMap.RLock() defer s.keyMap.RUnlock() @@ -261,7 +276,7 @@ func (s *Stat) Append(regions region.RegionsInfo, endTime time.Time) { s.mutex.Lock() defer s.mutex.Unlock() - s.layers[0].Append(axis, endTime) + s.layers[0].Append(axis, endTime, labeler) } func (s *Stat) rangeRoot(startTime, endTime time.Time) ([]time.Time, []matrix.Axis) { diff --git a/pkg/keyvisual/storage/stat_persist.go b/pkg/keyvisual/storage/stat_persist.go index f548813c10..00785bc8ec 100644 --- a/pkg/keyvisual/storage/stat_persist.go +++ b/pkg/keyvisual/storage/stat_persist.go @@ -1,3 +1,16 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package storage import ( diff --git a/pkg/tidb/model/codec.go b/pkg/tidb/model/codec.go index a341acadf8..6414d4bb28 100644 --- a/pkg/tidb/model/codec.go +++ b/pkg/tidb/model/codec.go @@ -17,7 +17,7 @@ import ( "bytes" "encoding/binary" - "github.com/pkg/errors" + "github.com/pingcap/errors" ) var ( @@ -34,85 +34,135 @@ const ( encPad = byte(0x0) ) -// Key represents high-level Key type. +// Key represents high-level TiDB Key type. type Key []byte -// TableID returns the table ID of the key, if the key is not table key, returns 0. -func (k Key) TableID() int64 { - _, key, err := DecodeBytes(k) +// KeyInfoBuffer can obtain the meta information of the TiDB Key. +// It can be reused, thereby reducing memory applications. +type KeyInfoBuffer []byte + +// DecodeKey obtains the KeyInfoBuffer from a TiDB Key +func (buf *KeyInfoBuffer) DecodeKey(key Key) (KeyInfoBuffer, error) { + _, result, err := decodeBytes(key, *buf) + if err != nil { - // should never happen - return 0 + *buf = (*buf)[:0] + return nil, err } - if !bytes.HasPrefix(key, tablePrefix) { - return 0 - } - key = key[len(tablePrefix):] - _, tableID, _ := DecodeInt(key) - return tableID + *buf = result + return result, nil } -// RowID returns the row ID of the key, if the key is not table key, returns 0. -func (k Key) RowID() int64 { - _, key, err := DecodeBytes(k) - if err != nil { - // should never happen - return 0 - } - if !bytes.HasPrefix(key, tablePrefix) { - return 0 +// MetaOrTable checks if the key is a meta key or table key. +// If the key is a meta key, it returns true and 0. +// If the key is a table key, it returns false and table ID. +// Otherwise, it returns false and 0. +func (buf KeyInfoBuffer) MetaOrTable() (isMeta bool, tableID int64) { + if bytes.HasPrefix(buf, metaPrefix) { + return true, 0 } - if len(key) < 19 || !(key[9] == '_' && key[10] == 'r') { - return 0 + if bytes.HasPrefix(buf, tablePrefix) { + _, tableID, _ := decodeInt(buf[len(tablePrefix):]) + return false, tableID } - key = key[11:19] - - _, rowID, _ := DecodeInt(key) - return rowID + return false, 0 } -// IndexID returns the row ID of the key, if the key is not table key, returns 0. -func (k Key) IndexID() int64 { - _, key, err := DecodeBytes(k) - if err != nil { - // should never happen - return 0 +// RowInfo returns the row ID of the key, if the key is not table key, returns 0. +func (buf KeyInfoBuffer) RowInfo() (isCommonHandle bool, rowID int64) { + if !bytes.HasPrefix(buf, tablePrefix) || len(buf) < 19 || !(buf[9] == '_' && buf[10] == 'r') { + return } - if !bytes.HasPrefix(key, tablePrefix) { - return 0 + isCommonHandle = len(buf) != 19 + if !isCommonHandle { + _, rowID, _ = decodeInt(buf[11:19]) } - if len(key) < 19 || !(key[9] == '_' && key[10] == 'i') { - return 0 + return +} + +// IndexInfo returns the row ID of the key, if the key is not table key, returns 0. +func (buf KeyInfoBuffer) IndexInfo() (indexID int64) { + if !bytes.HasPrefix(buf, tablePrefix) || len(buf) < 19 || !(buf[9] == '_' && buf[10] == 'i') { + return } - key = key[11:19] - _, indexID, _ := DecodeInt(key) - return indexID + _, indexID, _ = decodeInt(buf[11:19]) + return } -// MetaOrTable checks if the key is a meta key or table key. -// If the key is a meta key, it returns true and 0. -// If the key is a table key, it returns false and table ID. -// Otherwise, it returns false and 0. -func (k Key) MetaOrTable() (bool, int64) { - _, key, err := DecodeBytes(k) - if err != nil { - return false, 0 +// GenerateTableKey generates a table split key. +func (buf *KeyInfoBuffer) GenerateKey(tableID, rowID int64) Key { + if tableID == 0 { + return nil } - if bytes.HasPrefix(key, metaPrefix) { - return true, 0 + + data := *buf + if data == nil { + length := len(tablePrefix) + 8 + if rowID != 0 { + length = len(tablePrefix) + len(recordPrefix) + 8*2 + } + data = make([]byte, 0, length) + } else { + data = data[:0] } - if bytes.HasPrefix(key, tablePrefix) { - key = key[len(tablePrefix):] - _, tableID, _ := DecodeInt(key) - return false, tableID + + data = append(data, tablePrefix...) + data = encodeInt(data, tableID) + if rowID != 0 { + data = append(data, recordPrefix...) + data = encodeInt(data, rowID) } - return false, 0 + + *buf = data + + return encodeBytes(data) } var pads = make([]byte, encGroupSize) -// EncodeBytes guarantees the encoded value is in ascending order for comparison, +// decodeBytes decodes bytes which is encoded by encodeBytes before, +// returns the leftover bytes and decoded value if no error. +func decodeBytes(b []byte, buf []byte) (rest []byte, result []byte, err error) { + if buf == nil { + buf = make([]byte, 0, len(b)) + } + buf = buf[:0] + + for { + if len(b) < encGroupSize+1 { + return nil, nil, errors.New("insufficient bytes to decode value") + } + + groupBytes := b[:encGroupSize+1] + + group := groupBytes[:encGroupSize] + marker := groupBytes[encGroupSize] + + padCount := encMarker - marker + if padCount > encGroupSize { + return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes) + } + + realGroupSize := encGroupSize - padCount + buf = append(buf, group[:realGroupSize]...) + b = b[encGroupSize+1:] + + if padCount != 0 { + // Check validity of padding bytes. + for _, v := range group[realGroupSize:] { + if v != encPad { + return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes) + } + } + break + } + } + + return b, buf, nil +} + +// encodeBytes guarantees the encoded value is in ascending order for comparison, // encoding with the following rule: // [group1][marker1]...[groupN][markerN] // group is 8 bytes slice which is padding with 0. @@ -123,7 +173,7 @@ var pads = make([]byte, encGroupSize) // [1, 2, 3, 0] -> [1, 2, 3, 0, 0, 0, 0, 0, 251] // [1, 2, 3, 4, 5, 6, 7, 8] -> [1, 2, 3, 4, 5, 6, 7, 8, 255, 0, 0, 0, 0, 0, 0, 0, 0, 247] // Refer: https://github.com/facebook/mysql-5.6/wiki/MyRocks-record-format#memcomparable-format -func EncodeBytes(data []byte) Key { +func encodeBytes(data []byte) []byte { // Allocate more space to avoid unnecessary slice growing. // Assume that the byte slice size is about `(len(data) / encGroupSize + 1) * (encGroupSize + 1)` bytes, // that is `(len(data) / 8 + 1) * 9` in our implement. @@ -146,18 +196,9 @@ func EncodeBytes(data []byte) Key { return result } -// EncodeInt appends the encoded value to slice b and returns the appended slice. -// EncodeInt guarantees that the encoded value is in ascending order for comparison. -func EncodeInt(b []byte, v int64) []byte { - var data [8]byte - u := encodeIntToCmpUint(v) - binary.BigEndian.PutUint64(data[:], u) - return append(b, data[:]...) -} - -// DecodeInt decodes value encoded by EncodeInt before. +// decodeInt decodes value encoded by EncodeInt before. // It returns the leftover un-decoded slice, decoded value if no error. -func DecodeInt(b []byte) ([]byte, int64, error) { +func decodeInt(b []byte) ([]byte, int64, error) { if len(b) < 8 { return nil, 0, errors.New("insufficient bytes to decode value") } @@ -168,65 +209,19 @@ func DecodeInt(b []byte) ([]byte, int64, error) { return b, v, nil } -func encodeIntToCmpUint(v int64) uint64 { - return uint64(v) ^ signMask +// encodeInt appends the encoded value to slice b and returns the appended slice. +// encodeInt guarantees that the encoded value is in ascending order for comparison. +func encodeInt(b []byte, v int64) []byte { + var data [8]byte + u := encodeIntToCmpUint(v) + binary.BigEndian.PutUint64(data[:], u) + return append(b, data[:]...) } func decodeCmpUintToInt(u uint64) int64 { return int64(u ^ signMask) } -// DecodeBytes decodes bytes which is encoded by EncodeBytes before, -// returns the leftover bytes and decoded value if no error. -func DecodeBytes(b []byte) ([]byte, []byte, error) { - data := make([]byte, 0, len(b)) - for { - if len(b) < encGroupSize+1 { - return nil, nil, errors.New("insufficient bytes to decode value") - } - - groupBytes := b[:encGroupSize+1] - - group := groupBytes[:encGroupSize] - marker := groupBytes[encGroupSize] - - padCount := encMarker - marker - if padCount > encGroupSize { - return nil, nil, errors.Errorf("invalid marker byte, group bytes %q", groupBytes) - } - - realGroupSize := encGroupSize - padCount - data = append(data, group[:realGroupSize]...) - b = b[encGroupSize+1:] - - if padCount != 0 { - var padByte = encPad - // Check validity of padding bytes. - for _, v := range group[realGroupSize:] { - if v != padByte { - return nil, nil, errors.Errorf("invalid padding byte, group bytes %q", groupBytes) - } - } - break - } - } - return b, data, nil -} - -// GenerateTableKey generates a table split key. -func GenerateTableKey(tableID int64) []byte { - buf := make([]byte, 0, len(tablePrefix)+8) - buf = append(buf, tablePrefix...) - buf = EncodeInt(buf, tableID) - return buf -} - -// GenerateRowKey generates a row key. -func GenerateRowKey(tableID, rowID int64) []byte { - buf := make([]byte, 0, len(tablePrefix)+len(recordPrefix)+8*2) - buf = append(buf, tablePrefix...) - buf = EncodeInt(buf, tableID) - buf = append(buf, recordPrefix...) - buf = EncodeInt(buf, rowID) - return buf +func encodeIntToCmpUint(v int64) uint64 { + return uint64(v) ^ signMask } diff --git a/pkg/tidb/model/codec_test.go b/pkg/tidb/model/codec_test.go index acf682bb0c..bf365a442d 100644 --- a/pkg/tidb/model/codec_test.go +++ b/pkg/tidb/model/codec_test.go @@ -30,25 +30,96 @@ type testCodecSuite struct{} func (s *testCodecSuite) TestDecodeBytes(c *C) { key := "abcdefghijklmnopqrstuvwxyz" for i := 0; i < len(key); i++ { - _, k, err := DecodeBytes(EncodeBytes([]byte(key[:i]))) + _, k, err := decodeBytes(encodeBytes([]byte(key[:i])), nil) c.Assert(err, IsNil) c.Assert(string(k), Equals, key[:i]) } } -func (s *testCodecSuite) TestTableID(c *C) { - key := EncodeBytes([]byte("t\x80\x00\x00\x00\x00\x00\x00\xff")) - c.Assert(key.TableID(), Equals, int64(0xff)) +func (s *testCodecSuite) TestTiDBInfo(c *C) { + buf := new(KeyInfoBuffer) - key = EncodeBytes([]byte("t\x80\x00\x00\x00\x00\x00\x00\xff_i\x01\x02")) - c.Assert(key.TableID(), Equals, int64(0xff)) + // no encode + _, err := buf.DecodeKey([]byte("t\x80\x00\x00\x00\x00\x00\x00\xff")) + c.Assert(err, NotNil) - key = []byte("t\x80\x00\x00\x00\x00\x00\x00\xff") - c.Assert(key.TableID(), Equals, int64(0)) - - key = EncodeBytes([]byte("T\x00\x00\x00\x00\x00\x00\x00\xff")) - c.Assert(key.TableID(), Equals, int64(0)) + testcases := []struct { + Key string + IsMeta bool + TableID int64 + IsCommonHandle bool + RowID int64 + IndexID int64 + }{ + { + "T\x00\x00\x00\x00\x00\x00\x00\xff", + false, + 0, + false, + 0, + 0, + }, + { + "t\x80\x00\x00\x00\x00\x00\xff", + false, + 0, + false, + 0, + 0, + }, + { + "t\x80\x00\x00\x00\x00\x00\x00\xff", + false, + 0xff, + false, + 0, + 0, + }, + { + "t\x80\x00\x00\x00\x00\x00\x00\xff_i\x01\x02", + false, + 0xff, + false, + 0, + 0, + }, + { + "t\x80\x00\x00\x00\x00\x00\x00\xff_i\x80\x00\x00\x00\x00\x00\x00\x02", + false, + 0xff, + false, + 0, + 2, + }, + { + "t\x80\x00\x00\x00\x00\x00\x00\xff_r\x80\x00\x00\x00\x00\x00\x00\x02", + false, + 0xff, + false, + 2, + 0, + }, + { + "t\x80\x00\x00\x00\x00\x00\x00\xff_r\x03\x80\x00\x00\x00\x00\x02\r\xaf\x03\x80\x00\x00\x00\x00\x00\x00\x03\x03\x80\x00\x00\x00\x00\x00\b%", + false, + 0xff, + true, + 0, + 0, + }, + } - key = EncodeBytes([]byte("t\x80\x00\x00\x00\x00\x00\xff")) - c.Assert(key.TableID(), Equals, int64(0)) + for _, t := range testcases { + key := encodeBytes([]byte(t.Key)) + _, err := buf.DecodeKey(key) + c.Assert(err, IsNil) + isMeta, tableID := buf.MetaOrTable() + c.Assert(isMeta, Equals, t.IsMeta) + c.Assert(tableID, Equals, t.TableID) + isCommonHandle, rowID := buf.RowInfo() + c.Assert(isCommonHandle, Equals, t.IsCommonHandle) + c.Assert(rowID, Equals, t.RowID) + indexID := buf.IndexInfo() + c.Assert(indexID, Equals, t.IndexID) + } } diff --git a/ui/lib/apps/KeyViz/heatmap/axis/label-axis.ts b/ui/lib/apps/KeyViz/heatmap/axis/label-axis.ts index 138ae92eec..d0e4143d8b 100644 --- a/ui/lib/apps/KeyViz/heatmap/axis/label-axis.ts +++ b/ui/lib/apps/KeyViz/heatmap/axis/label-axis.ts @@ -62,7 +62,7 @@ export function labelAxisGroup(keyAxis: KeyAxisEntry[]) { ctx.stroke() ctx.closePath() - if (shouleShowLabelText(label)) { + if (shouldShowLabelText(label)) { ctx.font = label.focus ? focusFont : font ctx.fillStyle = label.focus ? textFillFocus : textFill ctx.translate( @@ -81,7 +81,7 @@ export function labelAxisGroup(keyAxis: KeyAxisEntry[]) { return labelAxisGroup } -function shouleShowLabelText(label: DisplayLabel): boolean { +function shouldShowLabelText(label: DisplayLabel): boolean { return ( label.endPos - label.startPos >= minTextHeight && label.val?.length !== 0 )