From 460d24d3a458f45109a42aac7854f07a9d91e74b Mon Sep 17 00:00:00 2001 From: Shenghui Wu <793703860@qq.com> Date: Wed, 6 Nov 2019 14:51:10 +0800 Subject: [PATCH] *: rename Arrow to Chunk (#13060) --- config/config.go | 6 +++--- config/config.toml.example | 4 ++-- distsql/distsql.go | 14 +++++++------- distsql/distsql_test.go | 12 ++++++------ distsql/select_result.go | 8 ++++---- go.mod | 2 +- go.sum | 6 +++--- sessionctx/variable/session.go | 18 +++++++++--------- sessionctx/variable/sysvar.go | 2 +- sessionctx/variable/tidb_vars.go | 4 ++-- sessionctx/variable/varsutil.go | 2 +- statistics/handle/handle.go | 2 +- store/mockstore/mocktikv/cop_handler_dag.go | 8 ++++---- tidb-server/main.go | 2 +- util/chunk/chunk_test.go | 2 +- 15 files changed, 46 insertions(+), 46 deletions(-) diff --git a/config/config.go b/config/config.go index 6f1add6aa93e4..b3cbfe1ec0012 100644 --- a/config/config.go +++ b/config/config.go @@ -369,8 +369,8 @@ type TiKVClient struct { MaxBatchWaitTime time.Duration `toml:"max-batch-wait-time" json:"max-batch-wait-time"` // BatchWaitSize is the max wait size for batch. BatchWaitSize uint `toml:"batch-wait-size" json:"batch-wait-size"` - // EnableArrow indicate the data encode in arrow format. - EnableArrow bool `toml:"enable-arrow" json:"enable-arrow"` + // EnableChunkRPC indicate the data encode in chunk format for coprocessor requests. + EnableChunkRPC bool `toml:"enable-chunk-rpc" json:"enable-chunk-rpc"` // If a Region has not been accessed for more than the given duration (in seconds), it // will be reloaded from the PD. RegionCacheTTL uint `toml:"region-cache-ttl" json:"region-cache-ttl"` @@ -499,7 +499,7 @@ var defaultConf = Config{ MaxBatchWaitTime: 0, BatchWaitSize: 8, - EnableArrow: true, + EnableChunkRPC: true, RegionCacheTTL: 600, }, diff --git a/config/config.toml.example b/config/config.toml.example index 5fcba7271ea24..130eea6719a69 100644 --- a/config/config.toml.example +++ b/config/config.toml.example @@ -283,8 +283,8 @@ max-batch-wait-time = 0 # Batch wait size, to avoid waiting too long. batch-wait-size = 8 -# Enable chunk encoded data. -enable-arrow = true +# Enable chunk encoded data for coprocessor requests. +enable-chunk-rpc = true # If a Region has not been accessed for more than the given duration (in seconds), it # will be reloaded from the PD. diff --git a/distsql/distsql.go b/distsql/distsql.go index 023f627cfb383..68ec78a3901fa 100644 --- a/distsql/distsql.go +++ b/distsql/distsql.go @@ -76,8 +76,8 @@ func Select(ctx context.Context, sctx sessionctx.Context, kvReq *kv.Request, fie }, nil } encodetype := tipb.EncodeType_TypeDefault - if enableTypeArrow(sctx) { - encodetype = tipb.EncodeType_TypeArrow + if canUseChunkRPC(sctx) { + encodetype = tipb.EncodeType_TypeChunk } return &selectResult{ label: "dag", @@ -152,18 +152,18 @@ func Checksum(ctx context.Context, client kv.Client, kvReq *kv.Request, vars *kv // SetEncodeType sets the encoding method for the DAGRequest. The supported encoding // methods are: -// 1. TypeArrow: the result is encoded using the Chunk format, refer util/chunk/chunk.go +// 1. TypeChunk: the result is encoded using the Chunk format, refer util/chunk/chunk.go // 2. TypeDefault: the result is encoded row by row func SetEncodeType(ctx sessionctx.Context, dagReq *tipb.DAGRequest) { - if enableTypeArrow(ctx) { - dagReq.EncodeType = tipb.EncodeType_TypeArrow + if canUseChunkRPC(ctx) { + dagReq.EncodeType = tipb.EncodeType_TypeChunk } else { dagReq.EncodeType = tipb.EncodeType_TypeDefault } } -func enableTypeArrow(ctx sessionctx.Context) bool { - if !ctx.GetSessionVars().EnableArrow { +func canUseChunkRPC(ctx sessionctx.Context) bool { + if !ctx.GetSessionVars().EnableChunkRPC { return false } if ctx.GetSessionVars().EnableStreaming { diff --git a/distsql/distsql_test.go b/distsql/distsql_test.go index 715e61cd0acaa..5195e1996eff2 100644 --- a/distsql/distsql_test.go +++ b/distsql/distsql_test.go @@ -127,7 +127,7 @@ func (s *testSuite) TestSelectMemTracker(c *C) { } func (s *testSuite) TestSelectNormalChunkSize(c *C) { - s.sctx.GetSessionVars().EnableArrow = false + s.sctx.GetSessionVars().EnableChunkRPC = false response, colTypes := s.createSelectNormal(100, 1000000, c, nil) response.Fetch(context.TODO()) s.testChunkSize(response, colTypes, c) @@ -289,7 +289,7 @@ func (s *testSuite) testChunkSize(response SelectResult, colTypes []*types.Field } func (s *testSuite) TestAnalyze(c *C) { - s.sctx.GetSessionVars().EnableArrow = false + s.sctx.GetSessionVars().EnableChunkRPC = false request, err := (&RequestBuilder{}).SetKeyRanges(nil). SetAnalyzeRequest(&tipb.AnalyzeReq{}). SetKeepOrder(true). @@ -315,7 +315,7 @@ func (s *testSuite) TestAnalyze(c *C) { } func (s *testSuite) TestChecksum(c *C) { - s.sctx.GetSessionVars().EnableArrow = false + s.sctx.GetSessionVars().EnableChunkRPC = false request, err := (&RequestBuilder{}).SetKeyRanges(nil). SetChecksumRequest(&tipb.ChecksumRequest{}). Build() @@ -370,7 +370,7 @@ func (resp *mockResponse) Next(ctx context.Context) (kv.ResultSubset, error) { resp.count += numRows var chunks []tipb.Chunk - if !enableTypeArrow(resp.ctx) { + if !canUseChunkRPC(resp.ctx) { datum := types.NewIntDatum(1) bytes := make([]byte, 0, 100) bytes, _ = codec.EncodeValue(nil, bytes, datum, datum, datum, datum) @@ -408,8 +408,8 @@ func (resp *mockResponse) Next(ctx context.Context) (kv.ResultSubset, error) { Chunks: chunks, OutputCounts: []int64{1}, } - if enableTypeArrow(resp.ctx) { - respPB.EncodeType = tipb.EncodeType_TypeArrow + if canUseChunkRPC(resp.ctx) { + respPB.EncodeType = tipb.EncodeType_TypeChunk } else { respPB.EncodeType = tipb.EncodeType_TypeDefault } diff --git a/distsql/select_result.go b/distsql/select_result.go index a2393d770f804..53c1b7945ac22 100644 --- a/distsql/select_result.go +++ b/distsql/select_result.go @@ -147,7 +147,7 @@ func (r *selectResult) NextRaw(ctx context.Context) (data []byte, err error) { // Next reads data to the chunk. func (r *selectResult) Next(ctx context.Context, chk *chunk.Chunk) error { chk.Reset() - // Check the returned data is default/arrow format. + // Check the returned data is default/chunk format. if r.selectResp == nil || r.respChkIdx == len(r.selectResp.Chunks) { err := r.getSelectResp() if err != nil || r.selectResp == nil { @@ -158,8 +158,8 @@ func (r *selectResult) Next(ctx context.Context, chk *chunk.Chunk) error { switch r.selectResp.GetEncodeType() { case tipb.EncodeType_TypeDefault: return r.readFromDefault(ctx, chk) - case tipb.EncodeType_TypeArrow: - return r.readFromArrow(ctx, chk) + case tipb.EncodeType_TypeChunk: + return r.readFromChunk(ctx, chk) } return errors.Errorf("unsupported encode type:%v", r.encodeType) } @@ -183,7 +183,7 @@ func (r *selectResult) readFromDefault(ctx context.Context, chk *chunk.Chunk) er return nil } -func (r *selectResult) readFromArrow(ctx context.Context, chk *chunk.Chunk) error { +func (r *selectResult) readFromChunk(ctx context.Context, chk *chunk.Chunk) error { if r.respChunkDecoder == nil { r.respChunkDecoder = chunk.NewDecoder( chunk.NewChunkWithCapacity(r.fieldTypes, 0), diff --git a/go.mod b/go.mod index a58247ec3639a..5fa386e598575 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/pingcap/parser v0.0.0-20191031081038-bfb0c3adf567 github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible - github.com/pingcap/tipb v0.0.0-20191029074152-e6f0f14af644 + github.com/pingcap/tipb v0.0.0-20191105120856-bd4b782c8393 github.com/prometheus/client_golang v0.9.0 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect diff --git a/go.sum b/go.sum index 210943b036429..9472b9fbc4e0f 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bc github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20191029074152-e6f0f14af644 h1:J+nYGNqumgP4jtBz5Nqre1wiE/HrLXrJpFpqOotfoNc= -github.com/pingcap/tipb v0.0.0-20191029074152-e6f0f14af644/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191105120856-bd4b782c8393 h1:8XcpRME085GsIe3eiJGhmuDPAjG8CUa8VE/QnQAwmfM= +github.com/pingcap/tipb v0.0.0-20191105120856-bd4b782c8393/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -204,7 +204,6 @@ github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -243,6 +242,7 @@ github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Y go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf h1:rmttwKPEgG/l4UscTDYtaJgeUsedKPKSyFfNQLI6q+I= go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= go.etcd.io/etcd v0.5.0-alpha.5.0.20190320044326-77d4b742cdbf h1:2pxGooJi3rmECPOvyqOyZgqqcKOF8Pg30aA1RXK4VuE= go.etcd.io/etcd v0.5.0-alpha.5.0.20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 9f04af1451ead..ec3a59511cb9c 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -366,8 +366,8 @@ type SessionVars struct { // TODO: remove this after tidb-server configuration "enable-streaming' removed. EnableStreaming bool - // EnableArrow indicates whether the coprocessor request can use arrow API. - EnableArrow bool + // EnableChunkRPC indicates whether the coprocessor request can use chunk API. + EnableChunkRPC bool writeStmtBufs WriteStmtBufs @@ -566,13 +566,13 @@ func NewSessionVars() *SessionVars { } terror.Log(vars.SetSystemVar(TiDBEnableStreaming, enableStreaming)) - var enableArrow string - if config.GetGlobalConfig().TiKVClient.EnableArrow { - enableArrow = "1" + var enableChunkRPC string + if config.GetGlobalConfig().TiKVClient.EnableChunkRPC { + enableChunkRPC = "1" } else { - enableArrow = "0" + enableChunkRPC = "0" } - terror.Log(vars.SetSystemVar(TiDBEnableArrow, enableArrow)) + terror.Log(vars.SetSystemVar(TiDBEnableChunkRPC, enableChunkRPC)) return vars } @@ -920,8 +920,8 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { s.DisableTxnAutoRetry = TiDBOptOn(val) case TiDBEnableStreaming: s.EnableStreaming = TiDBOptOn(val) - case TiDBEnableArrow: - s.EnableArrow = TiDBOptOn(val) + case TiDBEnableChunkRPC: + s.EnableChunkRPC = TiDBOptOn(val) case TiDBEnableCascadesPlanner: s.EnableCascadesPlanner = TiDBOptOn(val) case TiDBOptimizerSelectivityLevel: diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 6614ca9f38ee3..d253a9ba19834 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -684,7 +684,7 @@ var defaultSysVars = []*SysVar{ {ScopeSession, TIDBMemQuotaIndexLookupJoin, strconv.FormatInt(DefTiDBMemQuotaIndexLookupJoin, 10)}, {ScopeSession, TIDBMemQuotaNestedLoopApply, strconv.FormatInt(DefTiDBMemQuotaNestedLoopApply, 10)}, {ScopeSession, TiDBEnableStreaming, "0"}, - {ScopeSession, TiDBEnableArrow, "1"}, + {ScopeSession, TiDBEnableChunkRPC, "1"}, {ScopeSession, TxnIsolationOneShot, ""}, {ScopeSession, TiDBEnableTablePartition, "auto"}, {ScopeGlobal | ScopeSession, TiDBHashJoinConcurrency, strconv.Itoa(DefTiDBHashJoinConcurrency)}, diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index f44774bc67e7b..a095485cdd1f2 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -124,8 +124,8 @@ const ( // tidb_enable_streaming enables TiDB to use streaming API for coprocessor requests. TiDBEnableStreaming = "tidb_enable_streaming" - // tidb_enable_arrow enables TiDB to use Chunk format for coprocessor requests. - TiDBEnableArrow = "tidb_enable_arrow" + // tidb_enable_chunk_rpc enables TiDB to use Chunk format for coprocessor requests. + TiDBEnableChunkRPC = "tidb_enable_chunk_rpc" // tidb_optimizer_selectivity_level is used to control the selectivity estimation level. TiDBOptimizerSelectivityLevel = "tidb_optimizer_selectivity_level" diff --git a/sessionctx/variable/varsutil.go b/sessionctx/variable/varsutil.go index 682b2798097fb..6a783feb775a0 100644 --- a/sessionctx/variable/varsutil.go +++ b/sessionctx/variable/varsutil.go @@ -393,7 +393,7 @@ func ValidateSetSystemVar(vars *SessionVars, name string, value string) (string, return value, ErrWrongValueForVar.GenWithStackByArgs(name, value) case TiDBSkipUTF8Check, TiDBOptAggPushDown, TiDBOptInSubqToJoinAndAgg, TiDBEnableFastAnalyze, - TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, TiDBEnableArrow, + TiDBBatchInsert, TiDBDisableTxnAutoRetry, TiDBEnableStreaming, TiDBEnableChunkRPC, TiDBBatchDelete, TiDBBatchCommit, TiDBEnableCascadesPlanner, TiDBEnableWindowFunction, TiDBCheckMb4ValueInUTF8, TiDBLowResolutionTSO, TiDBEnableIndexMerge, TiDBEnableNoopFuncs, TiDBScatterRegion, TiDBGeneralLog, TiDBConstraintCheckInPlace, TiDBEnableVectorizedExpression, TiDBRecordPlanInSlowLog: diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go index 8a675c6f3f76f..e4f3b440a0e2c 100644 --- a/statistics/handle/handle.go +++ b/statistics/handle/handle.go @@ -84,7 +84,7 @@ func (h *Handle) Clear() { h.feedback = h.feedback[:0] h.mu.ctx.GetSessionVars().InitChunkSize = 1 h.mu.ctx.GetSessionVars().MaxChunkSize = 1 - h.mu.ctx.GetSessionVars().EnableArrow = false + h.mu.ctx.GetSessionVars().EnableChunkRPC = false h.mu.ctx.GetSessionVars().ProjectionConcurrency = 0 h.listHead = &SessionStatsCollector{mapper: make(tableDeltaMap), rateMap: make(errorRateDeltaMap)} h.globalMap = make(tableDeltaMap) diff --git a/store/mockstore/mocktikv/cop_handler_dag.go b/store/mockstore/mocktikv/cop_handler_dag.go index 155bac5c346aa..5516af7524c98 100644 --- a/store/mockstore/mocktikv/cop_handler_dag.go +++ b/store/mockstore/mocktikv/cop_handler_dag.go @@ -563,10 +563,10 @@ func (h *rpcHandler) fillUpData4SelectResponse(selResp *tipb.SelectResponse, dag switch dagReq.EncodeType { case tipb.EncodeType_TypeDefault: h.encodeDefault(selResp, rows, dagReq.OutputOffsets) - case tipb.EncodeType_TypeArrow: + case tipb.EncodeType_TypeChunk: colTypes := h.constructRespSchema(dagCtx) loc := dagCtx.evalCtx.sc.TimeZone - err := h.encodeArrow(selResp, rows, colTypes, dagReq.OutputOffsets, loc) + err := h.encodeChunk(selResp, rows, colTypes, dagReq.OutputOffsets, loc) if err != nil { return err } @@ -612,7 +612,7 @@ func (h *rpcHandler) encodeDefault(selResp *tipb.SelectResponse, rows [][][]byte selResp.EncodeType = tipb.EncodeType_TypeDefault } -func (h *rpcHandler) encodeArrow(selResp *tipb.SelectResponse, rows [][][]byte, colTypes []*types.FieldType, colOrdinal []uint32, loc *time.Location) error { +func (h *rpcHandler) encodeChunk(selResp *tipb.SelectResponse, rows [][][]byte, colTypes []*types.FieldType, colOrdinal []uint32, loc *time.Location) error { var chunks []tipb.Chunk respColTypes := make([]*types.FieldType, 0, len(colOrdinal)) for _, ordinal := range colOrdinal { @@ -642,7 +642,7 @@ func (h *rpcHandler) encodeArrow(selResp *tipb.SelectResponse, rows [][][]byte, chk.Reset() } selResp.Chunks = chunks - selResp.EncodeType = tipb.EncodeType_TypeArrow + selResp.EncodeType = tipb.EncodeType_TypeChunk return nil } diff --git a/tidb-server/main.go b/tidb-server/main.go index ef98bd357db00..931e887fff2f6 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -363,7 +363,7 @@ func loadConfig() string { var hotReloadConfigItems = []string{"Performance.MaxProcs", "Performance.MaxMemory", "Performance.CrossJoin", "Performance.FeedbackProbability", "Performance.QueryFeedbackLimit", "Performance.PseudoEstimateRatio", "OOMUseTmpStorage", "OOMAction", "MemQuotaQuery", "StmtSummary.MaxStmtCount", "StmtSummary.MaxSQLLength", "Log.QueryLogMaxLen", - "TiKVClient.EnableArrow"} + "TiKVClient.EnableChunkRPC"} func reloadConfig(nc, c *config.Config) { // Just a part of config items need to be reload explicitly. diff --git a/util/chunk/chunk_test.go b/util/chunk/chunk_test.go index 3a2cb60473a20..315c59a4be18f 100644 --- a/util/chunk/chunk_test.go +++ b/util/chunk/chunk_test.go @@ -100,7 +100,7 @@ func (s *testChunkSuite) TestChunk(c *check.C) { row := chk.GetRow(0) c.Assert(row.GetFloat32(0), check.Equals, f32Val) c.Assert(row.GetTime(2).Compare(tVal), check.Equals, 0) - // fsp no longer maintain in arrow + // fsp is no longer maintained in chunk c.Assert(row.GetDuration(3, 0).Duration, check.DeepEquals, durVal.Duration) c.Assert(row.GetEnum(4), check.DeepEquals, enumVal) c.Assert(row.GetSet(5), check.DeepEquals, setVal)