Merge remote-tracking branch 'upstream/master' into security-enhanced…

…-mode-1

README.md

@@ -10,12 +10,6 @@
 [![Coverage Status](https://codecov.io/gh/pingcap/tidb/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/tidb)
 [![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://godoc.org/github.com/pingcap/tidb)
 
-- [**Slack Channel**](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb)
-- **Twitter**: [@PingCAP](https://twitter.com/PingCAP)
-- [**Reddit**](https://www.reddit.com/r/TiDB/)
-- **Mailing list**: [lists.tidb.io](https://lists.tidb.io/g/main/subgroups)
-- [**For support, please contact PingCAP**](http://bit.ly/contact_us_via_github)
-
 ## What is TiDB?
 
 TiDB ("Ti" stands for Titanium) is an open-source NewSQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability.
@@ -48,12 +42,59 @@ TiDB ("Ti" stands for Titanium) is an open-source NewSQL database that supports
 
 For more details and latest updates, see [TiDB docs](https://docs.pingcap.com/tidb/stable) and [release notes](https://docs.pingcap.com/tidb/dev/release-notes).
 
+## Community
+
+You can join these groups and chats to discuss and ask TiDB related questions:
+
+- [Contributors Mailing list](https://lists.tidb.io/g/contributors)
+- [Slack Channel](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-tidb)
+- [Chinese Forum](https://asktug.com)
+
+In addition, you may enjoy following:
+
+- [@PingCAP](https://twitter.com/PingCAP) on Twitter
+- Question tagged [#tidb on StackOverflow](https://stackoverflow.com/questions/tagged/tidb)
+- The PingCAP Team [English Blog](https://en.pingcap.com/blog) and [Chinese Blog](https://pingcap.com/blog-cn/)
+- [TiDB Monthly](https://pingcap.com/weekly/)
+
+For support, please contact [PingCAP](http://bit.ly/contact_us_via_github).
+
 ## Quick start
 
-See [Quick Start Guide](https://pingcap.com/docs/stable/quick-start-with-tidb/ ), which includes deployment methods using TiUP, Ansible, Docker, and Kubernetes.
+### To start using TiDB
+
+See [Quick Start Guide](https://pingcap.com/docs/stable/quick-start-with-tidb/).
 
 ### To start developing TiDB
 
+If you want to build TiDB right away, there are two options:
+
+**You have a working [Go environment](https://golang.org/doc/install).**
+
+```
+mkdir -p $GOPATH/src/github.com/pingcap
+cd $GOPATH/src/github.com/pingcap
+git clone https://github.com/pingcap/tidb.git
+cd tidb
+make
+cd bin && ./tidb-server
+```
+
+**You have a working [Docker environment](https://docs.docker.com/engine/).**
+
+```
+docker pull pingcap/tidb:latest
+docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest
+```
+
+Now you can use official mysql client to connect to TiDB.
+
+```
+mysql -h 127.0.0.1 -P 4000 -u root -D test --prompt="tidb> " 
+```
+
+## Contributing
+
 The [community repository](https://github.com/pingcap/community) hosts all information about the TiDB community, including how to contribute to TiDB, how TiDB community is governed, how special interest groups are organized, etc.
 
 [<img src="docs/contribution-map.png" alt="contribution-map" width="180">](https://github.com/pingcap/tidb-map/blob/master/maps/contribution-map.md#tidb-is-an-open-source-distributed-htap-database-compatible-with-the-mysql-protocol)
@@ -73,25 +114,6 @@ View the current list of in-production TiDB adopters [here](https://docs.pingcap
 
 Read the [Roadmap](https://pingcap.com/docs/ROADMAP).
 
-## Getting help
-
-- [**Stack Overflow**](https://stackoverflow.com/questions/tagged/tidb)
-- [**User Group (Chinese)**](https://asktug.com)
-
-## Documentation
-
-+ [English](https://pingcap.com/docs)
-+ [简体中文](https://pingcap.com/docs-cn)
-
-## Blog
-
-- [English](https://www.pingcap.com/blog/)
-- [简体中文](https://pingcap.com/blog-cn/)
-
-## TiDB Monthly
-
-[TiDB Monthly](https://pingcap.com/weekly/)
-
 ## Architecture
 
 ![architecture](./docs/architecture.png)

cmd/explaintest/r/explain_easy.result

@@ -63,7 +63,7 @@ Delete	N/A	root		N/A
   └─TableRowIDScan(Probe)	10.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 explain format = 'brief' select count(b.c2) from t1 a, t2 b where a.c1 = b.c2 group by a.c1;
 id	estRows	task	access object	operator info
-Projection	9990.00	root		cast(Column#8, bigint(21) BINARY)->Column#7
+Projection	9990.00	root		Column#7
 └─HashJoin	9990.00	root		inner join, equal:[eq(test.t1.c1, test.t2.c2)]
   ├─HashAgg(Build)	7992.00	root		group by:test.t2.c2, funcs:count(Column#9)->Column#8, funcs:firstrow(test.t2.c2)->test.t2.c2
   │ └─TableReader	7992.00	root		data:HashAgg

cmd/explaintest/r/explain_easy_stats.result

@@ -64,7 +64,7 @@ Delete	N/A	root		N/A
   └─TableRowIDScan(Probe)	0.00	cop[tikv]	table:t1	keep order:false
 explain format = 'brief' select count(b.c2) from t1 a, t2 b where a.c1 = b.c2 group by a.c1;
 id	estRows	task	access object	operator info
-Projection	1985.00	root		cast(Column#8, bigint(21) BINARY)->Column#7
+Projection	1985.00	root		Column#7
 └─HashJoin	1985.00	root		inner join, equal:[eq(test.t1.c1, test.t2.c2)]
   ├─HashAgg(Build)	1985.00	root		group by:test.t2.c2, funcs:count(test.t2.c2)->Column#8, funcs:firstrow(test.t2.c2)->test.t2.c2
   │ └─TableReader	1985.00	root		data:Selection

ddl/backfilling.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"sync/atomic"
 	"time"
+	"unsafe"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"
@@ -331,15 +332,17 @@ func splitTableRanges(t table.PhysicalTable, store kv.Storage, startKey, endKey
 
 	maxSleep := 10000 // ms
 	bo := tikv.NewBackofferWithVars(context.Background(), maxSleep, nil)
-	ranges, err := tikv.SplitRegionRanges(bo, s.GetRegionCache(), []kv.KeyRange{kvRange})
+	tikvRange := *(*tikvstore.KeyRange)(unsafe.Pointer(&kvRange))
+	ranges, err := tikv.SplitRegionRanges(bo, s.GetRegionCache(), []tikvstore.KeyRange{tikvRange})
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
 	if len(ranges) == 0 {
 		errMsg := fmt.Sprintf("cannot find region in range [%s, %s]", startKey.String(), endKey.String())
 		return nil, errors.Trace(errInvalidSplitRegionRanges.GenWithStackByArgs(errMsg))
 	}
-	return ranges, nil
+	res := *(*[]kv.KeyRange)(unsafe.Pointer(&ranges))
+	return res, nil
 }
 
 func (w *worker) waitTaskResults(workers []*backfillWorker, taskCnt int,

ddl/table_split_test.go

@@ -82,5 +82,5 @@ func checkRegionStartWithTableID(c *C, id int64, store kvStore) {
 	c.Assert(err, IsNil)
 	// Region cache may be out of date, so we need to drop this expired region and load it again.
 	cache.InvalidateCachedRegion(loc.Region)
-	c.Assert([]byte(loc.StartKey), BytesEquals, []byte(regionStartKey))
+	c.Assert(loc.StartKey, BytesEquals, []byte(regionStartKey))
 }

executor/adapter.go

@@ -788,14 +788,14 @@ func (a *ExecStmt) logAudit() {
 }
 
 // FormatSQL is used to format the original SQL, e.g. truncating long SQL, appending prepared arguments.
-func FormatSQL(sql string, pps variable.PreparedParams) stringutil.StringerFunc {
+func FormatSQL(sql string) stringutil.StringerFunc {
 	return func() string {
 		cfg := config.GetGlobalConfig()
 		length := len(sql)
 		if maxQueryLen := atomic.LoadUint64(&cfg.Log.QueryLogMaxLen); uint64(length) > maxQueryLen {
 			sql = fmt.Sprintf("%.*q(len:%d)", maxQueryLen, sql, length)
 		}
-		return QueryReplacer.Replace(sql) + pps.String()
+		return QueryReplacer.Replace(sql)
 	}
 }
 
@@ -840,13 +840,7 @@ func (a *ExecStmt) FinishExecuteStmt(txnTS uint64, succ bool, hasMoreResults boo
 			totalTiFlashQueryFailCounter.Inc()
 		}
 	}
-	prevStmt := a.GetTextToLog()
-	if sessVars.EnableRedactLog {
-		sessVars.PrevStmt = FormatSQL(prevStmt, nil)
-	} else {
-		pps := types.CloneRow(sessVars.PreparedParams)
-		sessVars.PrevStmt = FormatSQL(prevStmt, pps)
-	}
+	sessVars.PrevStmt = FormatSQL(a.GetTextToLog())
 
 	executeDuration := time.Since(sessVars.StartTime) - sessVars.DurationCompile
 	if sessVars.InRestrictedSQL {
@@ -884,15 +878,8 @@ func (a *ExecStmt) LogSlowQuery(txnTS uint64, succ bool, hasMoreResults bool) {
 	if (!enable || costTime < threshold) && !force {
 		return
 	}
-	var sql stringutil.StringerFunc
-	normalizedSQL, digest := sessVars.StmtCtx.SQLDigest()
-	if sessVars.EnableRedactLog {
-		sql = FormatSQL(normalizedSQL, nil)
-	} else if sensitiveStmt, ok := a.StmtNode.(ast.SensitiveStmtNode); ok {
-		sql = FormatSQL(sensitiveStmt.SecureText(), nil)
-	} else {
-		sql = FormatSQL(a.Text, sessVars.PreparedParams)
-	}
+	sql := FormatSQL(a.GetTextToLog())
+	_, digest := sessVars.StmtCtx.SQLDigest()
 
 	var indexNames string
 	if len(sessVars.StmtCtx.IndexNames) > 0 {
@@ -1153,12 +1140,13 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
 // GetTextToLog return the query text to log.
 func (a *ExecStmt) GetTextToLog() string {
 	var sql string
-	if a.Ctx.GetSessionVars().EnableRedactLog {
-		sql, _ = a.Ctx.GetSessionVars().StmtCtx.SQLDigest()
+	sessVars := a.Ctx.GetSessionVars()
+	if sessVars.EnableRedactLog {
+		sql, _ = sessVars.StmtCtx.SQLDigest()
 	} else if sensitiveStmt, ok := a.StmtNode.(ast.SensitiveStmtNode); ok {
 		sql = sensitiveStmt.SecureText()
 	} else {
-		sql = a.Text
+		sql = sessVars.StmtCtx.OriginalSQL + sessVars.PreparedParams.String()
 	}
 	return sql
 }

executor/aggfuncs/aggfuncs.go

@@ -203,3 +203,9 @@ type SlidingWindowAggFunc interface {
 	// completely.
 	Slide(sctx sessionctx.Context, rows []chunk.Row, lastStart, lastEnd uint64, shiftStart, shiftEnd uint64, pr PartialResult) error
 }
+
+// MaxMinSlidingWindowAggFunc is the interface to evaluate the max/min agg function using sliding window
+type MaxMinSlidingWindowAggFunc interface {
+	// SetWindowStart sets the start position of window
+	SetWindowStart(start uint64)
+}

executor/aggfuncs/builder.go

@@ -446,21 +446,21 @@ func buildMaxMinInWindowFunction(aggFuncDesc *aggregation.AggFuncDesc, ordinal i
 	// build max/min aggFunc for window function using sliding window
 	switch baseAggFunc := base.(type) {
 	case *maxMin4Int:
-		return &maxMin4IntSliding{*baseAggFunc}
+		return &maxMin4IntSliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Uint:
-		return &maxMin4UintSliding{*baseAggFunc}
+		return &maxMin4UintSliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Float32:
-		return &maxMin4Float32Sliding{*baseAggFunc}
+		return &maxMin4Float32Sliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Float64:
-		return &maxMin4Float64Sliding{*baseAggFunc}
+		return &maxMin4Float64Sliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Decimal:
-		return &maxMin4DecimalSliding{*baseAggFunc}
+		return &maxMin4DecimalSliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4String:
-		return &maxMin4StringSliding{*baseAggFunc}
+		return &maxMin4StringSliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Time:
-		return &maxMin4TimeSliding{*baseAggFunc}
+		return &maxMin4TimeSliding{*baseAggFunc, windowInfo{}}
 	case *maxMin4Duration:
-		return &maxMin4DurationSliding{*baseAggFunc}
+		return &maxMin4DurationSliding{*baseAggFunc, windowInfo{}}
 	}
 	return base
 }