plan: calculate table access paths in DataSource.deriveStats (#6346)

plan/build_key_info.go

@@ -174,8 +174,11 @@ func (p *LogicalJoin) buildKeyInfo() {
 func (ds *DataSource) buildKeyInfo() {
 	ds.schema.Keys = nil
 	ds.baseLogicalPlan.buildKeyInfo()
-	indices := ds.availableIndices.indices
-	for _, idx := range indices {
+	for _, path := range ds.possibleAccessPaths {
+		if path.isTablePath {
+			continue
+		}
+		idx := path.index
 		if !idx.Unique {
 			continue
 		}

plan/common_plans.go

@@ -255,7 +255,7 @@ func (e *Execute) buildRangeForIndexScan(sctx sessionctx.Context, is *PhysicalIn
 	ranges := ranger.FullNewRange()
 	if len(idxCols) > 0 {
 		var err error
-		ranges, _, _, _, err = ranger.DetachCondAndBuildRangeForIndex(sctx, is.conditions, idxCols, colLengths)
+		ranges, _, _, _, err = ranger.DetachCondAndBuildRangeForIndex(sctx, is.AccessCondition, idxCols, colLengths)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}

plan/gen_physical_plans.go

@@ -251,9 +251,14 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *requiredProp, outerIdx int) [
 	if !ok {
 		return nil
 	}
-	indices := x.availableIndices.indices
-	includeTableScan := x.availableIndices.includeTableScan
-	if includeTableScan && len(innerJoinKeys) == 1 {
+	var tblPath *accessPath
+	for _, path := range x.possibleAccessPaths {
+		if path.isTablePath {
+			tblPath = path
+			break
+		}
+	}
+	if tblPath != nil && len(innerJoinKeys) == 1 {
 		pkCol := x.getPKIsHandleCol()
 		if pkCol != nil && innerJoinKeys[0].Equal(nil, pkCol) {
 			innerPlan := x.forceToTableScan(pkCol)
@@ -267,7 +272,11 @@ func (p *LogicalJoin) getIndexJoinByOuterIdx(prop *requiredProp, outerIdx int) [
 		remainedOfBest []expression.Expression
 		keyOff2IdxOff  []int
 	)
-	for _, indexInfo := range indices {
+	for _, path := range x.possibleAccessPaths {
+		if path.isTablePath {
+			continue
+		}
+		indexInfo := path.index
 		ranges, remained, tmpKeyOff2IdxOff := p.buildRangeForIndexJoin(indexInfo, x, innerJoinKeys)
 		// We choose the index by the number of used columns of the range, the much the better.
 		// Notice that there may be the cases like `t1.a=t2.a and b > 2 and b < 1`. So ranges can be nil though the conditions are valid.

plan/logical_plan_builder.go

@@ -1721,7 +1721,7 @@ func (b *planBuilder) buildDataSource(tn *ast.TableName) LogicalPlan {
 	tableInfo := tbl.Meta()
 	b.visitInfo = appendVisitInfo(b.visitInfo, mysql.SelectPriv, dbName.L, tableInfo.Name.L, "")
 
-	availableIdxes, err := getAvailableIndices(tn.IndexHints, tableInfo)
+	possiblePaths, err := getPossibleAccessPaths(tn.IndexHints, tableInfo)
 	if err != nil {
 		b.err = errors.Trace(err)
 		return nil
@@ -1735,12 +1735,12 @@ func (b *planBuilder) buildDataSource(tn *ast.TableName) LogicalPlan {
 	}
 
 	ds := DataSource{
-		DBName:           dbName,
-		tableInfo:        tableInfo,
-		statisticTable:   b.getStatsTable(tableInfo),
-		indexHints:       tn.IndexHints,
-		availableIndices: availableIdxes,
-		Columns:          make([]*model.ColumnInfo, 0, len(columns)),
+		DBName:              dbName,
+		tableInfo:           tableInfo,
+		statisticTable:      b.getStatsTable(tableInfo),
+		indexHints:          tn.IndexHints,
+		possibleAccessPaths: possiblePaths,
+		Columns:             make([]*model.ColumnInfo, 0, len(columns)),
 	}.init(b.ctx)
 
 	var handleCol *expression.Column

plan/logical_plans.go

@@ -14,13 +14,18 @@
 package plan
 
 import (
+	"math"
+
+	"github.com/juju/errors"
 	"github.com/pingcap/tidb/ast"
 	"github.com/pingcap/tidb/expression"
 	"github.com/pingcap/tidb/expression/aggregation"
 	"github.com/pingcap/tidb/model"
 	"github.com/pingcap/tidb/mysql"
 	"github.com/pingcap/tidb/statistics"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/ranger"
+	log "github.com/sirupsen/logrus"
 )
 
 var (
@@ -299,13 +304,85 @@ type DataSource struct {
 
 	statisticTable *statistics.Table
 
-	// availableIndices is used for storing result of availableIndices function.
-	availableIndices *availableIndices
+	// possibleAccessPaths stores all the possible access path for physical plan, including table scan.
+	possibleAccessPaths []*accessPath
+}
+
+// accessPath tells how we access one index or just access table.
+type accessPath struct {
+	index  *model.IndexInfo
+	ranges []*ranger.NewRange
+	// countAfterAccess is the row count after we apply range seek and before we use other filter to filter data.
+	countAfterAccess float64
+	// countAfterIndex is the row count after we apply filters on index and before we apply the table filters.
+	countAfterIndex float64
+	accessConds     []expression.Expression
+	eqCondCount     int
+	indexFilters    []expression.Expression
+	tableFilters    []expression.Expression
+	// isTablePath indicates whether this path is table path.
+	isTablePath bool
+	// forced means this path is generated by `use/force index()`.
+	forced bool
+}
+
+func (ds *DataSource) deriveTablePathStats(path *accessPath) error {
+	var err error
+	sc := ds.ctx.GetSessionVars().StmtCtx
+	path.countAfterAccess = float64(ds.statisticTable.Count)
+	path.tableFilters = ds.pushedDownConds
+	var pkCol *expression.Column
+	if ds.tableInfo.PKIsHandle {
+		if pkColInfo := ds.tableInfo.GetPkColInfo(); pkColInfo != nil {
+			pkCol = expression.ColInfo2Col(ds.schema.Columns, pkColInfo)
+		}
+	}
+	if pkCol == nil {
+		path.ranges = ranger.FullIntNewRange(false)
+		return nil
+	}
+	path.ranges = ranger.FullIntNewRange(mysql.HasUnsignedFlag(pkCol.RetType.Flag))
+	if len(ds.pushedDownConds) == 0 {
+		return nil
+	}
+	path.accessConds, path.tableFilters = ranger.DetachCondsForTableRange(ds.ctx, ds.pushedDownConds, pkCol)
+	path.ranges, err = ranger.BuildTableRange(path.accessConds, sc, pkCol.RetType)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	path.countAfterAccess, err = ds.statisticTable.GetRowCountByIntColumnRanges(sc, pkCol.ID, path.ranges)
+	return errors.Trace(err)
 }
 
-type availableIndices struct {
-	indices          []*model.IndexInfo
-	includeTableScan bool
+func (ds *DataSource) deriveIndexPathStats(path *accessPath) error {
+	var err error
+	sc := ds.ctx.GetSessionVars().StmtCtx
+	path.ranges = ranger.FullNewRange()
+	path.countAfterAccess = float64(ds.statisticTable.Count)
+	idxCols, lengths := expression.IndexInfo2Cols(ds.schema.Columns, path.index)
+	if len(idxCols) != 0 {
+		path.ranges, path.accessConds, path.indexFilters, path.eqCondCount, err = ranger.DetachCondAndBuildRangeForIndex(ds.ctx, ds.pushedDownConds, idxCols, lengths)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		path.countAfterAccess, err = ds.statisticTable.GetRowCountByIndexRanges(sc, path.index.ID, path.ranges)
+		if err != nil {
+			return errors.Trace(err)
+		}
+		path.indexFilters, path.tableFilters = splitIndexFilterConditions(path.indexFilters, path.index.Columns, ds.tableInfo)
+	} else {
+		path.indexFilters, path.tableFilters = splitIndexFilterConditions(ds.pushedDownConds, path.index.Columns, ds.tableInfo)
+	}
+	path.countAfterIndex = path.countAfterAccess
+	if path.indexFilters != nil {
+		selectivity, err := ds.statisticTable.Selectivity(ds.ctx, path.indexFilters)
+		if err != nil {
+			log.Warnf("An error happened: %v, we have to use the default selectivity", err.Error())
+			selectivity = selectionFactor
+		}
+		path.countAfterIndex = math.Max(path.countAfterAccess*selectivity, ds.statsAfterSelect.count)
+	}
+	return nil
 }
 
 func (ds *DataSource) getPKIsHandleCol() *expression.Column {

plan/optimizer.go

@@ -145,7 +145,10 @@ func logicalOptimize(flag uint64, logic LogicalPlan) (LogicalPlan, error) {
 
 func physicalOptimize(logic LogicalPlan) (PhysicalPlan, error) {
 	logic.preparePossibleProperties()
-	logic.deriveStats()
+	_, err := logic.deriveStats()
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
 	t, err := logic.findBestTask(&requiredProp{taskTp: rootTaskType, expectedCnt: math.MaxFloat64})
 	if err != nil {
 		return nil, errors.Trace(err)