opt: add optimizer_min_row_count session setting

Informs cockroachdb#64570
Informs cockroachdb#130201

Release note (sql change): The `optimizer_min_row_count` session setting
has been added which sets a lower bound on row count estimates for
relational expressions during query planning. A value of zero, which is
the default, indicates no lower bound. Note that if this is set to a
value greater than zero, a row count of zero can still be estimated for
expressions with a cardinality of zero, e.g., for a contradictory
filter. Setting this to a value higher than 0, such as 1, may yield
better query plans in some cases, such as when statistics are frequently
stale and inaccurate.

pkg/sql/exec_util.go

@@ -3912,6 +3912,10 @@ func (m *sessionDataMutator) SetOptimizerPreferBoundedCardinality(b bool) {
 	m.data.OptimizerPreferBoundedCardinality = b
 }
 
+func (m *sessionDataMutator) SetOptimizerMinRowCount(val float64) {
+	m.data.OptimizerMinRowCount = val
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the

pkg/sql/logictest/testdata/logic_test/information_schema

@@ -6316,6 +6316,7 @@ optimizer                                                  on
 optimizer_always_use_histograms                            on
 optimizer_hoist_uncorrelated_equality_subqueries           on
 optimizer_merge_joins_enabled                              on
+optimizer_min_row_count                                    0
 optimizer_prefer_bounded_cardinality                       off
 optimizer_prove_implication_with_virtual_computed_columns  on
 optimizer_push_limit_into_project_filtered_scan            on

pkg/sql/logictest/testdata/logic_test/pg_catalog

@@ -2985,6 +2985,7 @@ opt_split_scan_limit                                       2048                N
 optimizer_always_use_histograms                            on                  NULL      NULL        NULL        string
 optimizer_hoist_uncorrelated_equality_subqueries           on                  NULL      NULL        NULL        string
 optimizer_merge_joins_enabled                              on                  NULL      NULL        NULL        string
+optimizer_min_row_count                                    0                   NULL      NULL        NULL        string
 optimizer_prefer_bounded_cardinality                       off                 NULL      NULL        NULL        string
 optimizer_prove_implication_with_virtual_computed_columns  on                  NULL      NULL        NULL        string
 optimizer_push_limit_into_project_filtered_scan            on                  NULL      NULL        NULL        string
@@ -3184,6 +3185,7 @@ opt_split_scan_limit                                       2048                N
 optimizer_always_use_histograms                            on                  NULL  user     NULL      on                  on
 optimizer_hoist_uncorrelated_equality_subqueries           on                  NULL  user     NULL      on                  on
 optimizer_merge_joins_enabled                              on                  NULL  user     NULL      on                  on
+optimizer_min_row_count                                    0                   NULL  user     NULL      0                   0
 optimizer_prefer_bounded_cardinality                       off                 NULL  user     NULL      off                 off
 optimizer_prove_implication_with_virtual_computed_columns  on                  NULL  user     NULL      on                  on
 optimizer_push_limit_into_project_filtered_scan            on                  NULL  user     NULL      on                  on
@@ -3382,6 +3384,7 @@ optimizer                                                  NULL    NULL     NULL
 optimizer_always_use_histograms                            NULL    NULL     NULL     NULL        NULL
 optimizer_hoist_uncorrelated_equality_subqueries           NULL    NULL     NULL     NULL        NULL
 optimizer_merge_joins_enabled                              NULL    NULL     NULL     NULL        NULL
+optimizer_min_row_count                                    NULL    NULL     NULL     NULL        NULL
 optimizer_prefer_bounded_cardinality                       NULL    NULL     NULL     NULL        NULL
 optimizer_prove_implication_with_virtual_computed_columns  NULL    NULL     NULL     NULL        NULL
 optimizer_push_limit_into_project_filtered_scan            NULL    NULL     NULL     NULL        NULL

pkg/sql/logictest/testdata/logic_test/show_source

@@ -131,6 +131,7 @@ opt_split_scan_limit                                       2048
 optimizer_always_use_histograms                            on
 optimizer_hoist_uncorrelated_equality_subqueries           on
 optimizer_merge_joins_enabled                              on
+optimizer_min_row_count                                    0
 optimizer_prefer_bounded_cardinality                       off
 optimizer_prove_implication_with_virtual_computed_columns  on
 optimizer_push_limit_into_project_filtered_scan            on

pkg/sql/opt/exec/execbuilder/testdata/explain_redact

@@ -716,7 +716,7 @@ upsert bc
 query T
 EXPLAIN (OPT, MEMO, REDACT) INSERT INTO bc SELECT a::float + 1 FROM a ON CONFLICT (b) DO UPDATE SET b = bc.b + 100
 ----
-memo (optimized, ~35KB, required=[presentation: info:25] [distribution: test])
+memo (optimized, ~37KB, required=[presentation: info:25] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:25] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1156,7 +1156,7 @@ update ab
 query T
 EXPLAIN (OPT, MEMO, REDACT) UPDATE ab SET a = a || 'ab' WHERE a > 'a'
 ----
-memo (optimized, ~14KB, required=[presentation: info:15] [distribution: test])
+memo (optimized, ~15KB, required=[presentation: info:15] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:15] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1382,7 +1382,7 @@ update e
 query T
 EXPLAIN (OPT, MEMO, REDACT) UPDATE e SET e = 'eee' WHERE e > 'a'
 ----
-memo (optimized, ~17KB, required=[presentation: info:17] [distribution: test])
+memo (optimized, ~18KB, required=[presentation: info:17] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:17] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1706,7 +1706,7 @@ project
 query T
 EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM bc WHERE b >= 1.0 AND b < 2.0
 ----
-memo (optimized, ~12KB, required=[presentation: info:7] [distribution: test])
+memo (optimized, ~13KB, required=[presentation: info:7] [distribution: test])
  ├── G1: (explain G2 [presentation: b:1,c:2] [distribution: test])
  │    └── [presentation: info:7] [distribution: test]
  │         ├── best: (explain G2="[presentation: b:1,c:2] [distribution: test]" [presentation: b:1,c:2] [distribution: test])
@@ -2455,7 +2455,7 @@ project
 query T
 EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM bc JOIN f ON b = f + 1
 ----
-memo (optimized, ~27KB, required=[presentation: info:14] [distribution: test])
+memo (optimized, ~29KB, required=[presentation: info:14] [distribution: test])
  ├── G1: (explain G2 [presentation: b:1,c:2,f:7] [distribution: test])
  │    └── [presentation: info:14] [distribution: test]
  │         ├── best: (explain G2="[presentation: b:1,c:2,f:7] [distribution: test]" [presentation: b:1,c:2,f:7] [distribution: test])

pkg/sql/opt/memo/memo.go

@@ -200,6 +200,7 @@ type Memo struct {
 	unsafeAllowTriggersModifyingCascades       bool
 	legacyVarcharTyping                        bool
 	preferBoundedCardinality                   bool
+	minRowCount                                float64
 
 	// txnIsoLevel is the isolation level under which the plan was created. This
 	// affects the planning of some locking operations, so it must be included in
@@ -291,6 +292,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		unsafeAllowTriggersModifyingCascades:       evalCtx.SessionData().UnsafeAllowTriggersModifyingCascades,
 		legacyVarcharTyping:                        evalCtx.SessionData().LegacyVarcharTyping,
 		preferBoundedCardinality:                   evalCtx.SessionData().OptimizerPreferBoundedCardinality,
+		minRowCount:                                evalCtx.SessionData().OptimizerMinRowCount,
 		txnIsoLevel:                                evalCtx.TxnIsoLevel,
 	}
 	m.metadata.Init()
@@ -460,6 +462,7 @@ func (m *Memo) IsStale(
 		m.unsafeAllowTriggersModifyingCascades != evalCtx.SessionData().UnsafeAllowTriggersModifyingCascades ||
 		m.legacyVarcharTyping != evalCtx.SessionData().LegacyVarcharTyping ||
 		m.preferBoundedCardinality != evalCtx.SessionData().OptimizerPreferBoundedCardinality ||
+		m.minRowCount != evalCtx.SessionData().OptimizerMinRowCount ||
 		m.txnIsoLevel != evalCtx.TxnIsoLevel {
 		return true, nil
 	}

pkg/sql/opt/memo/memo_test.go

@@ -537,6 +537,11 @@ func TestMemoIsStale(t *testing.T) {
 	evalCtx.SessionData().OptimizerPreferBoundedCardinality = false
 	notStale()
 
+	evalCtx.SessionData().OptimizerMinRowCount = 1.0
+	stale()
+	evalCtx.SessionData().OptimizerMinRowCount = 0
+	notStale()
+
 	// User no longer has access to view.
 	catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
 	_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)

pkg/sql/opt/memo/statistics_builder.go

@@ -228,18 +228,20 @@ const (
 //
 // See props/statistics.go for more details.
 type statisticsBuilder struct {
-	ctx     context.Context
-	evalCtx *eval.Context
-	md      *opt.Metadata
+	ctx         context.Context
+	evalCtx     *eval.Context
+	md          *opt.Metadata
+	minRowCount float64
 }
 
 func (sb *statisticsBuilder) init(ctx context.Context, evalCtx *eval.Context, md *opt.Metadata) {
 	// This initialization pattern ensures that fields are not unwittingly
 	// reused. Field reuse must be explicit.
 	*sb = statisticsBuilder{
-		ctx:     ctx,
-		evalCtx: evalCtx,
-		md:      md,
+		ctx:         ctx,
+		evalCtx:     evalCtx,
+		md:          md,
+		minRowCount: evalCtx.SessionData().OptimizerMinRowCount,
 	}
 }
 
@@ -830,7 +832,7 @@ func (sb *statisticsBuilder) colAvgSize(tabID opt.TableID, col opt.ColumnID) uin
 
 func (sb *statisticsBuilder) buildScan(scan *ScanExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1079,7 +1081,7 @@ func (sb *statisticsBuilder) colStatScan(colSet opt.ColSet, scan *ScanExpr) *pro
 
 func (sb *statisticsBuilder) buildSelect(sel *SelectExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1120,7 +1122,7 @@ func (sb *statisticsBuilder) colStatSelect(
 
 func (sb *statisticsBuilder) buildProject(prj *ProjectExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1222,7 +1224,7 @@ func (sb *statisticsBuilder) buildInvertedFilter(
 	invFilter *InvertedFilterExpr, relProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1283,7 +1285,7 @@ func (sb *statisticsBuilder) buildJoin(
 	}
 
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1814,7 +1816,7 @@ func (sb *statisticsBuilder) colStatFromJoinRight(
 
 func (sb *statisticsBuilder) buildIndexJoin(indexJoin *IndexJoinExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1895,7 +1897,7 @@ func (sb *statisticsBuilder) buildZigzagJoin(
 	zigzag *ZigzagJoinExpr, relProps *props.Relational, h *joinPropsHelper,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1996,7 +1998,7 @@ func (sb *statisticsBuilder) buildZigzagJoin(
 
 func (sb *statisticsBuilder) buildGroupBy(groupNode RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2108,7 +2110,7 @@ func (sb *statisticsBuilder) colStatGroupBy(
 
 func (sb *statisticsBuilder) buildSetNode(setNode RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2213,7 +2215,7 @@ func (sb *statisticsBuilder) colStatSetNodeImpl(
 // buildValues builds the statistics for a VALUES expression.
 func (sb *statisticsBuilder) buildValues(values ValuesContainer, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2318,7 +2320,7 @@ func (sb *statisticsBuilder) colStatLiteralValues(
 
 func (sb *statisticsBuilder) buildLimit(limit *LimitExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2361,7 +2363,7 @@ func (sb *statisticsBuilder) colStatLimit(
 
 func (sb *statisticsBuilder) buildTopK(topK *TopKExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2403,7 +2405,7 @@ func (sb *statisticsBuilder) colStatTopK(colSet opt.ColSet, topK *TopKExpr) *pro
 
 func (sb *statisticsBuilder) buildOffset(offset *OffsetExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2456,7 +2458,7 @@ func (sb *statisticsBuilder) colStatOffset(
 
 func (sb *statisticsBuilder) buildMax1Row(max1Row *Max1RowExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2486,7 +2488,7 @@ func (sb *statisticsBuilder) colStatMax1Row(
 
 func (sb *statisticsBuilder) buildOrdinality(ord *OrdinalityExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2531,7 +2533,7 @@ func (sb *statisticsBuilder) colStatOrdinality(
 
 func (sb *statisticsBuilder) buildWindow(window *WindowExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2599,7 +2601,7 @@ func (sb *statisticsBuilder) buildProjectSet(
 	projectSet *ProjectSetExpr, relProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2732,7 +2734,7 @@ func (sb *statisticsBuilder) buildWithScan(
 	withScan *WithScanExpr, relProps, bindingProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2774,7 +2776,7 @@ func (sb *statisticsBuilder) colStatWithScan(
 
 func (sb *statisticsBuilder) buildMutation(mutation RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2812,7 +2814,7 @@ func (sb *statisticsBuilder) colStatMutation(
 
 func (sb *statisticsBuilder) buildLock(lock *LockExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2844,7 +2846,7 @@ func (sb *statisticsBuilder) colStatLock(colSet opt.ColSet, lock *LockExpr) *pro
 
 func (sb *statisticsBuilder) buildBarrier(barrier *BarrierExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2878,7 +2880,7 @@ func (sb *statisticsBuilder) colStatBarrier(
 
 func (sb *statisticsBuilder) buildCall(call *CallExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short-cut if cardinality is 0.
 		return
 	}

pkg/sql/opt/memo/statistics_builder_test.go

@@ -112,7 +112,8 @@ func TestGetStatsFromConstraint(t *testing.T) {
 		relProps := &props.Relational{Cardinality: props.AnyCardinality}
 		relProps.NotNullCols = cs.ExtractNotNullCols(ctx, &evalCtx)
 		s := relProps.Statistics()
-		s.Init(relProps)
+		const minRowCount = 0
+		s.Init(relProps, minRowCount)
 
 		// Calculate distinct counts.
 		sb.applyConstraintSet(cs, true /* tight */, sel, relProps, relProps.Statistics())

pkg/sql/opt/memo/testdata/memo

@@ -317,7 +317,7 @@ memo (optimized, ~6KB, required=[presentation: array_agg:6])
 memo
 SELECT array_agg(x) FROM (SELECT * FROM a) GROUP BY y
 ----
-memo (optimized, ~7KB, required=[presentation: array_agg:6])
+memo (optimized, ~8KB, required=[presentation: array_agg:6])
  ├── G1: (project G2 G3 array_agg)
  │    └── [presentation: array_agg:6]
  │         ├── best: (project G2 G3 array_agg)
@@ -373,7 +373,7 @@ memo (optimized, ~6KB, required=[presentation: array_cat_agg:6])
 memo
 SELECT array_cat_agg(arr) FROM (SELECT * FROM a) GROUP BY y
 ----
-memo (optimized, ~7KB, required=[presentation: array_cat_agg:6])
+memo (optimized, ~8KB, required=[presentation: array_cat_agg:6])
  ├── G1: (project G2 G3 array_cat_agg)
  │    └── [presentation: array_cat_agg:6]
  │         ├── best: (project G2 G3 array_cat_agg)