opt: add optimizer_min_row_count session setting

Informs #64570
Informs #130201

Release note (sql change): The `optimizer_min_row_count` session setting
has been added which sets a lower bound on row count estimates for
relational expressions during query planning. A value of zero, which is
the default, indicates no lower bound. Note that if this is set to a
value greater than zero, a row count of zero can still be estimated for
expressions with a cardinality of zero, e.g., for a contradictory
filter. Setting this to a value higher than 0, such as 1, may yield
better query plans in some cases, such as when statistics are frequently
stale and inaccurate.

pkg/sql/exec_util.go

@@ -3855,6 +3855,10 @@ func (m *sessionDataMutator) SetOptimizerPreferBoundedCardinality(b bool) {
 	m.data.OptimizerPreferBoundedCardinality = b
 }
 
+func (m *sessionDataMutator) SetOptimizerMinRowCount(val float64) {
+	m.data.OptimizerMinRowCount = val
+}
+
 // Utility functions related to scrubbing sensitive information on SQL Stats.
 
 // quantizeCounts ensures that the Count field in the

pkg/sql/logictest/testdata/logic_test/information_schema

@@ -6172,6 +6172,7 @@ optimizer                                                  on
 optimizer_always_use_histograms                            on
 optimizer_hoist_uncorrelated_equality_subqueries           on
 optimizer_merge_joins_enabled                              on
+optimizer_min_row_count                                    0
 optimizer_prefer_bounded_cardinality                       off
 optimizer_prove_implication_with_virtual_computed_columns  on
 optimizer_push_limit_into_project_filtered_scan            off

pkg/sql/logictest/testdata/logic_test/pg_catalog

@@ -2947,6 +2947,7 @@ opt_split_scan_limit                                       2048                N
 optimizer_always_use_histograms                            on                  NULL      NULL        NULL        string
 optimizer_hoist_uncorrelated_equality_subqueries           on                  NULL      NULL        NULL        string
 optimizer_merge_joins_enabled                              on                  NULL      NULL        NULL        string
+optimizer_min_row_count                                    0                   NULL      NULL        NULL        string
 optimizer_prefer_bounded_cardinality                       off                 NULL      NULL        NULL        string
 optimizer_prove_implication_with_virtual_computed_columns  on                  NULL      NULL        NULL        string
 optimizer_push_limit_into_project_filtered_scan            off                 NULL      NULL        NULL        string
@@ -3139,6 +3140,7 @@ opt_split_scan_limit                                       2048                N
 optimizer_always_use_histograms                            on                  NULL  user     NULL      on                  on
 optimizer_hoist_uncorrelated_equality_subqueries           on                  NULL  user     NULL      on                  on
 optimizer_merge_joins_enabled                              on                  NULL  user     NULL      on                  on
+optimizer_min_row_count                                    0                   NULL  user     NULL      0                   0
 optimizer_prefer_bounded_cardinality                       off                 NULL  user     NULL      off                 off
 optimizer_prove_implication_with_virtual_computed_columns  on                  NULL  user     NULL      on                  on
 optimizer_push_limit_into_project_filtered_scan            off                 NULL  user     NULL      off                 off
@@ -3330,6 +3332,7 @@ optimizer                                                  NULL    NULL     NULL
 optimizer_always_use_histograms                            NULL    NULL     NULL     NULL        NULL
 optimizer_hoist_uncorrelated_equality_subqueries           NULL    NULL     NULL     NULL        NULL
 optimizer_merge_joins_enabled                              NULL    NULL     NULL     NULL        NULL
+optimizer_min_row_count                                    NULL    NULL     NULL     NULL        NULL
 optimizer_prefer_bounded_cardinality                       NULL    NULL     NULL     NULL        NULL
 optimizer_prove_implication_with_virtual_computed_columns  NULL    NULL     NULL     NULL        NULL
 optimizer_push_limit_into_project_filtered_scan            NULL    NULL     NULL     NULL        NULL

pkg/sql/logictest/testdata/logic_test/show_source

@@ -128,6 +128,7 @@ opt_split_scan_limit                                       2048
 optimizer_always_use_histograms                            on
 optimizer_hoist_uncorrelated_equality_subqueries           on
 optimizer_merge_joins_enabled                              on
+optimizer_min_row_count                                    0
 optimizer_prefer_bounded_cardinality                       off
 optimizer_prove_implication_with_virtual_computed_columns  on
 optimizer_push_limit_into_project_filtered_scan            off

pkg/sql/opt/exec/execbuilder/testdata/explain_redact

@@ -700,7 +700,7 @@ upsert bc
 query T
 EXPLAIN (OPT, MEMO, REDACT) INSERT INTO bc SELECT a::float + 1 FROM a ON CONFLICT (b) DO UPDATE SET b = bc.b + 100
 ----
-memo (optimized, ~35KB, required=[presentation: info:25] [distribution: test])
+memo (optimized, ~37KB, required=[presentation: info:25] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:19] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1140,7 +1140,7 @@ update ab
 query T
 EXPLAIN (OPT, MEMO, REDACT) UPDATE ab SET a = a || 'ab' WHERE a > 'a'
 ----
-memo (optimized, ~14KB, required=[presentation: info:15] [distribution: test])
+memo (optimized, ~15KB, required=[presentation: info:15] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:11] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1366,7 +1366,7 @@ update e
 query T
 EXPLAIN (OPT, MEMO, REDACT) UPDATE e SET e = 'eee' WHERE e > 'a'
 ----
-memo (optimized, ~17KB, required=[presentation: info:17] [distribution: test])
+memo (optimized, ~18KB, required=[presentation: info:17] [distribution: test])
  ├── G1: (explain G2 [distribution: test])
  │    └── [presentation: info:13] [distribution: test]
  │         ├── best: (explain G2="[distribution: test]" [distribution: test])
@@ -1690,7 +1690,7 @@ project
 query T
 EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM bc WHERE b >= 1.0 AND b < 2.0
 ----
-memo (optimized, ~12KB, required=[presentation: info:5] [distribution: test])
+memo (optimized, ~13KB, required=[presentation: info:7] [distribution: test])
  ├── G1: (explain G2 [presentation: b:1,c:2] [distribution: test])
  │    └── [presentation: info:5] [distribution: test]
  │         ├── best: (explain G2="[presentation: b:1,c:2] [distribution: test]" [presentation: b:1,c:2] [distribution: test])
@@ -2439,10 +2439,10 @@ project
 query T
 EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM bc JOIN f ON b = f + 1
 ----
-memo (optimized, ~27KB, required=[presentation: info:10] [distribution: test])
- ├── G1: (explain G2 [presentation: b:1,c:2,f:5] [distribution: test])
- │    └── [presentation: info:10] [distribution: test]
- │         ├── best: (explain G2="[presentation: b:1,c:2,f:5] [distribution: test]" [presentation: b:1,c:2,f:5] [distribution: test])
+memo (optimized, ~29KB, required=[presentation: info:14] [distribution: test])
+ ├── G1: (explain G2 [presentation: b:1,c:2,f:7] [distribution: test])
+ │    └── [presentation: info:14] [distribution: test]
+ │         ├── best: (explain G2="[presentation: b:1,c:2,f:7] [distribution: test]" [presentation: b:1,c:2,f:7] [distribution: test])
  │         └── cost: 2247.29
  ├── G2: (project G3 G4 b c f)
  │    ├── [presentation: b:1,c:2,f:5] [distribution: test]

pkg/sql/opt/memo/memo.go

@@ -197,6 +197,7 @@ type Memo struct {
 	pushLimitIntoProjectFilteredScan           bool
 	legacyVarcharTyping                        bool
 	preferBoundedCardinality                   bool
+	minRowCount                                float64
 
 	// txnIsoLevel is the isolation level under which the plan was created. This
 	// affects the planning of some locking operations, so it must be included in
@@ -285,6 +286,7 @@ func (m *Memo) Init(ctx context.Context, evalCtx *eval.Context) {
 		pushLimitIntoProjectFilteredScan:           evalCtx.SessionData().OptimizerPushLimitIntoProjectFilteredScan,
 		legacyVarcharTyping:                        evalCtx.SessionData().LegacyVarcharTyping,
 		preferBoundedCardinality:                   evalCtx.SessionData().OptimizerPreferBoundedCardinality,
+		minRowCount:                                evalCtx.SessionData().OptimizerMinRowCount,
 		txnIsoLevel:                                evalCtx.TxnIsoLevel,
 	}
 	m.metadata.Init()
@@ -451,6 +453,7 @@ func (m *Memo) IsStale(
 		m.pushLimitIntoProjectFilteredScan != evalCtx.SessionData().OptimizerPushLimitIntoProjectFilteredScan ||
 		m.legacyVarcharTyping != evalCtx.SessionData().LegacyVarcharTyping ||
 		m.preferBoundedCardinality != evalCtx.SessionData().OptimizerPreferBoundedCardinality ||
+		m.minRowCount != evalCtx.SessionData().OptimizerMinRowCount ||
 		m.txnIsoLevel != evalCtx.TxnIsoLevel {
 		return true, nil
 	}

pkg/sql/opt/memo/memo_test.go

@@ -512,6 +512,11 @@ func TestMemoIsStale(t *testing.T) {
 	evalCtx.SessionData().OptimizerPreferBoundedCardinality = false
 	notStale()
 
+	evalCtx.SessionData().OptimizerMinRowCount = 1.0
+	stale()
+	evalCtx.SessionData().OptimizerMinRowCount = 0
+	notStale()
+
 	// User no longer has access to view.
 	catalog.View(tree.NewTableNameWithSchema("t", catconstants.PublicSchemaName, "abcview")).Revoked = true
 	_, err = o.Memo().IsStale(ctx, &evalCtx, catalog)

pkg/sql/opt/memo/statistics_builder.go

@@ -228,18 +228,20 @@ const (
 //
 // See props/statistics.go for more details.
 type statisticsBuilder struct {
-	ctx     context.Context
-	evalCtx *eval.Context
-	md      *opt.Metadata
+	ctx         context.Context
+	evalCtx     *eval.Context
+	md          *opt.Metadata
+	minRowCount float64
 }
 
 func (sb *statisticsBuilder) init(ctx context.Context, evalCtx *eval.Context, md *opt.Metadata) {
 	// This initialization pattern ensures that fields are not unwittingly
 	// reused. Field reuse must be explicit.
 	*sb = statisticsBuilder{
-		ctx:     ctx,
-		evalCtx: evalCtx,
-		md:      md,
+		ctx:         ctx,
+		evalCtx:     evalCtx,
+		md:          md,
+		minRowCount: evalCtx.SessionData().OptimizerMinRowCount,
 	}
 }
 
@@ -826,7 +828,7 @@ func (sb *statisticsBuilder) colAvgSize(tabID opt.TableID, col opt.ColumnID) uin
 
 func (sb *statisticsBuilder) buildScan(scan *ScanExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1075,7 +1077,7 @@ func (sb *statisticsBuilder) colStatScan(colSet opt.ColSet, scan *ScanExpr) *pro
 
 func (sb *statisticsBuilder) buildSelect(sel *SelectExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1116,7 +1118,7 @@ func (sb *statisticsBuilder) colStatSelect(
 
 func (sb *statisticsBuilder) buildProject(prj *ProjectExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1218,7 +1220,7 @@ func (sb *statisticsBuilder) buildInvertedFilter(
 	invFilter *InvertedFilterExpr, relProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1279,7 +1281,7 @@ func (sb *statisticsBuilder) buildJoin(
 	}
 
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1810,7 +1812,7 @@ func (sb *statisticsBuilder) colStatFromJoinRight(
 
 func (sb *statisticsBuilder) buildIndexJoin(indexJoin *IndexJoinExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1891,7 +1893,7 @@ func (sb *statisticsBuilder) buildZigzagJoin(
 	zigzag *ZigzagJoinExpr, relProps *props.Relational, h *joinPropsHelper,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -1992,7 +1994,7 @@ func (sb *statisticsBuilder) buildZigzagJoin(
 
 func (sb *statisticsBuilder) buildGroupBy(groupNode RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2104,7 +2106,7 @@ func (sb *statisticsBuilder) colStatGroupBy(
 
 func (sb *statisticsBuilder) buildSetNode(setNode RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2209,7 +2211,7 @@ func (sb *statisticsBuilder) colStatSetNodeImpl(
 // buildValues builds the statistics for a VALUES expression.
 func (sb *statisticsBuilder) buildValues(values ValuesContainer, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2314,7 +2316,7 @@ func (sb *statisticsBuilder) colStatLiteralValues(
 
 func (sb *statisticsBuilder) buildLimit(limit *LimitExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2357,7 +2359,7 @@ func (sb *statisticsBuilder) colStatLimit(
 
 func (sb *statisticsBuilder) buildTopK(topK *TopKExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2399,7 +2401,7 @@ func (sb *statisticsBuilder) colStatTopK(colSet opt.ColSet, topK *TopKExpr) *pro
 
 func (sb *statisticsBuilder) buildOffset(offset *OffsetExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2452,7 +2454,7 @@ func (sb *statisticsBuilder) colStatOffset(
 
 func (sb *statisticsBuilder) buildMax1Row(max1Row *Max1RowExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2482,7 +2484,7 @@ func (sb *statisticsBuilder) colStatMax1Row(
 
 func (sb *statisticsBuilder) buildOrdinality(ord *OrdinalityExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2527,7 +2529,7 @@ func (sb *statisticsBuilder) colStatOrdinality(
 
 func (sb *statisticsBuilder) buildWindow(window *WindowExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2595,7 +2597,7 @@ func (sb *statisticsBuilder) buildProjectSet(
 	projectSet *ProjectSetExpr, relProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2728,7 +2730,7 @@ func (sb *statisticsBuilder) buildWithScan(
 	withScan *WithScanExpr, relProps, bindingProps *props.Relational,
 ) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2770,7 +2772,7 @@ func (sb *statisticsBuilder) colStatWithScan(
 
 func (sb *statisticsBuilder) buildMutation(mutation RelExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2808,7 +2810,7 @@ func (sb *statisticsBuilder) colStatMutation(
 
 func (sb *statisticsBuilder) buildLock(lock *LockExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2840,7 +2842,7 @@ func (sb *statisticsBuilder) colStatLock(colSet opt.ColSet, lock *LockExpr) *pro
 
 func (sb *statisticsBuilder) buildBarrier(barrier *BarrierExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short cut if cardinality is 0.
 		return
 	}
@@ -2874,7 +2876,7 @@ func (sb *statisticsBuilder) colStatBarrier(
 
 func (sb *statisticsBuilder) buildCall(call *CallExpr, relProps *props.Relational) {
 	s := relProps.Statistics()
-	if zeroCardinality := s.Init(relProps); zeroCardinality {
+	if zeroCardinality := s.Init(relProps, sb.minRowCount); zeroCardinality {
 		// Short-cut if cardinality is 0.
 		return
 	}

pkg/sql/opt/memo/statistics_builder_test.go

@@ -112,7 +112,8 @@ func TestGetStatsFromConstraint(t *testing.T) {
 		relProps := &props.Relational{Cardinality: props.AnyCardinality}
 		relProps.NotNullCols = cs.ExtractNotNullCols(ctx, &evalCtx)
 		s := relProps.Statistics()
-		s.Init(relProps)
+		const minRowCount = 0
+		s.Init(relProps, minRowCount)
 
 		// Calculate distinct counts.
 		sb.applyConstraintSet(cs, true /* tight */, sel, relProps, relProps.Statistics())

pkg/sql/opt/memo/testdata/memo

@@ -317,7 +317,7 @@ memo (optimized, ~6KB, required=[presentation: array_agg:6])
 memo
 SELECT array_agg(x) FROM (SELECT * FROM a) GROUP BY y
 ----
-memo (optimized, ~7KB, required=[presentation: array_agg:6])
+memo (optimized, ~8KB, required=[presentation: array_agg:6])
  ├── G1: (project G2 G3 array_agg)
  │    └── [presentation: array_agg:6]
  │         ├── best: (project G2 G3 array_agg)
@@ -373,7 +373,7 @@ memo (optimized, ~6KB, required=[presentation: array_cat_agg:6])
 memo
 SELECT array_cat_agg(arr) FROM (SELECT * FROM a) GROUP BY y
 ----
-memo (optimized, ~7KB, required=[presentation: array_cat_agg:6])
+memo (optimized, ~8KB, required=[presentation: array_cat_agg:6])
  ├── G1: (project G2 G3 array_cat_agg)
  │    └── [presentation: array_cat_agg:6]
  │         ├── best: (project G2 G3 array_cat_agg)