FB8-167: Add variable to force range plans if force index is used (fa…

…cebook#967) (facebook#967)

Summary:
Jira ticket: https://jira.percona.com/browse/FB8-167

Reference Patch: facebook@b01ff6a
Currently, when force index is used, full table scans are highly penalized, but it is still possible to do a full index scan which is expensive. In many cases, we only use force index when we know there is better way to traverse the index, and not necessarily because we want a full index scan.

To address these cases, add a session variable called optimizer_force_index_for_range, which when turned on, will try to search for a 'range' plan before falling back to an index plan.

Pull Request resolved: facebook#967

Reviewed By: lloyd

Differential Revision: D14387089

Pulled By: lth

mysql-test/collections/disabled_rocksdb.def

@@ -74,8 +74,6 @@ rocksdb.bypass_select_range_sk_bloom : BUG#0000 Rebase: not-yet-implemented
 rocksdb.bypass_select_basic_bloom : BUG#0000 Rebase: not-yet-implemented
 rocksdb.rocksdb_deadlock_detect_rc : BUG#0000 Rebase: not-yet-implemented
 rocksdb.rocksdb_deadlock_detect_rr : BUG#0000 Rebase: not-yet-implemented
-rocksdb.bloomfilter2 : BUG#0000 Rebase: not-yet-implemented
-rocksdb.bloomfilter3 : BUG#0000 Rebase: not-yet-implemented
 rocksdb.track_and_verify_wals_in_manifest : BUG#0000 Rebase: not-yet-implemented
 rocksdb.mysqldump : BUG#0000 Rebase: not-yet-implemented
 rocksdb_rpl.rocksdb_slave_check_before_image_consistency : BUG#0000 Rebase: not-yet-implemented

mysql-test/r/mysqld--help-notwin.result

@@ -867,6 +867,9 @@ The following options may be given as the first argument:
  value is 0 then mysqld will reserve max_connections*5 or
  max_connections + table_open_cache*2 (whichever is
  larger) number of file descriptors
+ --optimizer-force-index-for-range 
+ If enabled, FORCE INDEX will also try to force a range
+ plan.
  --optimizer-max-subgraph-pairs=# 
  Maximum depth of subgraph pairs a query can have before
  the hypergraph join optimizer starts reducing the search
@@ -2592,6 +2595,7 @@ offline-mode FALSE
 old FALSE
 old-alter-table FALSE
 old-style-user-limits FALSE
+optimizer-force-index-for-range FALSE
 optimizer-max-subgraph-pairs 100000
 optimizer-prune-level 1
 optimizer-search-depth 62

mysql-test/r/opt_hints_set_var.result

@@ -461,6 +461,23 @@ VARIABLE_VALUE
 0
 VARIABLE_VALUE
 1
+EXPLAIN SELECT DISTINCT a FROM t1 FORCE INDEX (PRIMARY);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	index	PRIMARY	PRIMARY	4	NULL	4	100.00	Using index
+Warnings:
+Note	1003	/* select#1 */ select distinct `test`.`t1`.`a` AS `a` from `test`.`t1` FORCE INDEX (PRIMARY)
+EXPLAIN SELECT /*+ SET_VAR(optimizer_force_index_for_range=1) */ DISTINCT a FROM t1 FORCE INDEX (PRIMARY);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	range	PRIMARY	PRIMARY	4	NULL	5	100.00	Using index for group-by
+Warnings:
+Note	1003	/* select#1 */ select /*+ SET_VAR(optimizer_force_index_for_range=1) */ distinct `test`.`t1`.`a` AS `a` from `test`.`t1` FORCE INDEX (PRIMARY)
+CALL test_hint("SET_VAR(optimizer_force_index_for_range=1)", "optimizer_force_index_for_range");
+VARIABLE_VALUE
+OFF
+VARIABLE_VALUE
+ON
+VARIABLE_VALUE
+OFF
 DROP TABLE t1, t2, t3;
 CREATE TABLE t1
 (

mysql-test/r/optimizer_force_index_for_range.result

@@ -0,0 +1,86 @@
+CREATE TABLE t (i INT, j INT, PRIMARY KEY (i, j));
+INSERT INTO t VALUES (1, 1);
+INSERT INTO t VALUES (1, 2);
+INSERT INTO t VALUES (2, 1);
+INSERT INTO t VALUES (2, 2);
+INSERT INTO t VALUES (3, 1);
+INSERT INTO t VALUES (3, 2);
+# Test range plans
+SET optimizer_force_index_for_range = ON;
+EXPLAIN SELECT i FROM t WHERE i IN (1, 2, 3) AND j IN (1, 2);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` where ((`test`.`t`.`i` in (1,2,3)) and (`test`.`t`.`j` in (1,2)))
+EXPLAIN SELECT i FROM t FORCE INDEX (PRIMARY) WHERE i IN (1, 2, 3) AND j IN (1, 2);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	range	PRIMARY	PRIMARY	8	NULL	#	100.00	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY) where ((`test`.`t`.`i` in (1,2,3)) and (`test`.`t`.`j` in (1,2)))
+SET optimizer_force_index_for_range = OFF;
+EXPLAIN SELECT i FROM t WHERE i IN (1, 2, 3) AND j IN (1, 2);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` where ((`test`.`t`.`i` in (1,2,3)) and (`test`.`t`.`j` in (1,2)))
+EXPLAIN SELECT i FROM t FORCE INDEX (PRIMARY) WHERE i IN (1, 2, 3) AND j IN (1, 2);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY) where ((`test`.`t`.`i` in (1,2,3)) and (`test`.`t`.`j` in (1,2)))
+# Test group-by plans.
+SET optimizer_force_index_for_range = ON;
+EXPLAIN SELECT DISTINCT i FROM t;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using index
+Warnings:
+Note	1003	/* select#1 */ select distinct `test`.`t`.`i` AS `i` from `test`.`t`
+EXPLAIN SELECT DISTINCT i FROM t FORCE INDEX (PRIMARY);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	range	PRIMARY	PRIMARY	4	NULL	#	100.00	Using index for group-by
+Warnings:
+Note	1003	/* select#1 */ select distinct `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY)
+SET optimizer_force_index_for_range = OFF;
+EXPLAIN SELECT DISTINCT i FROM t;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using index
+Warnings:
+Note	1003	/* select#1 */ select distinct `test`.`t`.`i` AS `i` from `test`.`t`
+EXPLAIN SELECT DISTINCT i FROM t FORCE INDEX (PRIMARY);
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	100.00	Using index
+Warnings:
+Note	1003	/* select#1 */ select distinct `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY)
+# Test skip-scan plans.
+SET optimizer_switch = 'skip_scan=on';
+SET optimizer_force_index_for_range = ON;
+EXPLAIN SELECT i FROM t WHERE j > 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	33.33	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` where (`test`.`t`.`j` > 1)
+EXPLAIN SELECT i FROM t FORCE INDEX (PRIMARY) WHERE j > 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	range	PRIMARY	PRIMARY	8	NULL	#	100.00	Using where; Using index for skip scan
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY) where (`test`.`t`.`j` > 1)
+SET optimizer_force_index_for_range = OFF;
+EXPLAIN SELECT i FROM t WHERE j > 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	33.33	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` where (`test`.`t`.`j` > 1)
+EXPLAIN SELECT i FROM t FORCE INDEX (PRIMARY) WHERE j > 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	PRIMARY	PRIMARY	8	NULL	#	33.33	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY) where (`test`.`t`.`j` > 1)
+SET optimizer_switch = 'skip_scan=off';
+# Test that in absence of range plan, index is used.
+SET optimizer_force_index_for_range = ON;
+EXPLAIN SELECT i FROM t FORCE INDEX (PRIMARY) WHERE j > 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t	NULL	index	NULL	PRIMARY	8	NULL	#	33.33	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t`.`i` AS `i` from `test`.`t` FORCE INDEX (PRIMARY) where (`test`.`t`.`j` > 1)
+DROP TABLE t;

mysql-test/suite/sys_vars/r/optimizer_force_index_for_range_basic.result

@@ -0,0 +1,93 @@
+SET @session_start_value = @@session.optimizer_force_index_for_range;
+SELECT @session_start_value;
+@session_start_value
+0
+SET @global_start_value = @@global.optimizer_force_index_for_range;
+SELECT @global_start_value;
+@global_start_value
+0
+SET @@session.optimizer_force_index_for_range = 0;
+SET @@session.optimizer_force_index_for_range = DEFAULT;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@session.optimizer_force_index_for_range = 1;
+SET @@session.optimizer_force_index_for_range = DEFAULT;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET optimizer_force_index_for_range = 1;
+SELECT @@optimizer_force_index_for_range;
+@@optimizer_force_index_for_range
+1
+SELECT session.optimizer_force_index_for_range;
+ERROR 42S02: Unknown table 'session' in field list
+SELECT local.optimizer_force_index_for_range;
+ERROR 42S02: Unknown table 'local' in field list
+SET session optimizer_force_index_for_range = 0;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@session.optimizer_force_index_for_range = 0;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@session.optimizer_force_index_for_range = 1;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+1
+SET @@session.optimizer_force_index_for_range = -1;
+ERROR 42000: Variable 'optimizer_force_index_for_range' can't be set to the value of '-1'
+SET @@session.optimizer_force_index_for_range = 2;
+ERROR 42000: Variable 'optimizer_force_index_for_range' can't be set to the value of '2'
+SET @@session.optimizer_force_index_for_range = "T";
+ERROR 42000: Variable 'optimizer_force_index_for_range' can't be set to the value of 'T'
+SET @@session.optimizer_force_index_for_range = "Y";
+ERROR 42000: Variable 'optimizer_force_index_for_range' can't be set to the value of 'Y'
+SET @@session.optimizer_force_index_for_range = NO;
+ERROR 42000: Variable 'optimizer_force_index_for_range' can't be set to the value of 'NO'
+SET @@global.optimizer_force_index_for_range = 1;
+SELECT @@global.optimizer_force_index_for_range;
+@@global.optimizer_force_index_for_range
+1
+SET @@global.optimizer_force_index_for_range = 0;
+SELECT count(VARIABLE_VALUE) FROM performance_schema.global_variables WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+count(VARIABLE_VALUE)
+1
+SELECT IF(@@session.optimizer_force_index_for_range, "ON", "OFF") = VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+IF(@@session.optimizer_force_index_for_range, "ON", "OFF") = VARIABLE_VALUE
+1
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+1
+SELECT VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+VARIABLE_VALUE
+ON
+SET @@session.optimizer_force_index_for_range = OFF;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@session.optimizer_force_index_for_range = ON;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+1
+SET @@session.optimizer_force_index_for_range = TRUE;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+1
+SET @@session.optimizer_force_index_for_range = FALSE;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@session.optimizer_force_index_for_range = @session_start_value;
+SELECT @@session.optimizer_force_index_for_range;
+@@session.optimizer_force_index_for_range
+0
+SET @@global.optimizer_force_index_for_range = @global_start_value;
+SELECT @@global.optimizer_force_index_for_range;
+@@global.optimizer_force_index_for_range
+0

mysql-test/suite/sys_vars/t/optimizer_force_index_for_range_basic.test

@@ -0,0 +1,102 @@
+--source include/load_sysvars.inc
+
+
+# Saving initial value of optimizer_force_index_for_range in a temporary variable
+
+SET @session_start_value = @@session.optimizer_force_index_for_range;
+SELECT @session_start_value;
+SET @global_start_value = @@global.optimizer_force_index_for_range;
+SELECT @global_start_value;
+
+# Display the DEFAULT value of optimizer_force_index_for_range
+
+SET @@session.optimizer_force_index_for_range = 0;
+SET @@session.optimizer_force_index_for_range = DEFAULT;
+SELECT @@session.optimizer_force_index_for_range;
+
+SET @@session.optimizer_force_index_for_range = 1;
+SET @@session.optimizer_force_index_for_range = DEFAULT;
+SELECT @@session.optimizer_force_index_for_range;
+
+
+# Check if optimizer_force_index_for_range can be accessed with and without @@ sign
+
+SET optimizer_force_index_for_range = 1;
+SELECT @@optimizer_force_index_for_range;
+
+--Error ER_UNKNOWN_TABLE
+SELECT session.optimizer_force_index_for_range;
+
+--Error ER_UNKNOWN_TABLE
+SELECT local.optimizer_force_index_for_range;
+
+SET session optimizer_force_index_for_range = 0;
+SELECT @@session.optimizer_force_index_for_range;
+
+# change the value of optimizer_force_index_for_range to a valid value
+
+SET @@session.optimizer_force_index_for_range = 0;
+SELECT @@session.optimizer_force_index_for_range;
+SET @@session.optimizer_force_index_for_range = 1;
+SELECT @@session.optimizer_force_index_for_range;
+
+
+# Change the value of optimizer_force_index_for_range to invalid value
+
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.optimizer_force_index_for_range = -1;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.optimizer_force_index_for_range = 2;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.optimizer_force_index_for_range = "T";
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.optimizer_force_index_for_range = "Y";
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.optimizer_force_index_for_range = NO;
+
+
+# Test if accessing global optimizer_force_index_for_range gives error
+
+SET @@global.optimizer_force_index_for_range = 1;
+SELECT @@global.optimizer_force_index_for_range;
+SET @@global.optimizer_force_index_for_range = 0;
+
+
+# Check if the value in GLOBAL Table contains variable value
+
+SELECT count(VARIABLE_VALUE) FROM performance_schema.global_variables WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+
+
+# Check if the value in GLOBAL Table matches value in variable
+
+SELECT IF(@@session.optimizer_force_index_for_range, "ON", "OFF") = VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+SELECT @@session.optimizer_force_index_for_range;
+SELECT VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='optimizer_force_index_for_range';
+
+
+# Check if ON and OFF values can be used on variable
+
+SET @@session.optimizer_force_index_for_range = OFF;
+SELECT @@session.optimizer_force_index_for_range;
+SET @@session.optimizer_force_index_for_range = ON;
+SELECT @@session.optimizer_force_index_for_range;
+
+
+# Check if TRUE and FALSE values can be used on variable
+
+SET @@session.optimizer_force_index_for_range = TRUE;
+SELECT @@session.optimizer_force_index_for_range;
+SET @@session.optimizer_force_index_for_range = FALSE;
+SELECT @@session.optimizer_force_index_for_range;
+
+
+# Restore initial value
+
+SET @@session.optimizer_force_index_for_range = @session_start_value;
+SELECT @@session.optimizer_force_index_for_range;
+SET @@global.optimizer_force_index_for_range = @global_start_value;
+SELECT @@global.optimizer_force_index_for_range;

mysql-test/t/opt_hints_set_var.test

@@ -259,6 +259,13 @@ t2.a IN (SELECT t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
 CALL test_hint("SET_VAR(optimizer_search_depth=1)", "optimizer_search_depth");
 CALL test_hint("SET_VAR(optimizer_prune_level=0)", "optimizer_prune_level");
 
+# Testing optimizer_force_index_for_range variable
+EXPLAIN SELECT DISTINCT a FROM t1 FORCE INDEX (PRIMARY);
+
+EXPLAIN SELECT /*+ SET_VAR(optimizer_force_index_for_range=1) */ DISTINCT a FROM t1 FORCE INDEX (PRIMARY);
+
+CALL test_hint("SET_VAR(optimizer_force_index_for_range=1)", "optimizer_force_index_for_range");
+
 DROP TABLE t1, t2, t3;