Add index length limit (facebook#673) (facebook#673)

Summary: Address facebook#665. Changes: * Adds a variable `rocksdb_large_prefix` to mirror `innodb_large_prefix`. If on, the index length limit is 3072 bytes. If off, it is 767. * Modifies `ha_rocksdb::max_supported_key_part_length` to return the new limits. The values for `large_prefix` should be the same between master and slave. In 5.6 `innodb_large_prefix` defaults to off. In 5.7.7 it is deprecated and defaults to on. https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_large_prefix Closes facebook#673 Differential Revision: D5554460 Pulled By: pbjc
hermanlee · Jul 13, 2022 · 749f1dc · 749f1dc
1 parent f8d7093
commit 749f1dc
Show file tree

Hide file tree

Showing 14 changed files with 288 additions and 6 deletions.
diff --git a/mysql-test/suite/rocksdb/r/dup_key_update.result b/mysql-test/suite/rocksdb/r/dup_key_update.result
@@ -178,16 +178,20 @@ id1	id2	id3
 9	17	9
 DROP TABLE t1;
 DROP TABLE t2;
+set global rocksdb_large_prefix=1;
 CREATE TABLE t1 (id1 varchar(128) CHARACTER SET latin1 COLLATE latin1_bin,
 id2 varchar(256) CHARACTER SET utf8 COLLATE utf8_bin,
 id3 varchar(200) CHARACTER SET latin1 COLLATE latin1_swedish_ci,
 PRIMARY KEY (id1, id2, id3),
 UNIQUE KEY (id3, id1)) ENGINE=ROCKSDB;
+set global rocksdb_large_prefix=DEFAULT;
+set global rocksdb_large_prefix=1;
 CREATE TABLE t2 (id1 varchar(128) CHARACTER SET latin1 COLLATE latin1_bin,
 id2 varchar(256) CHARACTER SET utf8 COLLATE utf8_bin,
 id3 varchar(200) CHARACTER SET latin1 COLLATE latin1_swedish_ci,
 PRIMARY KEY (id1, id2, id3),
 UNIQUE KEY (id3, id1) COMMENT 'rev:cf') ENGINE=ROCKSDB;
+set global rocksdb_large_prefix=DEFAULT;
 INSERT INTO t1 VALUES (1, 1, 1) ON DUPLICATE KEY UPDATE id2 = 9;
 SELECT * FROM t1 WHERE id1 = 1;
 id1	id2	id3

diff --git a/mysql-test/suite/rocksdb/r/index.result b/mysql-test/suite/rocksdb/r/index.result
@@ -40,6 +40,33 @@ t1	0	PRIMARY	1	pk	A	#	NULL	NULL		LSMTREE
 t1	1	a	1	a	A	#	NULL	NULL	YES	LSMTREE		simple index on a
 ALTER TABLE t1 DROP KEY a;
 DROP TABLE t1;
+set global rocksdb_large_prefix=0;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(768))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3073))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 3072 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=DEFAULT;
 #
 # Issue #376: MyRocks: ORDER BY optimizer is unable to use the index extension
 #

diff --git a/mysql-test/suite/rocksdb/r/index_primary.result b/mysql-test/suite/rocksdb/r/index_primary.result
@@ -46,3 +46,26 @@ SHOW KEYS IN t1;
 Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
 t1	0	PRIMARY	1	b	A	#	NULL	NULL		LSMTREE		
 DROP TABLE t1;
+set global rocksdb_large_prefix=0;
+CREATE TABLE t1 (
+a BLOB(1024),
+PRIMARY KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(1024),
+PRIMARY KEY (a(768))
+) ENGINE=rocksdb;
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+set global rocksdb_large_prefix=1;
+CREATE TABLE t1 (
+a BLOB(4096),
+PRIMARY KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(4096),
+PRIMARY KEY (a(3073))
+) ENGINE=rocksdb;
+ERROR 42000: Specified key was too long; max key length is 3072 bytes
+set global rocksdb_large_prefix=DEFAULT;
diff --git a/mysql-test/suite/rocksdb/r/index_type_btree.result b/mysql-test/suite/rocksdb/r/index_type_btree.result
@@ -40,3 +40,30 @@ t1	0	PRIMARY	1	pk	A	#	NULL	NULL		LSMTREE
 t1	1	a	1	a	A	#	NULL	NULL	YES	LSMTREE		simple index on a
 ALTER TABLE t1 DROP KEY a;
 DROP TABLE t1;
+set global rocksdb_large_prefix=0;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(768))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3073))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 3072 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=DEFAULT;
diff --git a/mysql-test/suite/rocksdb/r/index_type_hash.result b/mysql-test/suite/rocksdb/r/index_type_hash.result
@@ -40,3 +40,30 @@ t1	0	PRIMARY	1	pk	A	#	NULL	NULL		LSMTREE
 t1	1	a	1	a	A	#	NULL	NULL	YES	LSMTREE		simple index on a
 ALTER TABLE t1 DROP KEY a;
 DROP TABLE t1;
+set global rocksdb_large_prefix=0;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(1024),
+KEY (a(768))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+CREATE TABLE t1 (
+a BLOB(4096),
+KEY (a(3073))
+) ENGINE=rocksdb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 3072 bytes
+DROP TABLE t1;
+set global rocksdb_large_prefix=DEFAULT;
diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result
@@ -912,6 +912,7 @@ rocksdb_info_log_level	error_level
 rocksdb_io_write_timeout	0
 rocksdb_is_fd_close_on_exec	ON
 rocksdb_keep_log_file_num	1000
+rocksdb_large_prefix	OFF
 rocksdb_lock_scanned_rows	OFF
 rocksdb_lock_wait_timeout	1
 rocksdb_log_file_time_to_roll	0
@@ -2149,7 +2150,9 @@ SET @old_mode = @@sql_mode;
 SET sql_mode = 'strict_all_tables';
 create table t1 (a int, b text, c varchar(400), Primary Key(a), Key(c, b(255))) engine=rocksdb;
 drop table t1;
+set global rocksdb_large_prefix=1;
 create table t1 (a int, b text, c varchar(400), Primary Key(a), Key(b(1255))) engine=rocksdb;
+set global rocksdb_large_prefix=0;
 insert into t1 values (1, '1abcde', '1abcde'), (2, '2abcde', '2abcde'), (3, '3abcde', '3abcde');
 select * from t1;
 a	b	c
@@ -2170,7 +2173,7 @@ a	b	c
 3	3abcde	3abcde
 drop table t1;
 create table t1 (a int, b text, c varchar(400), Primary Key(a), Key(b(2255))) engine=rocksdb;
-ERROR 42000: Specified key was too long; max key length is 2048 bytes
+ERROR 42000: Specified key was too long; max key length is 767 bytes
 SET sql_mode = @old_mode;
 drop table t0;
 #

diff --git a/mysql-test/suite/rocksdb/t/dup_key_update.test b/mysql-test/suite/rocksdb/t/dup_key_update.test
@@ -22,17 +22,21 @@ CREATE TABLE t2 (id1 INT, id2 INT, id3 INT,
 DROP TABLE t1;
 DROP TABLE t2;
 
+set global rocksdb_large_prefix=1;
 CREATE TABLE t1 (id1 varchar(128) CHARACTER SET latin1 COLLATE latin1_bin,
                  id2 varchar(256) CHARACTER SET utf8 COLLATE utf8_bin,
                  id3 varchar(200) CHARACTER SET latin1 COLLATE latin1_swedish_ci,
                  PRIMARY KEY (id1, id2, id3),
                  UNIQUE KEY (id3, id1)) ENGINE=ROCKSDB;
+set global rocksdb_large_prefix=DEFAULT;
 
+set global rocksdb_large_prefix=1;
 CREATE TABLE t2 (id1 varchar(128) CHARACTER SET latin1 COLLATE latin1_bin,
                  id2 varchar(256) CHARACTER SET utf8 COLLATE utf8_bin,
                  id3 varchar(200) CHARACTER SET latin1 COLLATE latin1_swedish_ci,
                  PRIMARY KEY (id1, id2, id3),
                  UNIQUE KEY (id3, id1) COMMENT 'rev:cf') ENGINE=ROCKSDB;
+set global rocksdb_large_prefix=DEFAULT;
 
 --source suite/rocksdb/include/dup_key_update.inc
 

diff --git a/mysql-test/suite/rocksdb/t/index.inc b/mysql-test/suite/rocksdb/t/index.inc
@@ -119,3 +119,37 @@ DROP TABLE t1;
 
 --enable_parsing
 
+#
+# Test index prefix length limits.
+#
+set global rocksdb_large_prefix=0;
+
+CREATE TABLE t1 (
+  a BLOB(1024),
+  KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+# Should display warning
+CREATE TABLE t1 (
+  a BLOB(1024),
+  KEY (a(768))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+set global rocksdb_large_prefix=1;
+
+CREATE TABLE t1 (
+  a BLOB(4096),
+  KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+# Should display warning
+CREATE TABLE t1 (
+  a BLOB(4096),
+  KEY (a(3073))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+set global rocksdb_large_prefix=DEFAULT;
diff --git a/mysql-test/suite/rocksdb/t/index_primary.test b/mysql-test/suite/rocksdb/t/index_primary.test
@@ -62,3 +62,35 @@ ALTER TABLE t1 ADD CONSTRAINT PRIMARY KEY pk (a);
 SHOW KEYS IN t1;
 DROP TABLE t1;
 
+#
+# Test index prefix length limits.
+#
+set global rocksdb_large_prefix=0;
+
+CREATE TABLE t1 (
+  a BLOB(1024),
+  PRIMARY KEY (a(767))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+--error ER_TOO_LONG_KEY
+CREATE TABLE t1 (
+  a BLOB(1024),
+  PRIMARY KEY (a(768))
+) ENGINE=rocksdb;
+
+set global rocksdb_large_prefix=1;
+
+CREATE TABLE t1 (
+  a BLOB(4096),
+  PRIMARY KEY (a(3072))
+) ENGINE=rocksdb;
+DROP TABLE t1;
+
+--error ER_TOO_LONG_KEY
+CREATE TABLE t1 (
+  a BLOB(4096),
+  PRIMARY KEY (a(3073))
+) ENGINE=rocksdb;
+
+set global rocksdb_large_prefix=DEFAULT;
diff --git a/mysql-test/suite/rocksdb/t/rocksdb.test b/mysql-test/suite/rocksdb/t/rocksdb.test
@@ -1657,7 +1657,9 @@ SET @old_mode = @@sql_mode;
 SET sql_mode = 'strict_all_tables';
 create table t1 (a int, b text, c varchar(400), Primary Key(a), Key(c, b(255))) engine=rocksdb;
 drop table t1;
+set global rocksdb_large_prefix=1;
 create table t1 (a int, b text, c varchar(400), Primary Key(a), Key(b(1255))) engine=rocksdb;
+set global rocksdb_large_prefix=0;
 insert into t1 values (1, '1abcde', '1abcde'), (2, '2abcde', '2abcde'), (3, '3abcde', '3abcde');
 select * from t1;
 --replace_column 9 #

diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_large_prefix_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_large_prefix_basic.result
@@ -0,0 +1,64 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+SET @start_global_value = @@global.ROCKSDB_LARGE_PREFIX;
+SELECT @start_global_value;
+@start_global_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_LARGE_PREFIX to 1"
+SET @@global.ROCKSDB_LARGE_PREFIX   = 1;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_LARGE_PREFIX = DEFAULT;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+"Trying to set variable @@global.ROCKSDB_LARGE_PREFIX to 0"
+SET @@global.ROCKSDB_LARGE_PREFIX   = 0;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_LARGE_PREFIX = DEFAULT;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+"Trying to set variable @@global.ROCKSDB_LARGE_PREFIX to on"
+SET @@global.ROCKSDB_LARGE_PREFIX   = on;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_LARGE_PREFIX = DEFAULT;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+"Trying to set variable @@session.ROCKSDB_LARGE_PREFIX to 444. It should fail because it is not session."
+SET @@session.ROCKSDB_LARGE_PREFIX   = 444;
+ERROR HY000: Variable 'rocksdb_large_prefix' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_LARGE_PREFIX to 'aaa'"
+SET @@global.ROCKSDB_LARGE_PREFIX   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+"Trying to set variable @@global.ROCKSDB_LARGE_PREFIX to 'bbb'"
+SET @@global.ROCKSDB_LARGE_PREFIX   = 'bbb';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+SET @@global.ROCKSDB_LARGE_PREFIX = @start_global_value;
+SELECT @@global.ROCKSDB_LARGE_PREFIX;
+@@global.ROCKSDB_LARGE_PREFIX
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;
diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_large_prefix_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_large_prefix_basic.test
@@ -0,0 +1,18 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+
+--let $sys_var=ROCKSDB_LARGE_PREFIX
+--let $read_only=0
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;