Skip to content

Commit

Permalink
Add a reverse CF test for unsorted bulkload
Browse files Browse the repository at this point in the history 
Summary:
Add a reverse CF test for unsorted bulkload, and force index in selects
for consistency.

@update-submodule: rocksdb

Depends on D5684756 (b2f4b46)

Differential Revision: D6026211
  • Loading branch information
Aliaksei Sandryhaila authored and Herman Lee committed Dec 7, 2021
1 parent 0f32865 commit 352107c
Show file tree
Hide file tree
Showing 6 changed files with 281 additions and 145 deletions.
36 changes: 24 additions & 12 deletions mysql-test/suite/rocksdb/r/bulk_load_unsorted.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ SET rocksdb_bulk_load_size=3;
SET rocksdb_bulk_load_allow_unsorted=1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1");
SET rocksdb_bulk_load=1;
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
SET rocksdb_bulk_load=0;
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
-3 5
-1 3
Expand All @@ -16,32 +16,34 @@ a b
DROP TABLE t1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1", KEY(b));
SET rocksdb_bulk_load=1;
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
6 -4
4 -2
2 0
-1 3
-3 5
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
-3 5
-1 3
2 0
4 -2
6 -4
DROP TABLE t1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1");
CREATE TABLE t2(a INT, b INT, PRIMARY KEY(a) COMMENT "cf1");
SET rocksdb_bulk_load=1;
INSERT INTO t1 VALUES (1,1);
INSERT INTO t2 VALUES (1,1);
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
1 1
INSERT INTO t1 VALUES (2,2);
SELECT * FROM t2;
SELECT * FROM t2 FORCE INDEX (PRIMARY);
a b
1 1
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
1 1
SET rocksdb_bulk_load=0;
SELECT * FROM t1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
1 1
2 2
Expand Down Expand Up @@ -97,5 +99,15 @@ count(a)
select count(b) from t3;
count(b)
5000000
SELECT * FROM t1 FORCE INDEX (PRIMARY) LIMIT 3;
a b
-4999998 5000000
-4999996 4999998
-4999994 4999996
SELECT * FROM t2 FORCE INDEX (PRIMARY) LIMIT 3;
a b
4999999 -4999997
4999997 -4999995
4999995 -4999993
DROP TABLE t1, t2, t3;
SET rocksdb_bulk_load_allow_unsorted=0;
113 changes: 113 additions & 0 deletions mysql-test/suite/rocksdb/r/bulk_load_unsorted_rev.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
DROP TABLE IF EXISTS t1;
SET rocksdb_bulk_load_size=3;
SET rocksdb_bulk_load_allow_unsorted=1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1");
SET rocksdb_bulk_load=1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
6 -4
4 -2
2 0
-1 3
-3 5
DROP TABLE t1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1", KEY(b));
SET rocksdb_bulk_load=1;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
6 -4
4 -2
2 0
-1 3
-3 5
DROP TABLE t1;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1");
CREATE TABLE t2(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1");
SET rocksdb_bulk_load=1;
INSERT INTO t1 VALUES (1,1);
INSERT INTO t2 VALUES (1,1);
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
1 1
INSERT INTO t1 VALUES (2,2);
SELECT * FROM t2 FORCE INDEX (PRIMARY);
a b
1 1
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
1 1
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
a b
2 2
1 1
DROP TABLE t1, t2;
CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1");
CREATE TABLE t2(a INT, b INT, PRIMARY KEY(b) COMMENT "rev:cf1");
CREATE TABLE t3(a INT, b INT, PRIMARY KEY(a) COMMENT "rev:cf1")
PARTITION BY KEY() PARTITIONS 4;
set session transaction isolation level repeatable read;
select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
STAT_TYPE VALUE
DB_NUM_SNAPSHOTS 0
start transaction with consistent snapshot;
select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
STAT_TYPE VALUE
DB_NUM_SNAPSHOTS 1
set rocksdb_bulk_load=1;
set rocksdb_bulk_load_size=100000;
LOAD DATA INFILE <input_file> INTO TABLE t1;
LOAD DATA INFILE <input_file> INTO TABLE t2;
LOAD DATA INFILE <input_file> INTO TABLE t3;
set rocksdb_bulk_load=0;
SHOW TABLE STATUS WHERE name LIKE 't%';
Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment
t1 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL
t2 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL
t3 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL partitioned
ANALYZE TABLE t1, t2, t3;
Table Op Msg_type Msg_text
test.t1 analyze status OK
test.t2 analyze status OK
test.t3 analyze status OK
SHOW TABLE STATUS WHERE name LIKE 't%';
Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment
t1 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL
t2 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL
t3 ROCKSDB 10 Fixed 5000000 # # # # 0 NULL NULL NULL NULL latin1_swedish_ci NULL partitioned
select count(a) from t1;
count(a)
5000000
select count(b) from t1;
count(b)
5000000
select count(a) from t2;
count(a)
5000000
select count(b) from t2;
count(b)
5000000
select count(a) from t3;
count(a)
5000000
select count(b) from t3;
count(b)
5000000
SELECT * FROM t1 FORCE INDEX (PRIMARY) LIMIT 3;
a b
4999999 -4999997
4999997 -4999995
4999995 -4999993
SELECT * FROM t2 FORCE INDEX (PRIMARY) LIMIT 3;
a b
-4999998 5000000
-4999996 4999998
-4999994 4999996
DROP TABLE t1, t2, t3;
SET rocksdb_bulk_load_allow_unsorted=0;
137 changes: 137 additions & 0 deletions mysql-test/suite/rocksdb/t/bulk_load_unsorted.inc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
--disable_warnings
DROP TABLE IF EXISTS t1;
--enable_warnings

SET rocksdb_bulk_load_size=3;
SET rocksdb_bulk_load_allow_unsorted=1;

### Test individual INSERTs ###

# A table with only a PK won't have rows until the bulk load is finished
eval CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf");
SET rocksdb_bulk_load=1;
--disable_query_log
let $sign = 1;
let $max = 5;
let $i = 1;
while ($i <= $max) {
let $a = 1 + $sign * $i;
let $b = 1 - $sign * $i;
let $sign = -$sign;
let $insert = INSERT INTO t1 VALUES ($a, $b);
eval $insert;
inc $i;
}
--enable_query_log
SELECT * FROM t1 FORCE INDEX (PRIMARY);
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
DROP TABLE t1;

# A table with a PK and a SK shows rows immediately
eval CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf", KEY(b));
SET rocksdb_bulk_load=1;
--disable_query_log
let $sign = 1;
let $max = 5;
let $i = 1;
while ($i <= $max) {
let $a = 1 + $sign * $i;
let $b = 1 - $sign * $i;
let $sign = -$sign;
let $insert = INSERT INTO t1 VALUES ($a, $b);
eval $insert;
inc $i;
}
--enable_query_log

SELECT * FROM t1 FORCE INDEX (PRIMARY);
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
DROP TABLE t1;

# Inserting into another table finishes bulk load to the previous table
eval CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf");
eval CREATE TABLE t2(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf");

SET rocksdb_bulk_load=1;
INSERT INTO t1 VALUES (1,1);
INSERT INTO t2 VALUES (1,1);
SELECT * FROM t1 FORCE INDEX (PRIMARY);
INSERT INTO t1 VALUES (2,2);
SELECT * FROM t2 FORCE INDEX (PRIMARY);
SELECT * FROM t1 FORCE INDEX (PRIMARY);
SET rocksdb_bulk_load=0;
SELECT * FROM t1 FORCE INDEX (PRIMARY);
DROP TABLE t1, t2;

### Test bulk load from a file ###
eval CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf");
eval CREATE TABLE t2(a INT, b INT, PRIMARY KEY(b) COMMENT "$pk_cf");
eval CREATE TABLE t3(a INT, b INT, PRIMARY KEY(a) COMMENT "$pk_cf")
PARTITION BY KEY() PARTITIONS 4;

--let $file = `SELECT CONCAT(@@datadir, "test_loadfile.txt")`
# Create a text file with data to import into the table.
# PK and SK are not in any order
--let ROCKSDB_INFILE = $file
perl;
my $fn = $ENV{'ROCKSDB_INFILE'};
open(my $fh, '>', $fn) || die "perl open($fn): $!";
my $max = 5000000;
my $sign = 1;
for (my $ii = 0; $ii < $max; $ii++)
{
my $a = 1 + $sign * $ii;
my $b = 1 - $sign * $ii;
$sign = -$sign;
print $fh "$a\t$b\n";
}
close($fh);
EOF
--file_exists $file

# Make sure a snapshot held by another user doesn't block the bulk load
connect (other,localhost,root,,);
set session transaction isolation level repeatable read;
select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';
start transaction with consistent snapshot;
select * from information_schema.rocksdb_dbstats where stat_type='DB_NUM_SNAPSHOTS';

connection default;
set rocksdb_bulk_load=1;
set rocksdb_bulk_load_size=100000;
--disable_query_log
--echo LOAD DATA INFILE <input_file> INTO TABLE t1;
eval LOAD DATA INFILE '$file' INTO TABLE t1;
--echo LOAD DATA INFILE <input_file> INTO TABLE t2;
eval LOAD DATA INFILE '$file' INTO TABLE t2;
--echo LOAD DATA INFILE <input_file> INTO TABLE t3;
eval LOAD DATA INFILE '$file' INTO TABLE t3;
--enable_query_log
set rocksdb_bulk_load=0;

--remove_file $file

# Make sure row count index stats are correct
--replace_column 6 # 7 # 8 # 9 #
SHOW TABLE STATUS WHERE name LIKE 't%';

ANALYZE TABLE t1, t2, t3;

--replace_column 6 # 7 # 8 # 9 #
SHOW TABLE STATUS WHERE name LIKE 't%';

# Make sure all the data is there.
select count(a) from t1;
select count(b) from t1;
select count(a) from t2;
select count(b) from t2;
select count(a) from t3;
select count(b) from t3;

SELECT * FROM t1 FORCE INDEX (PRIMARY) LIMIT 3;
SELECT * FROM t2 FORCE INDEX (PRIMARY) LIMIT 3;

DROP TABLE t1, t2, t3;
SET rocksdb_bulk_load_allow_unsorted=0;
Loading

0 comments on commit 352107c

Please sign in to comment.