Add a new option to make MySQL server truncate binlogs during crash r…

…ecovery. (facebook#1061)

Summary:
This diff adds a new option to trim binlog during recovery. The following happens if this option is set:
  1. All prepared trasnactions in the engine are rolled back during engine recovery
  2. The binlog file (which was marked 'IN USE') will be truncated to the position as reported by engine as the last commited gtid position in the binlog

Reference Patch: facebook@96308865f3c
Reference Patch: facebook@6288eda9d6e
Reference Patch: facebook@12937e8de06
Reference Patch: facebook@fe3826a6743

Originally Reviewed By: abhinav04sharma

Pull Request resolved: facebook#1061

Test Plan: Build mysql

Reviewed By: luqun

Differential Revision: D18812324

Pulled By: bhatvinay

mysql-test/include/have_rocksdb.inc

@@ -0,0 +1,11 @@
+if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'rocksdb' AND support IN ('DEFAULT')`)
+{
+  --skip Test requires default engine RocksDB
+}
+
+--disable_query_log
+# Table statistics can vary depending on when the memtables are flushed, so
+# flush them at the beginning of the test to ensure the test runs consistently.
+set global rocksdb_force_flush_memtable_now = true;
+--enable_query_log
+

mysql-test/r/mysqld--help-notwin.result

@@ -1584,6 +1584,10 @@ The following options may be given as the first argument:
  This option is used to let the server know when to
  extract the write set which will be used for various
  purposes. 
+ --trim-binlog-to-recover 
+ Trim the last binlog (if required) to the position until
+ which the engine has successfully committed all
+ transactions.
  --updatable-views-with-limit=name 
  YES = Don't issue an error message (warning only) if a
  VIEW without presence of a key of the underlying table is
@@ -2031,6 +2035,7 @@ transaction-isolation REPEATABLE-READ
 transaction-prealloc-size 4096
 transaction-read-only FALSE
 transaction-write-set-extraction XXHASH64
+trim-binlog-to-recover FALSE
 updatable-views-with-limit YES
 upgrade AUTO
 validate-config FALSE

mysql-test/suite/binlog/r/binlog_rotate_block_on_trxs_rbr.result

@@ -0,0 +1,49 @@
+CALL mtr.add_suppression("Timeout waiting for reply of binlog *");
+SET @save.rpl_semi_sync_master_timeout = @@global.rpl_semi_sync_master_timeout;
+SET @save.rpl_semi_sync_master_enabled = @@global.rpl_semi_sync_master_enabled;
+[connection default]
+CREATE TABLE t1(c1 INT PRIMARY KEY);
+CREATE TABLE blackhole(c1 INT PRIMARY KEY) ENGINE=blackhole;
+INSERT INTO t1 VALUES(1);
+COMMIT;
+# Create a 20 sec semisync timeout
+SET global rpl_semi_sync_master_timeout=20000;
+SET global rpl_semi_sync_master_enabled=1;
+[connection conn1]
+INSERT INTO blackhole VALUES (1);;
+[connection conn2]
+FLUSH LOGS;
+[connection conn2]
+[connection conn1]
+[connection default]
+INSERT INTO t1 VALUES (2);
+# The first binlog file should contain the trx on blackhole table since
+# the flush was blocked by the trx. The insert (of value 2) into t1
+# should be in the rotated file (second binlog file)
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (mtr.test_suppressions)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE t1(c1 INT PRIMARY KEY)
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE blackhole(c1 INT PRIMARY KEY) ENGINE=blackhole
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (test.blackhole)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000002	#	Query	#	#	BEGIN
+master-bin.000002	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000002	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000002	#	Xid	#	#	COMMIT /* XID */
+DROP TABLE t1;
+DROP TABLE blackhole;
+SET @@global.rpl_semi_sync_master_timeout = @save.rpl_semi_sync_master_timeout;
+SET @@global.rpl_semi_sync_master_enabled = @save.rpl_semi_sync_master_enabled;

mysql-test/suite/binlog/r/binlog_rotate_block_on_trxs_sbr.result

@@ -0,0 +1,45 @@
+CALL mtr.add_suppression("Timeout waiting for reply of binlog *");
+SET @save.rpl_semi_sync_master_timeout = @@global.rpl_semi_sync_master_timeout;
+SET @save.rpl_semi_sync_master_enabled = @@global.rpl_semi_sync_master_enabled;
+[connection default]
+CREATE TABLE t1(c1 INT PRIMARY KEY);
+CREATE TABLE blackhole(c1 INT PRIMARY KEY) ENGINE=blackhole;
+INSERT INTO t1 VALUES(1);
+COMMIT;
+# Create a 20 sec semisync timeout
+SET global rpl_semi_sync_master_timeout=20000;
+SET global rpl_semi_sync_master_enabled=1;
+[connection conn1]
+INSERT INTO blackhole VALUES (1);;
+[connection conn2]
+FLUSH LOGS;
+[connection conn2]
+[connection conn1]
+[connection default]
+INSERT INTO t1 VALUES (2);
+# The first binlog file should contain the trx on blackhole table since
+# the flush was blocked by the trx. The insert (of value 2) into t1
+# should be in the rotated file (second binlog file)
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'Timeout waiting for reply of binlog *' COLLATE 'utf8mb4_0900_ai_ci'))
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE t1(c1 INT PRIMARY KEY)
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE blackhole(c1 INT PRIMARY KEY) ENGINE=blackhole
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(1)
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO blackhole VALUES (1)
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000002	#	Query	#	#	BEGIN
+master-bin.000002	#	Query	#	#	use `test`; INSERT INTO t1 VALUES (2)
+master-bin.000002	#	Xid	#	#	COMMIT /* XID */
+DROP TABLE t1;
+DROP TABLE blackhole;
+SET @@global.rpl_semi_sync_master_timeout = @save.rpl_semi_sync_master_timeout;
+SET @@global.rpl_semi_sync_master_enabled = @save.rpl_semi_sync_master_enabled;

mysql-test/suite/binlog/r/binlog_trim_option_only.result

@@ -0,0 +1,14 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+CREATE TABLE t1 (pk int primary key);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+INSERT INTO t1 VALUES (4);
+INSERT INTO t1 VALUES (5);
+DROP TABLE t1;
+include/sync_slave_sql_with_master.inc
+include/rpl_end.inc

mysql-test/suite/binlog/r/binlog_truncate_across_flush_rbr.result

@@ -0,0 +1,91 @@
+CALL mtr.add_suppression("Taking backup from .*");
+CREATE TABLE t1(c1 INT);
+CREATE TABLE blackhole (c1 INT PRIMARY KEY) ENGINE=BLACKHOLE;
+INSERT INTO t1 VALUES(1);
+INSERT INTO blackhole VALUES(1);
+COMMIT;
+INSERT INTO t1 VALUES(2);
+COMMIT;
+FLUSH LOGS;
+# Crash right after flushing binary log
+SET SESSION DEBUG="+d,crash_after_flush_binlog";
+BEGIN;
+INSERT INTO t1 VALUES(3);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+#
+# Verify that a transaction cannot be recovered during server
+# recovery from a crash, which happened after flushing it
+# to binary log. This is because the transaction is still marked
+# as prepared in engine and will be rollbacked when
+# trim-binlog-to-recover is set
+#
+include/assert.inc [There should be 2 rows in table t1]
+INSERT INTO t1 VALUES(4);
+COMMIT;
+FLUSH ENGINE LOGS;
+#
+# verify that the latest binlog file is trimmed to the starting position
+# of the first gtid event
+#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (mtr.test_suppressions)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE t1(c1 INT)
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE blackhole (c1 INT PRIMARY KEY) ENGINE=BLACKHOLE
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (test.blackhole)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000003	#	Query	#	#	BEGIN
+master-bin.000003	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000003	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000003	#	Xid	#	#	COMMIT /* XID */
+master-bin.000003	#	Query	#	#	use `test`; FLUSH ENGINE LOGS
+# Crash right after flushing binary log
+SET SESSION DEBUG="+d,crash_after_flush_binlog";
+BEGIN;
+INSERT INTO blackhole VALUES(2);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+INSERT INTO t1 VALUES(5);
+COMMIT;
+FLUSH ENGINE LOGS;
+include/assert.inc [There should be 4 rows in table t1]
+#
+# verify that the latest binlog file is trimmed to the starting position
+# of the first gtid event
+#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000003	#	Query	#	#	BEGIN
+master-bin.000003	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000003	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000003	#	Xid	#	#	COMMIT /* XID */
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000004	#	Query	#	#	BEGIN
+master-bin.000004	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000004	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+master-bin.000004	#	Xid	#	#	COMMIT /* XID */
+master-bin.000004	#	Query	#	#	use `test`; FLUSH ENGINE LOGS
+DROP TABLE t1;
+DROP TABLE blackhole;

mysql-test/suite/binlog/r/binlog_truncate_across_flush_sbr.result

@@ -0,0 +1,84 @@
+CALL mtr.add_suppression("Taking backup from .*");
+CREATE TABLE t1(c1 INT);
+CREATE TABLE blackhole (c1 INT PRIMARY KEY) ENGINE=BLACKHOLE;
+INSERT INTO t1 VALUES(1);
+INSERT INTO blackhole VALUES(1);
+COMMIT;
+INSERT INTO t1 VALUES(2);
+COMMIT;
+FLUSH LOGS;
+# Crash right after flushing binary log
+SET SESSION DEBUG="+d,crash_after_flush_binlog";
+BEGIN;
+INSERT INTO t1 VALUES(3);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+#
+# Verify that a transaction cannot be recovered during server
+# recovery from a crash, which happened after flushing it
+# to binary log. This is because the transaction is still marked
+# as prepared in engine and will be rollbacked when
+# trim-binlog-to-recover is set
+#
+include/assert.inc [There should be 2 rows in table t1]
+INSERT INTO t1 VALUES(4);
+COMMIT;
+FLUSH ENGINE LOGS;
+#
+# verify that the latest binlog file is trimmed to the starting position
+# of the first gtid event
+#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `mtr`; INSERT INTO test_suppressions (pattern) VALUES ( NAME_CONST('pattern',_utf8mb4'Taking backup from .*' COLLATE 'utf8mb4_0900_ai_ci'))
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE t1(c1 INT)
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE blackhole (c1 INT PRIMARY KEY) ENGINE=BLACKHOLE
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(1)
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO blackhole VALUES(1)
+master-bin.000001	#	Query	#	#	COMMIT
+master-bin.000001	#	Query	#	#	BEGIN
+master-bin.000001	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(2)
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000003	#	Query	#	#	BEGIN
+master-bin.000003	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(4)
+master-bin.000003	#	Xid	#	#	COMMIT /* XID */
+master-bin.000003	#	Query	#	#	use `test`; FLUSH ENGINE LOGS
+# Crash right after flushing binary log
+SET SESSION DEBUG="+d,crash_after_flush_binlog";
+BEGIN;
+INSERT INTO blackhole VALUES(2);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+INSERT INTO t1 VALUES(5);
+COMMIT;
+FLUSH ENGINE LOGS;
+include/assert.inc [There should be 4 rows in table t1]
+#
+# verify that the latest binlog file is trimmed to the starting position
+# of the first gtid event
+#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000003	#	Query	#	#	BEGIN
+master-bin.000003	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(4)
+master-bin.000003	#	Xid	#	#	COMMIT /* XID */
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000004	#	Query	#	#	BEGIN
+master-bin.000004	#	Query	#	#	use `test`; INSERT INTO t1 VALUES(5)
+master-bin.000004	#	Xid	#	#	COMMIT /* XID */
+master-bin.000004	#	Query	#	#	use `test`; FLUSH ENGINE LOGS
+DROP TABLE t1;
+DROP TABLE blackhole;

mysql-test/suite/binlog/r/binlog_truncate_crash_recovery.result

@@ -0,0 +1,38 @@
+CALL mtr.add_suppression("Taking backup from .*");
+CREATE TABLE t1(c1 INT);
+INSERT INTO t1 VALUES(100);
+COMMIT;
+FLUSH LOGS;
+# Crash right after flushing engine log
+SET SESSION DEBUG="+d,crash_after_flush_engine_log";
+BEGIN;
+INSERT INTO t1 VALUES(1);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+#
+# Verify that a transaction can not be recovered during server
+# recovery from a crash, which happened after flushing it to
+# engine log and before flushing it to binary log.
+#
+SELECT count(*) FROM t1 WHERE c1=1;
+count(*)
+0
+# Crash right after flushing binary log
+SET SESSION DEBUG="+d,crash_after_flush_binlog";
+BEGIN;
+INSERT INTO t1 VALUES(2);
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# Restart the master server
+#
+# Verify that a transaction cannot be recovered during server
+# recovery from a crash, which happened after flushing it
+# to binary log. This is because the transaction is still marked
+# as prepared in engine and will be rollbacked when
+# trim-binlog-to-recover is set
+#
+SELECT count(*) FROM t1 WHERE c1=2;
+count(*)
+0
+DROP TABLE t1;