FB8-187: New variable binlog_rows_event_max_rows to control the numbe…

…r of rows in a single rows event (facebook#974) (facebook#974)

Summary:
Jira issue: https://jira.percona.com/browse/FB8-187

Reference Patch: facebook@dfe6746cdfd
Number of rows in a single event can be indirectly controlled with
binlog_rows_event_max_size. This new variable will give more direct control to
the user.

Originally Reviewed By: anirbanr-fb

Pull Request resolved: facebook#974

Reviewed By: lloyd

Differential Revision: D14421324

Pulled By: lth

mysql-test/r/mysqld--help-notwin.result

@@ -218,6 +218,8 @@ The following options may be given as the first argument:
  such updates, only the modified parts of the JSON
  document are included in the binary log, so small changes
  of big documents may need significantly less space.
+ --binlog-rows-event-max-rows[=#] 
+ Max number of rows in a single rows event
  --binlog-rows-query-log-events 
  Allow writing of Rows_query_log events into binary log.
  --binlog-stmt-cache-size=# 
@@ -2382,6 +2384,7 @@ binlog-row-event-max-size 8192
 binlog-row-image FULL
 binlog-row-metadata FACEBOOK
 binlog-row-value-options 
+binlog-rows-event-max-rows 18446744073709551615
 binlog-rows-query-log-events FALSE
 binlog-stmt-cache-size 32768
 binlog-transaction-compression FALSE

mysql-test/suite/binlog/r/binlog_rows_event_max_rows.result

@@ -0,0 +1,30 @@
+RESET BINARY LOGS AND GTIDS;
+SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows;
+SET @@global.binlog_rows_event_max_rows= 1;
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1), (2), (3);
+UPDATE t1 SET a = 10;
+SET @@global.binlog_rows_event_max_rows= 2;
+DELETE FROM t1;
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+binlog.000001	#	Query	#	#	use `test`; CREATE TABLE t1 (a INT)
+binlog.000001	#	Query	#	#	BEGIN
+binlog.000001	#	Table_map	#	#	table_id: # (test.t1)
+binlog.000001	#	Write_rows	#	#	table_id: #
+binlog.000001	#	Write_rows	#	#	table_id: #
+binlog.000001	#	Write_rows	#	#	table_id: # flags: STMT_END_F
+binlog.000001	#	Xid	#	#	COMMIT /* XID */
+binlog.000001	#	Query	#	#	BEGIN
+binlog.000001	#	Table_map	#	#	table_id: # (test.t1)
+binlog.000001	#	Update_rows	#	#	table_id: #
+binlog.000001	#	Update_rows	#	#	table_id: #
+binlog.000001	#	Update_rows	#	#	table_id: # flags: STMT_END_F
+binlog.000001	#	Xid	#	#	COMMIT /* XID */
+binlog.000001	#	Query	#	#	BEGIN
+binlog.000001	#	Table_map	#	#	table_id: # (test.t1)
+binlog.000001	#	Delete_rows	#	#	table_id: #
+binlog.000001	#	Delete_rows	#	#	table_id: # flags: STMT_END_F
+binlog.000001	#	Xid	#	#	COMMIT /* XID */
+DROP TABLE t1;
+SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows;

mysql-test/suite/binlog/t/binlog_rows_event_max_rows.test

@@ -0,0 +1,30 @@
+source include/have_binlog_format_row.inc;
+
+# Deletes all the binary logs
+RESET BINARY LOGS AND GTIDS;
+
+SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows;
+
+# setup
+let $MYSQLD_DATADIR = `select @@datadir`;
+let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`;
+SET @@global.binlog_rows_event_max_rows= 1;
+CREATE TABLE t1 (a INT);
+
+# each insert should be in a separate row event because binlog_rows_event_max_rows = 1
+INSERT INTO t1 VALUES (1), (2), (3);
+
+# each update should be in a separate row event because binlog_rows_event_max_rows = 1
+UPDATE t1 SET a = 10;
+
+# there should be two delete events, 1st one with 2 rows and 2nd one with one
+# row because binlog_rows_event_max_rows = 2
+SET @@global.binlog_rows_event_max_rows= 2;
+DELETE FROM t1;
+
+--source include/show_binlog_events.inc
+
+# cleanup
+DROP TABLE t1;
+
+SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows;

mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result

@@ -79,6 +79,7 @@ SET PERSIST_ONLY binlog_row_event_max_size = @@GLOBAL.binlog_row_event_max_size;
 SET PERSIST_ONLY binlog_row_image = @@GLOBAL.binlog_row_image;
 SET PERSIST_ONLY binlog_row_metadata = @@GLOBAL.binlog_row_metadata;
 SET PERSIST_ONLY binlog_row_value_options = @@GLOBAL.binlog_row_value_options;
+SET PERSIST_ONLY binlog_rows_event_max_rows = @@GLOBAL.binlog_rows_event_max_rows;
 SET PERSIST_ONLY binlog_rows_query_log_events = @@GLOBAL.binlog_rows_query_log_events;
 SET PERSIST_ONLY binlog_stmt_cache_size = @@GLOBAL.binlog_stmt_cache_size;
 SET PERSIST_ONLY binlog_transaction_compression = @@GLOBAL.binlog_transaction_compression;
@@ -314,6 +315,7 @@ RESET PERSIST binlog_row_event_max_size;
 RESET PERSIST binlog_row_image;
 RESET PERSIST binlog_row_metadata;
 RESET PERSIST binlog_row_value_options;
+RESET PERSIST binlog_rows_event_max_rows;
 RESET PERSIST binlog_rows_query_log_events;
 RESET PERSIST binlog_stmt_cache_size;
 RESET PERSIST binlog_transaction_compression;

mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result

@@ -61,6 +61,7 @@ ERROR HY000: Variable 'binlog_row_event_max_size' is a read only variable
 SET PERSIST binlog_row_image = @@GLOBAL.binlog_row_image;
 SET PERSIST binlog_row_metadata = @@GLOBAL.binlog_row_metadata;
 SET PERSIST binlog_row_value_options = @@GLOBAL.binlog_row_value_options;
+SET PERSIST binlog_rows_event_max_rows = @@GLOBAL.binlog_rows_event_max_rows;
 SET PERSIST binlog_rows_query_log_events = @@GLOBAL.binlog_rows_query_log_events;
 SET PERSIST binlog_stmt_cache_size = @@GLOBAL.binlog_stmt_cache_size;
 SET PERSIST binlog_transaction_compression = @@GLOBAL.binlog_transaction_compression;
@@ -297,6 +298,7 @@ Warning	3615	Variable binlog_row_event_max_size does not exist in persisted conf
 RESET PERSIST IF EXISTS binlog_row_image;
 RESET PERSIST IF EXISTS binlog_row_metadata;
 RESET PERSIST IF EXISTS binlog_row_value_options;
+RESET PERSIST IF EXISTS binlog_rows_event_max_rows;
 RESET PERSIST IF EXISTS binlog_rows_query_log_events;
 RESET PERSIST IF EXISTS binlog_stmt_cache_size;
 RESET PERSIST IF EXISTS binlog_transaction_compression;

mysql-test/suite/sys_vars/r/binlog_rows_event_max_rows_basic.result

@@ -0,0 +1,45 @@
+SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows;
+SELECT @old_binlog_rows_event_max_rows;
+@old_binlog_rows_event_max_rows
+18446744073709551615
+SET @@global.binlog_rows_event_max_rows = DEFAULT;
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+18446744073709551615
+# binlog_rows_event_max_rows is a global variable.
+SET @@session.binlog_rows_event_max_rows = 1;
+ERROR HY000: Variable 'binlog_rows_event_max_rows' is a GLOBAL variable and should be set with SET GLOBAL
+SELECT @@binlog_rows_event_max_rows;
+@@binlog_rows_event_max_rows
+18446744073709551615
+SET @@global.binlog_rows_event_max_rows = 512;
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+512
+SET @@global.binlog_rows_event_max_rows = 1000000;
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+1000000
+SET @@global.binlog_rows_event_max_rows = 1.01;
+ERROR 42000: Incorrect argument type to variable 'binlog_rows_event_max_rows'
+SET @@global.binlog_rows_event_max_rows = 'ten';
+ERROR 42000: Incorrect argument type to variable 'binlog_rows_event_max_rows'
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+1000000
+# set binlog_rows_event_max_rows to wrong value
+SET @@global.binlog_rows_event_max_rows = 1500000;
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+1500000
+# set binlog_rows_event_max_rows to wrong value
+SET @@global.binlog_rows_event_max_rows = 0;
+Warnings:
+Warning	1292	Truncated incorrect binlog_rows_event_max_rows value: '0'
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+1
+SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows;
+SELECT @@global.binlog_rows_event_max_rows;
+@@global.binlog_rows_event_max_rows
+18446744073709551615

mysql-test/suite/sys_vars/t/binlog_rows_event_max_rows_basic.test

@@ -0,0 +1,33 @@
+--source include/load_sysvars.inc
+
+SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows;
+SELECT @old_binlog_rows_event_max_rows;
+
+SET @@global.binlog_rows_event_max_rows = DEFAULT;
+SELECT @@global.binlog_rows_event_max_rows;
+
+-- echo # binlog_rows_event_max_rows is a global variable.
+--error ER_GLOBAL_VARIABLE
+SET @@session.binlog_rows_event_max_rows = 1;
+SELECT @@binlog_rows_event_max_rows;
+
+SET @@global.binlog_rows_event_max_rows = 512;
+SELECT @@global.binlog_rows_event_max_rows;
+SET @@global.binlog_rows_event_max_rows = 1000000;
+SELECT @@global.binlog_rows_event_max_rows;
+
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@global.binlog_rows_event_max_rows = 1.01;
+--error ER_WRONG_TYPE_FOR_VAR
+SET @@global.binlog_rows_event_max_rows = 'ten';
+SELECT @@global.binlog_rows_event_max_rows;
+-- echo # set binlog_rows_event_max_rows to wrong value
+SET @@global.binlog_rows_event_max_rows = 1500000;
+SELECT @@global.binlog_rows_event_max_rows;
+-- echo # set binlog_rows_event_max_rows to wrong value
+SET @@global.binlog_rows_event_max_rows = 0;
+SELECT @@global.binlog_rows_event_max_rows;
+
+
+SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows;
+SELECT @@global.binlog_rows_event_max_rows;

sql/binlog.cc

@@ -11567,6 +11567,7 @@ Rows_log_event *THD::binlog_prepare_pending_rows_event(
       pending->get_table_id() != table->s->table_map_id ||
       pending->get_general_type_code() != general_type_code ||
       pending->get_data_size() + needed > binlog_row_event_max_size ||
+      pending->m_row_count >= opt_binlog_rows_event_max_rows ||
       pending->read_write_bitmaps_cmp(table) == false ||
       !(pending->m_extra_row_info.compare_extra_row_info(
           extra_row_info, part_id, source_part_id))) {

sql/log_event.h

@@ -2910,7 +2910,7 @@ class Rows_log_event : public virtual mysql::binlog::event::Rows_event,
                         bool old_row_is_record1);
 #endif
 
-  uint m_row_count; /* The number of rows added to the event */
+  ulonglong m_row_count; /* The number of rows added to the event */
 
  protected:
   /*

sql/mysqld.cc

@@ -1368,6 +1368,7 @@ restart server again with old database files. */
 bool clone_recovery_error = false;
 
 ulong binlog_row_event_max_size;
+ulonglong opt_binlog_rows_event_max_rows = 0;
 bool opt_log_only_query_comments = false;
 bool opt_binlog_trx_meta_data = false;
 bool opt_log_column_names = false;

sql/mysqld.h

@@ -353,7 +353,9 @@ extern long opt_binlog_group_commit_sync_delay;
 extern ulong opt_binlog_group_commit_sync_no_delay_count;
 extern ulong max_binlog_size, max_relay_log_size;
 extern ulong replica_max_allowed_packet;
+extern ulonglong opt_binlog_rows_event_max_rows;
 extern ulong binlog_row_event_max_size;
+extern ulong opt_binlog_rows_event_max_size;
 extern ulong binlog_checksum_options;
 extern ulong binlog_row_metadata;
 extern const char *binlog_checksum_type_names[];

sql/sys_vars.cc

@@ -1291,6 +1291,12 @@ static Sys_var_ulong Sys_binlog_group_commit_sync_no_delay_count(
     CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 100000 /* max connections */),
     DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG);
 
+static Sys_var_ulonglong Sys_binlog_rows_event_max_rows(
+    "binlog_rows_event_max_rows", "Max number of rows in a single rows event",
+    GLOBAL_VAR(opt_binlog_rows_event_max_rows), CMD_LINE(OPT_ARG),
+    VALID_RANGE(1, ULLONG_MAX), DEFAULT(ULLONG_MAX), BLOCK_SIZE(1),
+    NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(nullptr));
+
 static bool check_outside_trx(sys_var *var, THD *thd, set_var *) {
   if (thd->in_active_multi_stmt_transaction()) {
     my_error(ER_VARIABLE_NOT_SETTABLE_IN_TRANSACTION, MYF(0), var->name.str);