From 10e80f772273e893b7999cd2f9541096202fe3d9 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Thu, 21 Mar 2019 16:58:35 -0700 Subject: [PATCH] FB8-187: New variable binlog_rows_event_max_rows to control the number of rows in a single rows event (#974) (#974) Summary: Jira issue: https://jira.percona.com/browse/FB8-187 Reference Patch: https://github.com/facebook/mysql-5.6/commit/dfe6746cdfd Number of rows in a single event can be indirectly controlled with binlog_rows_event_max_size. This new variable will give more direct control to the user. Originally Reviewed By: anirbanr-fb Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/974 Reviewed By: lloyd Differential Revision: D14421324 Pulled By: lth --- mysql-test/r/mysqld--help-notwin.result | 3 ++ .../r/binlog_rows_event_max_rows.result | 30 +++++++++++++ .../binlog/t/binlog_rows_event_max_rows.test | 30 +++++++++++++ .../r/binlog_persist_only_variables.result | 2 + .../r/binlog_persist_variables.result | 2 + .../r/binlog_rows_event_max_rows_basic.result | 45 +++++++++++++++++++ .../t/binlog_rows_event_max_rows_basic.test | 33 ++++++++++++++ sql/binlog.cc | 1 + sql/log_event.h | 2 +- sql/mysqld.cc | 1 + sql/mysqld.h | 2 + sql/sys_vars.cc | 6 +++ 12 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/binlog/r/binlog_rows_event_max_rows.result create mode 100644 mysql-test/suite/binlog/t/binlog_rows_event_max_rows.test create mode 100644 mysql-test/suite/sys_vars/r/binlog_rows_event_max_rows_basic.result create mode 100644 mysql-test/suite/sys_vars/t/binlog_rows_event_max_rows_basic.test diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 524d1782191e..7e9b00ae190b 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -223,6 +223,8 @@ The following options may be given as the first argument: such updates, only the modified parts of the JSON document are included in the binary log, so small changes of big documents may need significantly less space. + --binlog-rows-event-max-rows[=#] + Max number of rows in a single rows event --binlog-rows-query-log-events Allow writing of Rows_query_log events into binary log. --binlog-stmt-cache-size=# @@ -2415,6 +2417,7 @@ binlog-row-event-max-size 8192 binlog-row-image FULL binlog-row-metadata FACEBOOK binlog-row-value-options +binlog-rows-event-max-rows 18446744073709551615 binlog-rows-query-log-events FALSE binlog-stmt-cache-size 32768 binlog-transaction-compression FALSE diff --git a/mysql-test/suite/binlog/r/binlog_rows_event_max_rows.result b/mysql-test/suite/binlog/r/binlog_rows_event_max_rows.result new file mode 100644 index 000000000000..c34657bcf10d --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_rows_event_max_rows.result @@ -0,0 +1,30 @@ +RESET MASTER; +SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows; +SET @@global.binlog_rows_event_max_rows= 1; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3); +UPDATE t1 SET a = 10; +SET @@global.binlog_rows_event_max_rows= 2; +DELETE FROM t1; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +binlog.000001 # Query # # use `test`; CREATE TABLE t1 (a INT) +binlog.000001 # Query # # BEGIN +binlog.000001 # Table_map # # table_id: # (test.t1) +binlog.000001 # Write_rows # # table_id: # +binlog.000001 # Write_rows # # table_id: # +binlog.000001 # Write_rows # # table_id: # flags: STMT_END_F +binlog.000001 # Xid # # COMMIT /* XID */ +binlog.000001 # Query # # BEGIN +binlog.000001 # Table_map # # table_id: # (test.t1) +binlog.000001 # Update_rows # # table_id: # +binlog.000001 # Update_rows # # table_id: # +binlog.000001 # Update_rows # # table_id: # flags: STMT_END_F +binlog.000001 # Xid # # COMMIT /* XID */ +binlog.000001 # Query # # BEGIN +binlog.000001 # Table_map # # table_id: # (test.t1) +binlog.000001 # Delete_rows # # table_id: # +binlog.000001 # Delete_rows # # table_id: # flags: STMT_END_F +binlog.000001 # Xid # # COMMIT /* XID */ +DROP TABLE t1; +SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows; diff --git a/mysql-test/suite/binlog/t/binlog_rows_event_max_rows.test b/mysql-test/suite/binlog/t/binlog_rows_event_max_rows.test new file mode 100644 index 000000000000..2b1685a30bc9 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_rows_event_max_rows.test @@ -0,0 +1,30 @@ +source include/have_binlog_format_row.inc; + +# Deletes all the binary logs +RESET MASTER; + +SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows; + +# setup +let $MYSQLD_DATADIR = `select @@datadir`; +let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; +SET @@global.binlog_rows_event_max_rows= 1; +CREATE TABLE t1 (a INT); + +# each insert should be in a separate row event because binlog_rows_event_max_rows = 1 +INSERT INTO t1 VALUES (1), (2), (3); + +# each update should be in a separate row event because binlog_rows_event_max_rows = 1 +UPDATE t1 SET a = 10; + +# there should be two delete events, 1st one with 2 rows and 2nd one with one +# row because binlog_rows_event_max_rows = 2 +SET @@global.binlog_rows_event_max_rows= 2; +DELETE FROM t1; + +--source include/show_binlog_events.inc + +# cleanup +DROP TABLE t1; + +SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result index 8d562d14f474..45bf7223b4fd 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result @@ -79,6 +79,7 @@ SET PERSIST_ONLY binlog_row_event_max_size = @@GLOBAL.binlog_row_event_max_size; SET PERSIST_ONLY binlog_row_image = @@GLOBAL.binlog_row_image; SET PERSIST_ONLY binlog_row_metadata = @@GLOBAL.binlog_row_metadata; SET PERSIST_ONLY binlog_row_value_options = @@GLOBAL.binlog_row_value_options; +SET PERSIST_ONLY binlog_rows_event_max_rows = @@GLOBAL.binlog_rows_event_max_rows; SET PERSIST_ONLY binlog_rows_query_log_events = @@GLOBAL.binlog_rows_query_log_events; SET PERSIST_ONLY binlog_stmt_cache_size = @@GLOBAL.binlog_stmt_cache_size; SET PERSIST_ONLY binlog_transaction_compression = @@GLOBAL.binlog_transaction_compression; @@ -330,6 +331,7 @@ RESET PERSIST binlog_row_event_max_size; RESET PERSIST binlog_row_image; RESET PERSIST binlog_row_metadata; RESET PERSIST binlog_row_value_options; +RESET PERSIST binlog_rows_event_max_rows; RESET PERSIST binlog_rows_query_log_events; RESET PERSIST binlog_stmt_cache_size; RESET PERSIST binlog_transaction_compression; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result index 799cef96f527..25289b3a02f4 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result @@ -61,6 +61,7 @@ ERROR HY000: Variable 'binlog_row_event_max_size' is a read only variable SET PERSIST binlog_row_image = @@GLOBAL.binlog_row_image; SET PERSIST binlog_row_metadata = @@GLOBAL.binlog_row_metadata; SET PERSIST binlog_row_value_options = @@GLOBAL.binlog_row_value_options; +SET PERSIST binlog_rows_event_max_rows = @@GLOBAL.binlog_rows_event_max_rows; SET PERSIST binlog_rows_query_log_events = @@GLOBAL.binlog_rows_query_log_events; SET PERSIST binlog_stmt_cache_size = @@GLOBAL.binlog_stmt_cache_size; SET PERSIST binlog_transaction_compression = @@GLOBAL.binlog_transaction_compression; @@ -313,6 +314,7 @@ Warning 3615 Variable binlog_row_event_max_size does not exist in persisted conf RESET PERSIST IF EXISTS binlog_row_image; RESET PERSIST IF EXISTS binlog_row_metadata; RESET PERSIST IF EXISTS binlog_row_value_options; +RESET PERSIST IF EXISTS binlog_rows_event_max_rows; RESET PERSIST IF EXISTS binlog_rows_query_log_events; RESET PERSIST IF EXISTS binlog_stmt_cache_size; RESET PERSIST IF EXISTS binlog_transaction_compression; diff --git a/mysql-test/suite/sys_vars/r/binlog_rows_event_max_rows_basic.result b/mysql-test/suite/sys_vars/r/binlog_rows_event_max_rows_basic.result new file mode 100644 index 000000000000..9a7e725a7a5e --- /dev/null +++ b/mysql-test/suite/sys_vars/r/binlog_rows_event_max_rows_basic.result @@ -0,0 +1,45 @@ +SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows; +SELECT @old_binlog_rows_event_max_rows; +@old_binlog_rows_event_max_rows +18446744073709551615 +SET @@global.binlog_rows_event_max_rows = DEFAULT; +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +18446744073709551615 +# binlog_rows_event_max_rows is a global variable. +SET @@session.binlog_rows_event_max_rows = 1; +ERROR HY000: Variable 'binlog_rows_event_max_rows' is a GLOBAL variable and should be set with SET GLOBAL +SELECT @@binlog_rows_event_max_rows; +@@binlog_rows_event_max_rows +18446744073709551615 +SET @@global.binlog_rows_event_max_rows = 512; +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +512 +SET @@global.binlog_rows_event_max_rows = 1000000; +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +1000000 +SET @@global.binlog_rows_event_max_rows = 1.01; +ERROR 42000: Incorrect argument type to variable 'binlog_rows_event_max_rows' +SET @@global.binlog_rows_event_max_rows = 'ten'; +ERROR 42000: Incorrect argument type to variable 'binlog_rows_event_max_rows' +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +1000000 +# set binlog_rows_event_max_rows to wrong value +SET @@global.binlog_rows_event_max_rows = 1500000; +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +1500000 +# set binlog_rows_event_max_rows to wrong value +SET @@global.binlog_rows_event_max_rows = 0; +Warnings: +Warning 1292 Truncated incorrect binlog_rows_event_max_rows value: '0' +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +1 +SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows; +SELECT @@global.binlog_rows_event_max_rows; +@@global.binlog_rows_event_max_rows +18446744073709551615 diff --git a/mysql-test/suite/sys_vars/t/binlog_rows_event_max_rows_basic.test b/mysql-test/suite/sys_vars/t/binlog_rows_event_max_rows_basic.test new file mode 100644 index 000000000000..717d696775b9 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/binlog_rows_event_max_rows_basic.test @@ -0,0 +1,33 @@ +--source include/load_sysvars.inc + +SET @old_binlog_rows_event_max_rows = @@global.binlog_rows_event_max_rows; +SELECT @old_binlog_rows_event_max_rows; + +SET @@global.binlog_rows_event_max_rows = DEFAULT; +SELECT @@global.binlog_rows_event_max_rows; + +-- echo # binlog_rows_event_max_rows is a global variable. +--error ER_GLOBAL_VARIABLE +SET @@session.binlog_rows_event_max_rows = 1; +SELECT @@binlog_rows_event_max_rows; + +SET @@global.binlog_rows_event_max_rows = 512; +SELECT @@global.binlog_rows_event_max_rows; +SET @@global.binlog_rows_event_max_rows = 1000000; +SELECT @@global.binlog_rows_event_max_rows; + +--error ER_WRONG_TYPE_FOR_VAR +SET @@global.binlog_rows_event_max_rows = 1.01; +--error ER_WRONG_TYPE_FOR_VAR +SET @@global.binlog_rows_event_max_rows = 'ten'; +SELECT @@global.binlog_rows_event_max_rows; +-- echo # set binlog_rows_event_max_rows to wrong value +SET @@global.binlog_rows_event_max_rows = 1500000; +SELECT @@global.binlog_rows_event_max_rows; +-- echo # set binlog_rows_event_max_rows to wrong value +SET @@global.binlog_rows_event_max_rows = 0; +SELECT @@global.binlog_rows_event_max_rows; + + +SET @@global.binlog_rows_event_max_rows = @old_binlog_rows_event_max_rows; +SELECT @@global.binlog_rows_event_max_rows; diff --git a/sql/binlog.cc b/sql/binlog.cc index e78820ad6ac5..3dcb3d5f8c1f 100644 --- a/sql/binlog.cc +++ b/sql/binlog.cc @@ -11539,6 +11539,7 @@ Rows_log_event *THD::binlog_prepare_pending_rows_event( pending->get_table_id() != table->s->table_map_id || pending->get_general_type_code() != general_type_code || pending->get_data_size() + needed > binlog_row_event_max_size || + pending->m_row_count >= opt_binlog_rows_event_max_rows || pending->read_write_bitmaps_cmp(table) == false || !(pending->m_extra_row_info.compare_extra_row_info( extra_row_info, part_id, source_part_id))) { diff --git a/sql/log_event.h b/sql/log_event.h index 0c2aee9a2056..4f29ebc8c046 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -2790,7 +2790,7 @@ class Rows_log_event : public virtual binary_log::Rows_event, public Log_event { bool old_row_is_record1); #endif - uint m_row_count; /* The number of rows added to the event */ + ulonglong m_row_count; /* The number of rows added to the event */ protected: /* diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0bcb3f969a34..0cc13b2d0e33 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1352,6 +1352,7 @@ restart server again with old database files. */ bool clone_recovery_error = false; ulong binlog_row_event_max_size; +ulonglong opt_binlog_rows_event_max_rows = 0; bool opt_log_only_query_comments = false; bool opt_binlog_trx_meta_data = false; bool opt_log_column_names = false; diff --git a/sql/mysqld.h b/sql/mysqld.h index 70b9909cd4d0..c7279e458076 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -367,7 +367,9 @@ extern long opt_binlog_group_commit_sync_delay; extern ulong opt_binlog_group_commit_sync_no_delay_count; extern ulong max_binlog_size, max_relay_log_size; extern ulong replica_max_allowed_packet; +extern ulonglong opt_binlog_rows_event_max_rows; extern ulong binlog_row_event_max_size; +extern ulong opt_binlog_rows_event_max_size; extern ulong binlog_checksum_options; extern ulong binlog_row_metadata; extern const char *binlog_checksum_type_names[]; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d7c1a23d512d..92ed619376bd 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1237,6 +1237,12 @@ static Sys_var_ulong Sys_binlog_group_commit_sync_no_delay_count( CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 100000 /* max connections */), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG); +static Sys_var_ulonglong Sys_binlog_rows_event_max_rows( + "binlog_rows_event_max_rows", "Max number of rows in a single rows event", + GLOBAL_VAR(opt_binlog_rows_event_max_rows), CMD_LINE(OPT_ARG), + VALID_RANGE(1, ULLONG_MAX), DEFAULT(ULLONG_MAX), BLOCK_SIZE(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(nullptr)); + static bool check_outside_trx(sys_var *var, THD *thd, set_var *) { if (thd->in_active_multi_stmt_transaction()) { my_error(ER_VARIABLE_NOT_SETTABLE_IN_TRANSACTION, MYF(0), var->name.str);