Add trx meta data in the binlog using comments in rows query event

Summary:
If enabled, For every trx before the commit event is written we write a
rows query event which contains meta data in JSON format wrapped inside a
comment.

Storing timestamps in the meta data to calculate slave lag in millisecond
granularity.

Reviewed By: tianx

Differential Revision: D5220355

fbshipit-source-id: a336f92

client/mysqlbinlog.cc

@@ -1510,6 +1510,13 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
       }
     }
     case ROWS_QUERY_LOG_EVENT:
+       // case: if events contains trx meta data print it else fall through
+       // like a normal rows query event
+       if (((Rows_query_log_event*) ev)->has_trx_meta_data())
+       {
+         ev->print(result_file, print_event_info);
+         break;
+       }
     case WRITE_ROWS_EVENT:
     case DELETE_ROWS_EVENT:
     case UPDATE_ROWS_EVENT:

mysql-test/r/mysqld--help-notwin-profiling.result

@@ -147,6 +147,10 @@ The following options may be given as the first argument:
  non-transactional engines for the binary log. If you
  often use statements updating a great number of rows, you
  can increase this to get more performance
+ --binlog-trx-meta-data 
+ Log meta data about every trx in the binary log. This
+ information is logged as a comment in a Rows_query_log
+ event in JSON format.
  --binlogging-impossible-mode=name 
  On a fatal error when statements cannot be binlogged the
  behaviour can be ignore the error and let the master
@@ -1728,6 +1732,7 @@ binlog-row-event-max-size 8192
 binlog-row-image FULL
 binlog-rows-query-log-events FALSE
 binlog-stmt-cache-size 32768
+binlog-trx-meta-data FALSE
 binlogging-impossible-mode IGNORE_ERROR
 block-create-memory FALSE
 block-create-myisam FALSE

mysql-test/r/mysqld--help-notwin.result

@@ -147,6 +147,10 @@ The following options may be given as the first argument:
  non-transactional engines for the binary log. If you
  often use statements updating a great number of rows, you
  can increase this to get more performance
+ --binlog-trx-meta-data 
+ Log meta data about every trx in the binary log. This
+ information is logged as a comment in a Rows_query_log
+ event in JSON format.
  --binlogging-impossible-mode=name 
  On a fatal error when statements cannot be binlogged the
  behaviour can be ignore the error and let the master
@@ -1726,6 +1730,7 @@ binlog-row-event-max-size 8192
 binlog-row-image FULL
 binlog-rows-query-log-events FALSE
 binlog-stmt-cache-size 32768
+binlog-trx-meta-data FALSE
 binlogging-impossible-mode IGNORE_ERROR
 block-create-memory FALSE
 block-create-myisam FALSE

mysql-test/suite/rpl/r/binlog_trx_meta_data.result

@@ -0,0 +1,16 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
+set @@global.binlog_trx_meta_data= true;
+set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
+set @@global.binlog_trx_meta_data= true;
+include/sync_slave_sql_with_master.inc
+include/sync_slave_sql_with_master.inc
+meta	count(*)
+# /*::TRX_META_DATA::{"timestamps":[ts]}*/	134
+meta	count(*)
+# /*::TRX_META_DATA::{"timestamps":[ts,ts]}*/	134
+include/rpl_end.inc

mysql-test/suite/rpl/t/binlog_trx_meta_data.test

@@ -0,0 +1,61 @@
+source include/master-slave.inc;
+source include/have_binlog_format_row.inc;
+
+# Enable trx meta data
+connection master;
+set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
+set @@global.binlog_trx_meta_data= true;
+
+connection slave;
+set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
+set @@global.binlog_trx_meta_data= true;
+
+# Generate load
+connection master;
+let $skip_cleanup= 1;
+let $sync_with_master= 1;
+let $databases=2;
+let $iter=5;
+source extra/rpl_tests/rpl_parallel_simple_load.test;
+enable_result_log;
+
+# Check binlogs for meta data
+connection master;
+let $MYSQLD_DATADIR = `select @@datadir`;
+let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`;
+exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/master-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
+set sql_log_bin=0;
+create table test.meta_data(meta text);
+eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data;
+select meta, count(*) from test.meta_data;
+drop table test.meta_data;
+set sql_log_bin=1;
+remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
+
+connection slave;
+let $MYSQLD_DATADIR = `select @@datadir`;
+let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`;
+exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/slave-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
+set sql_log_bin=0;
+create table test.meta_data(meta text);
+eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data;
+select meta, count(*) from test.meta_data;
+drop table test.meta_data;
+set sql_log_bin=1;
+remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
+
+# Consistency check
+connection master;
+let $include_silent=1;
+source extra/rpl_tests/rpl_parallel_simple_load_consistency.test;
+let $include_silent=;
+
+# Cleanup
+connection master;
+source extra/rpl_tests/rpl_parallel_simple_load_cleanup.test;
+connection slave;
+set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data;
+connection master;
+set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data;
+
+source include/rpl_end.inc;

mysql-test/suite/sys_vars/r/binlog_trx_meta_data_basic.result

@@ -0,0 +1,85 @@
+SET @start_value = @@global.binlog_trx_meta_data;
+SELECT @start_value;
+@start_value
+0
+SET @@global.binlog_trx_meta_data = DEFAULT;
+SELECT @@global.binlog_trx_meta_data = TRUE;
+@@global.binlog_trx_meta_data = TRUE
+0
+SET @@global.binlog_trx_meta_data = ON;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+1
+SET @@global.binlog_trx_meta_data = OFF;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+0
+SET @@global.binlog_trx_meta_data = 2;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '2'
+SET @@global.binlog_trx_meta_data = -1;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '-1'
+SET @@global.binlog_trx_meta_data = TRUEF;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'TRUEF'
+SET @@global.binlog_trx_meta_data = TRUE_F;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'TRUE_F'
+SET @@global.binlog_trx_meta_data = FALSE0;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'FALSE0'
+SET @@global.binlog_trx_meta_data = OON;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'OON'
+SET @@global.binlog_trx_meta_data = ONN;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'ONN'
+SET @@global.binlog_trx_meta_data = OOFF;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'OOFF'
+SET @@global.binlog_trx_meta_data = 0FF;
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '0FF'
+SET @@global.binlog_trx_meta_data = ' ';
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ' '
+SET @@global.binlog_trx_meta_data = " ";
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ' '
+SET @@global.binlog_trx_meta_data = '';
+ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ''
+SET @@session.binlog_trx_meta_data = OFF;
+ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable and should be set with SET GLOBAL
+SELECT @@session.binlog_trx_meta_data;
+ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable
+SELECT IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE 
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='binlog_trx_meta_data';
+IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE
+1
+SET @@global.binlog_trx_meta_data = 0;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+0
+SET @@global.binlog_trx_meta_data = 1;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+1
+SET @@global.binlog_trx_meta_data = TRUE;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+1
+SET @@global.binlog_trx_meta_data = FALSE;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+0
+SET @@global.binlog_trx_meta_data = ON;
+SELECT @@binlog_trx_meta_data = @@global.binlog_trx_meta_data;
+@@binlog_trx_meta_data = @@global.binlog_trx_meta_data
+1
+SET binlog_trx_meta_data = ON;
+ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable and should be set with SET GLOBAL
+SET local.binlog_trx_meta_data = OFF;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'binlog_trx_meta_data = OFF' at line 1
+SELECT local.binlog_trx_meta_data;
+ERROR 42S02: Unknown table 'local' in field list
+SET global.binlog_trx_meta_data = ON;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'binlog_trx_meta_data = ON' at line 1
+SELECT global.binlog_trx_meta_data;
+ERROR 42S02: Unknown table 'global' in field list
+SELECT binlog_trx_meta_data = @@session.binlog_trx_meta_data;
+ERROR 42S22: Unknown column 'binlog_trx_meta_data' in 'field list'
+SET @@global.binlog_trx_meta_data = @start_value;
+SELECT @@global.binlog_trx_meta_data;
+@@global.binlog_trx_meta_data
+0

mysql-test/suite/sys_vars/t/binlog_trx_meta_data_basic.test

@@ -0,0 +1,81 @@
+--source include/have_innodb.inc
+--source include/load_sysvars.inc
+
+SET @start_value = @@global.binlog_trx_meta_data;
+SELECT @start_value;
+
+
+SET @@global.binlog_trx_meta_data = DEFAULT;
+SELECT @@global.binlog_trx_meta_data = TRUE;
+
+
+SET @@global.binlog_trx_meta_data = ON;
+SELECT @@global.binlog_trx_meta_data;
+SET @@global.binlog_trx_meta_data = OFF;
+SELECT @@global.binlog_trx_meta_data;
+
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = 2;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = -1;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = TRUEF;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = TRUE_F;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = FALSE0;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = OON;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = ONN;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = OOFF;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = 0FF;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = ' ';
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = " ";
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@global.binlog_trx_meta_data = '';
+
+
+--Error ER_GLOBAL_VARIABLE
+SET @@session.binlog_trx_meta_data = OFF;
+--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.binlog_trx_meta_data;
+
+
+SELECT IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE 
+FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES 
+WHERE VARIABLE_NAME='binlog_trx_meta_data';
+
+
+SET @@global.binlog_trx_meta_data = 0;
+SELECT @@global.binlog_trx_meta_data;
+SET @@global.binlog_trx_meta_data = 1;
+SELECT @@global.binlog_trx_meta_data;
+
+SET @@global.binlog_trx_meta_data = TRUE;
+SELECT @@global.binlog_trx_meta_data;
+SET @@global.binlog_trx_meta_data = FALSE;
+SELECT @@global.binlog_trx_meta_data;
+
+SET @@global.binlog_trx_meta_data = ON;
+SELECT @@binlog_trx_meta_data = @@global.binlog_trx_meta_data;
+
+--Error ER_GLOBAL_VARIABLE
+SET binlog_trx_meta_data = ON;
+--Error ER_PARSE_ERROR
+SET local.binlog_trx_meta_data = OFF;
+--Error ER_UNKNOWN_TABLE
+SELECT local.binlog_trx_meta_data;
+--Error ER_PARSE_ERROR
+SET global.binlog_trx_meta_data = ON;
+--Error ER_UNKNOWN_TABLE
+SELECT global.binlog_trx_meta_data;
+--Error ER_BAD_FIELD_ERROR
+SELECT binlog_trx_meta_data = @@session.binlog_trx_meta_data;
+
+SET @@global.binlog_trx_meta_data = @start_value;
+SELECT @@global.binlog_trx_meta_data;