From c09eb8052a6ad36b0f9c6632b26bc2eed90b3734 Mon Sep 17 00:00:00 2001 From: Santosh Praneeth Banda Date: Tue, 23 Dec 2014 15:31:04 -0800 Subject: [PATCH] Add column names to rbr table map log events. Summary: Column names added to table map log events are used by slave to find out the corresponding field in slave's table whose index may be different than the index of the field in master's table due to a schema change. This behavior is controlled for each table using Alter table rbr_column_names=1 This is an inplace alter which does metadata only change in the table frm file. During a schema change on a replica set, slaves may have different schema than master which will break RBR. The following functionality needs to be added to online schema change (OSC): a. Before starting OSC, run alter table rbr_column_names=1 on master of the replica set. This will get replicated to the slaves. b. Run OSC on all slaves first and then on master. c. Run alter table rbr_column_names=0 on master of the replica set to turn off the feature. The column names will increase the binlog size, so they should be used temporarily on per-table basis only while OSC is running on the replica set. This change is backward compatible but older MySQL versions will not understand the new format of table map log events. So, before using this feature all the servers in the replica set must be upgraded to new MySQL with this change. Note mysqlbinlog is not updated to show table column names when available. Test Plan: mtr tests Reviewers: ebergen, jtolmer Reviewed By: jtolmer --- .../r/mysqld--help-notwin-profiling.result | 2 +- mysql-test/r/mysqld--help-notwin.result | 2 +- .../suite/rpl/r/rpl_rbr_column_names.result | 80 +++++++++++ ...pe_conversion_with_rbr_column_names.result | 32 +++++ .../suite/rpl/t/rpl_rbr_column_names.test | 73 ++++++++++ ...type_conversion_with_rbr_column_names.test | 42 ++++++ sql/handler.cc | 3 +- sql/handler.h | 5 + sql/lex.h | 1 + sql/log_event.cc | 75 +++++++++- sql/log_event.h | 15 +- sql/mysqld.cc | 1 + sql/rpl_record.cc | 131 +++++++++++++++++- sql/rpl_record.h | 1 + sql/rpl_utility.cc | 74 ++++++++-- sql/rpl_utility.h | 20 ++- sql/sql_alter.h | 2 + sql/sql_cmd.h | 1 + sql/sql_show.cc | 4 + sql/sql_table.cc | 7 +- sql/sql_yacc.yy | 7 + sql/table.cc | 3 + sql/table.h | 3 + 23 files changed, 562 insertions(+), 22 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_rbr_column_names.result create mode 100644 mysql-test/suite/rpl/r/rpl_type_conversion_with_rbr_column_names.result create mode 100644 mysql-test/suite/rpl/t/rpl_rbr_column_names.test create mode 100644 mysql-test/suite/rpl/t/rpl_type_conversion_with_rbr_column_names.test diff --git a/mysql-test/r/mysqld--help-notwin-profiling.result b/mysql-test/r/mysqld--help-notwin-profiling.result index ac0af730aff9..3961ee834753 100644 --- a/mysql-test/r/mysqld--help-notwin-profiling.result +++ b/mysql-test/r/mysqld--help-notwin-profiling.result @@ -1404,7 +1404,7 @@ performance-schema-max-rwlock-instances -1 performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-stage-classes 150 -performance-schema-max-statement-classes 173 +performance-schema-max-statement-classes 174 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 performance-schema-max-thread-classes 50 diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 898718a7c636..ae654f1a92c8 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -1402,7 +1402,7 @@ performance-schema-max-rwlock-instances -1 performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-stage-classes 150 -performance-schema-max-statement-classes 173 +performance-schema-max-statement-classes 174 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 performance-schema-max-thread-classes 50 diff --git a/mysql-test/suite/rpl/r/rpl_rbr_column_names.result b/mysql-test/suite/rpl/r/rpl_rbr_column_names.result new file mode 100644 index 000000000000..9ea47557ec67 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_rbr_column_names.result @@ -0,0 +1,80 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +call mtr.add_suppression("Slave SQL.*Column [0-9] of table .test.t[0-9]*. cannot be converted from type.* Error_code: 1677"); +create table t1 (a int, b text) rbr_column_names=1; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` text +) ENGINE=MyISAM DEFAULT CHARSET=latin1 RBR_COLUMN_NAMES=1 +drop table t1; +create table t1 (b text); +Check slave can deal with missing columns +insert into t1 values(1, "column1"); +select * from t1; +b +column1 +drop table t1; +create table t1 (b text, a int); +Check slave can deal with columns in different order +insert into t1(a, b) values(2, "column2"); +update t1 set b="update column2" where a=2; +select * from t1; +b a +update column2 2 +drop table t1; +create table t1 (extra_column text, a int, b text); +Check slave can deal with new columns added at the beginning +insert into t1 values(2, "column2"); +select * from t1; +extra_column a b +NULL 2 column2 +drop table t1; +create table t1(a int, extra_column text, b text); +Check slave can deal with new columns added in the middle +insert into t1 values(2, "column2"); +insert into t1 values(3, "column3"); +delete from t1 where a=3 and b="column3"; +select * from t1; +a extra_column b +2 NULL column2 +drop table t1; +create table t1 (a int, b int); +Check slave fails with an error due to type mismatch +insert into t1 values(1, "column2"); +include/wait_for_slave_sql_error.inc [errno=1677] +include/stop_slave.inc +include/rpl_reset.inc +Check rbr_column_names can be changed using ALTER TABLE +alter table t1 rbr_column_names = 0; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` text +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +alter table t1 rbr_column_names = 1; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` text +) ENGINE=MyISAM DEFAULT CHARSET=latin1 RBR_COLUMN_NAMES=1 +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 RBR_COLUMN_NAMES=1 +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_type_conversion_with_rbr_column_names.result b/mysql-test/suite/rpl/r/rpl_type_conversion_with_rbr_column_names.result new file mode 100644 index 000000000000..28ca9ad3bf62 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_type_conversion_with_rbr_column_names.result @@ -0,0 +1,32 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +set @saved_slave_type_conversions = @@global.slave_type_conversions; +set @@global.slave_type_conversions = 'ALL_LOSSY,ALL_NON_LOSSY'; +create table t1(a int, b tinyblob, c int) rbr_column_names=1; +Check slave can deal with reordering of columns +drop table t1; +create table t1(a int, c tinyint, b mediumblob) rbr_column_names=1; +insert into t1 values(1, "tiny blob", 100); +select * from t1; +a c b +1 100 tiny blob +Check slave can deal with deleted columns +drop table t1; +create table t1(c tinyint, b mediumblob) rbr_column_names=1; +insert into t1 values(1, "tiny blob", 100); +select * from t1; +c b +100 tiny blob +Check slave can deal with added columns +drop table t1; +create table t1(a int, d int, c tinyint, b mediumblob) rbr_column_names=1; +insert into t1 values(1, "tiny blob", 100); +select * from t1; +a d c b +1 NULL 100 tiny blob +set @@global.slave_type_conversions = @saved_slave_type_conversions; +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_rbr_column_names.test b/mysql-test/suite/rpl/t/rpl_rbr_column_names.test new file mode 100644 index 000000000000..bd6126f2846f --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_rbr_column_names.test @@ -0,0 +1,73 @@ +source include/master-slave.inc; +source include/have_binlog_format_row.inc; +call mtr.add_suppression("Slave SQL.*Column [0-9] of table .test.t[0-9]*. cannot be converted from type.* Error_code: 1677"); + +connection master; +create table t1 (a int, b text) rbr_column_names=1; +show create table t1; +sync_slave_with_master; + +drop table t1; +create table t1 (b text); +--echo Check slave can deal with missing columns +connection master; +insert into t1 values(1, "column1"); +sync_slave_with_master; +select * from t1; + +drop table t1; +create table t1 (b text, a int); +--echo Check slave can deal with columns in different order +connection master; +insert into t1(a, b) values(2, "column2"); +update t1 set b="update column2" where a=2; +sync_slave_with_master; +select * from t1; + +drop table t1; +create table t1 (extra_column text, a int, b text); +--echo Check slave can deal with new columns added at the beginning +connection master; +insert into t1 values(2, "column2"); +sync_slave_with_master; +select * from t1; + +drop table t1; +create table t1(a int, extra_column text, b text); +--echo Check slave can deal with new columns added in the middle +connection master; +insert into t1 values(2, "column2"); +insert into t1 values(3, "column3"); +delete from t1 where a=3 and b="column3"; +sync_slave_with_master; +select * from t1; + +drop table t1; +create table t1 (a int, b int); +--echo Check slave fails with an error due to type mismatch +connection master; +insert into t1 values(1, "column2"); +connection slave; +let $slave_sql_errno=convert_error(ER_SLAVE_CONVERSION_FAILED); +source include/wait_for_slave_sql_error.inc; +source include/stop_slave.inc; +let $rpl_only_running_threads= 1; +source include/rpl_reset.inc; + +--echo Check rbr_column_names can be changed using ALTER TABLE +connection master; +alter table t1 rbr_column_names = 0; +show create table t1; +sync_slave_with_master; +show create table t1; + +connection master; +alter table t1 rbr_column_names = 1; +show create table t1; +sync_slave_with_master; +show create table t1; + + +connection master; +drop table t1; +source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl/t/rpl_type_conversion_with_rbr_column_names.test b/mysql-test/suite/rpl/t/rpl_type_conversion_with_rbr_column_names.test new file mode 100644 index 000000000000..10cd0573cfe5 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_type_conversion_with_rbr_column_names.test @@ -0,0 +1,42 @@ +# This test verifies slave_type_conversions when +# rbr_column_names is enabled for a table. +source include/have_binlog_format_row.inc; +source include/master-slave.inc; + +connection slave; +set @saved_slave_type_conversions = @@global.slave_type_conversions; +set @@global.slave_type_conversions = 'ALL_LOSSY,ALL_NON_LOSSY'; + +connection master; +create table t1(a int, b tinyblob, c int) rbr_column_names=1; +sync_slave_with_master; + +--echo Check slave can deal with reordering of columns +drop table t1; +create table t1(a int, c tinyint, b mediumblob) rbr_column_names=1; +connection master; +insert into t1 values(1, "tiny blob", 100); +sync_slave_with_master; +select * from t1; + +--echo Check slave can deal with deleted columns +drop table t1; +create table t1(c tinyint, b mediumblob) rbr_column_names=1; +connection master; +insert into t1 values(1, "tiny blob", 100); +sync_slave_with_master; +select * from t1; + +--echo Check slave can deal with added columns +drop table t1; +create table t1(a int, d int, c tinyint, b mediumblob) rbr_column_names=1; +connection master; +insert into t1 values(1, "tiny blob", 100); +sync_slave_with_master; +select * from t1; + +set @@global.slave_type_conversions = @saved_slave_type_conversions; +connection master; +drop table t1; +source include/rpl_end.inc; + diff --git a/sql/handler.cc b/sql/handler.cc index 971e6e217ccf..822307599667 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4568,7 +4568,8 @@ handler::check_if_supported_inplace_alter(TABLE *altered_table, Alter_inplace_info::ALTER_COLUMN_NAME | Alter_inplace_info::ALTER_COLUMN_DEFAULT | Alter_inplace_info::CHANGE_CREATE_OPTION | - Alter_inplace_info::ALTER_RENAME; + Alter_inplace_info::ALTER_RENAME | + Alter_inplace_info::ALTER_RBR_COLUMN_NAMES; /* Is there at least one operation that requires copy algorithm? */ if (ha_alter_info->handler_flags & ~inplace_offline_operations) diff --git a/sql/handler.h b/sql/handler.h index 9f01c35018b3..7840a0c8e3ce 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -476,6 +476,7 @@ given at all. */ given at all. */ #define HA_CREATE_USED_STATS_SAMPLE_PAGES (1L << 24) +#define HA_CREATE_USED_RBR_COLUMN_NAMES (1L << 47) /* @@ -1102,6 +1103,8 @@ typedef struct st_ha_create_information uint extra_size; /* length of extra data segment */ bool varchar; /* 1 if table has a VARCHAR */ enum ha_storage_media storage_media; /* DEFAULT, DISK or MEMORY */ + bool rbr_column_names; /* If true, column names for this table are logged + in Table_map_log_events */ } HA_CREATE_INFO; /** @@ -1253,6 +1256,8 @@ class Alter_inplace_info */ static const HA_ALTER_FLAGS RECREATE_TABLE = 1L << 29; + static const HA_ALTER_FLAGS ALTER_RBR_COLUMN_NAMES = 1L << 47; + /** Create options (like MAX_ROWS) for the new version of table. diff --git a/sql/lex.h b/sql/lex.h index 7c1276d7b82c..bcb94ca09c10 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -459,6 +459,7 @@ static SYMBOL symbols[] = { { "QUERY", SYM(QUERY_SYM)}, { "QUICK", SYM(QUICK)}, { "RANGE", SYM(RANGE_SYM)}, + { "RBR_COLUMN_NAMES", SYM(RBR_COLUMN_NAMES_SYM)}, { "READ", SYM(READ_SYM)}, { "READ_ONLY", SYM(READ_ONLY_SYM)}, { "READ_WRITE", SYM(READ_WRITE_SYM)}, diff --git a/sql/log_event.cc b/sql/log_event.cc index b560db95bcd4..3b92f3008199 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -11719,7 +11719,28 @@ int Rows_log_event::do_apply_event(Relay_log_info const *rli) /* WRITE ROWS EVENTS store the bitmap in m_cols instead of m_cols_ai */ MY_BITMAP *after_image= ((get_general_type_code() == UPDATE_ROWS_EVENT) ? &m_cols_ai : &m_cols); - bitmap_intersect(table->write_set, after_image); + table_def *tabledef= NULL; + TABLE *conv_table= NULL; + assert(rli->get_table_data(table, &tabledef, &conv_table)); + if (tabledef->have_column_names()) + { + bitmap_clear_all(table->write_set); + // use column names to find out the correct field indices + // on slave's table. + for (uint i = 0; i < tabledef->size(); ++i) + { + if (bitmap_is_set(after_image, i)) + { + const char* col_name = tabledef->get_column_name(i); + Field *const field = find_field_in_table_sef(table, col_name); + // field may be NULL if the field is removed on slave. + if (field) + bitmap_set_bit(table->write_set, field->field_index); + } + } + } + else + bitmap_intersect(table->write_set, after_image); this->slave_exec_mode= slave_exec_mode_options; // fix the mode @@ -12286,7 +12307,9 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, m_null_bits(0), m_meta_memory(NULL), m_primary_key_fields(0), - m_primary_key_fields_size(0) + m_primary_key_fields_size(0), + m_column_names(0), + m_column_names_size(0) { uchar cbuf[sizeof(m_colcnt) + 1]; uchar *cbuf_end; @@ -12320,6 +12343,19 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, m_coltype[i]= m_table->field[i]->binlog_type(); } + my_bool log_column_names = tbl->s->rbr_column_names; + + if (log_column_names) + { + // get column_names size. + for (unsigned int i= 0 ; i < m_table->s->fields ; i++) + { + // + 1 for storing the length of the column name. + // + 1 for '\0' + m_column_names_size += strlen(m_table->s->field[i]->field_name) + 1 + 1; + } + } + /* Calculate a bitmap for the results of maybe_null() for all columns. The bitmap is used to determine when there is a column from the master @@ -12355,6 +12391,13 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, &m_field_metadata, (m_colcnt * 2), NULL); + if (m_column_names_size) + { + // allocate memory for column names. + m_column_names = (uchar*) my_malloc(m_column_names_size, MYF(MY_WME)); + m_data_size += m_column_names_size; + } + memset(m_field_metadata, 0, (m_colcnt * 2)); /* @@ -12409,6 +12452,18 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, ptr = net_store_length(ptr, (pkey_info->key_part[i].fieldnr - 1)); } } + + if (log_column_names) + { + uint index = 0; + for (uint i= 0 ; i < m_table->s->fields ; i++) + { + uint length = strlen(m_table->s->field[i]->field_name) + 1; + m_column_names[index++] = length; + strcpy((char*)(m_column_names + index), m_table->s->field[i]->field_name); + index += length; + } + } } #endif /* !defined(MYSQL_CLIENT) */ @@ -12429,7 +12484,7 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, m_memory(NULL), m_table_id(ULONGLONG_MAX), m_flags(0), m_data_size(0), m_field_metadata(0), m_field_metadata_size(0), m_null_bits(0), m_meta_memory(NULL), m_primary_key_fields(0), - m_primary_key_fields_size(0) + m_primary_key_fields_size(0), m_column_names(0), m_column_names_size(0) { unsigned int bytes_read= 0; DBUG_ENTER("Table_map_log_event::Table_map_log_event(const char*,uint,...)"); @@ -12529,6 +12584,12 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, ptr_after_colcnt = (uchar*)ptr_after_colcnt + m_primary_key_fields_size; } + m_column_names_size = event_len - (ptr_after_colcnt - (uchar *)buf); + if (m_column_names_size) + { + m_column_names = (uchar*) my_malloc(m_column_names_size, MYF(MY_WME)); + memcpy(m_column_names, ptr_after_colcnt, m_column_names_size); + } } } } @@ -12541,6 +12602,7 @@ Table_map_log_event::~Table_map_log_event() { my_free(m_meta_memory); my_free(m_memory); + my_free(m_column_names); my_free(m_primary_key_fields); } @@ -12710,7 +12772,7 @@ int Table_map_log_event::do_apply_event(Relay_log_info const *rli) new (&table_list->m_tabledef) table_def(m_coltype, m_colcnt, m_field_metadata, m_field_metadata_size, - m_null_bits, m_flags); + m_null_bits, m_flags, m_column_names); table_list->m_tabledef_valid= TRUE; table_list->m_conv_table= NULL; table_list->open_type= OT_BASE_ONLY; @@ -12839,7 +12901,10 @@ bool Table_map_log_event::write_data_body(IO_CACHE *file) wrapper_my_b_safe_write(file, m_size_buf, (size_t) (m_size_buf_end - m_size_buf)) || wrapper_my_b_safe_write(file, m_primary_key_fields, - m_primary_key_fields_size)); + m_primary_key_fields_size) || + wrapper_my_b_safe_write(file, m_column_names, + m_column_names_size)); + } #endif diff --git a/sql/log_event.h b/sql/log_event.h index 74cb77309956..87db4012b7bc 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -3936,7 +3936,8 @@ class Table_map_log_event : public Log_event table_def *create_table_def() { return new table_def(m_coltype, m_colcnt, m_field_metadata, - m_field_metadata_size, m_null_bits, m_flags); + m_field_metadata_size, m_null_bits, m_flags, + m_column_names); } #endif const Table_id& get_table_id() const { return m_table_id; } @@ -4026,6 +4027,18 @@ class Table_map_log_event : public Log_event uchar *m_meta_memory; uchar *m_primary_key_fields; uint m_primary_key_fields_size; + /* Table column names are added to the Table_map_log_event at the end + in the following format: + a) Length of the column name including the terminating '\0' is added in + one byte (strlen(column_name) + 1). One byte is enough since maximum + column name length is 64. + b) column_name is appended to the buffer including the terminating '\0'. + */ + uchar *m_column_names; + // Since m_column_names buffer contains terminating '\0' in the middle, + // using strlen() will not give correct length, so track the actual length + // in this variable. + ulong m_column_names_size; }; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ea562f0b83c5..ba77a7c6fe17 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4084,6 +4084,7 @@ SHOW_VAR com_status_vars[]= { {"prepare_sql", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_PREPARE]), SHOW_LONG_STATUS}, {"purge", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_PURGE]), SHOW_LONG_STATUS}, {"purge_before_date", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_PURGE_BEFORE]), SHOW_LONG_STATUS}, + {"rbr_column_names", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_RBR_COLUMN_NAMES]), SHOW_LONG_STATUS}, {"release_savepoint", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_RELEASE_SAVEPOINT]), SHOW_LONG_STATUS}, {"rename_table", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_RENAME_TABLE]), SHOW_LONG_STATUS}, {"rename_user", (char*) offsetof(STATUS_VAR, com_stat[(uint) SQLCOM_RENAME_USER]), SHOW_LONG_STATUS}, diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index 7ed317ea1736..605c807b8429 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -155,6 +155,129 @@ pack_row(TABLE *table, MY_BITMAP const* cols, } #endif +#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) +/** + Unpack a row into @c table->record[0]. + + This should be only used when table_map_log_event contains column names. + This function iterates over the columns of master and finds corresponding + field in slave's table using the column names stored in table_def. + + Note, a hash for field names is created if number of fields in the table + is above MAX_FIELDS_BEFORE_HASH. We will use this hash for lookup when + available (see find_field_in_table_sef). + + @param table Table to unpack into + @param colcnt Number of columns to read from record + @param row_data + Packed row data + @param cols Pointer to bitset describing columns to fill in + @param curr_row_end + Pointer to variable that will hold the value of the + one-after-end position for the current row + @param master_reclength + Pointer to variable that will be set to the length of the + record on the master side + @param row_end + Pointer to variable that will hold the value of the + end position for the data in the row event + + @retval 0 No error + + @retval HA_ERR_GENERIC + A generic, internal, error caused the unpacking to fail. + +*/ +int unpack_row_with_column_info(TABLE *table, uint const colcnt, + uchar const *const row_data, + MY_BITMAP const *cols, + uchar const **const current_row_end, + ulong *const master_reclength, + uchar const *const row_end, + table_def* tabledef, + TABLE *conv_table) +{ + DBUG_ENTER("unpack_row_with_column_info"); + uchar const *null_bits= row_data; + size_t const master_null_byte_count= (bitmap_bits_set(cols) + 7) / 8; + uchar const *pack_ptr= row_data + master_null_byte_count; + uint null_bit_index = 0; + + for (uint i = 0; i < tabledef->size(); ++i) + { + if (!bitmap_is_set(cols, i)) + // Field not actually present in the row_data + continue; + const char* col_name = tabledef->get_column_name(i); + DBUG_ASSERT(col_name); + // use conversion table if present. + Field *conv_field = conv_table ? conv_table->field[i] : NULL; + Field *const field = conv_field ? conv_field : + find_field_in_table_sef(table, col_name); + int is_null= (null_bits[null_bit_index / 8] + >> (null_bit_index % 8)) & 0x01; + if (field) + { + if (is_null) + { + // Handle null column case. + if (field->maybe_null()) + { + field->reset(); + field->set_null(); + } + else + { + field->set_default(); + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_BAD_NULL_ERROR, ER(ER_BAD_NULL_ERROR), + field->field_name); + } + } + else + { + field->set_notnull(); + uint16 const metadata = tabledef->field_metadata(i); + uint32 len = tabledef->calc_field_size(i, (uchar *) pack_ptr); + if (pack_ptr + len > row_end) + { + pack_ptr += len; + my_error(ER_SLAVE_CORRUPT_EVENT, MYF(0)); + DBUG_RETURN(ER_SLAVE_CORRUPT_EVENT); + } + pack_ptr= field->unpack(field->ptr, pack_ptr, metadata, TRUE); + } + if (conv_field) + { + Copy_field copy; + copy.set(find_field_in_table_sef(table, col_name), conv_field, TRUE); + (*copy.do_copy)(©); + } + } + else + { + // This column is removed on slave, so skip this field. + if (!is_null) + { + uint32 len = tabledef->calc_field_size(i, (uchar *) pack_ptr); + if (pack_ptr + len > row_end) + { + pack_ptr += len; + my_error(ER_SLAVE_CORRUPT_EVENT, MYF(0)); + DBUG_RETURN(ER_SLAVE_CORRUPT_EVENT); + } + pack_ptr += len; + } + } + ++null_bit_index; + } + *current_row_end = pack_ptr; + if (master_reclength) + { + *master_reclength = table->s->reclength; + } + DBUG_RETURN(0); +} /** Unpack a row into @c table->record[0]. @@ -198,7 +321,6 @@ pack_row(TABLE *table, MY_BITMAP const* cols, @retval HA_ERR_GENERIC A generic, internal, error caused the unpacking to fail. */ -#if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) int unpack_row(Relay_log_info const *rli, TABLE *table, uint const colcnt, @@ -254,6 +376,13 @@ unpack_row(Relay_log_info const *rli, if (rli && !table_found) DBUG_RETURN(HA_ERR_GENERIC); + if (tabledef->have_column_names()) + { + DBUG_RETURN(unpack_row_with_column_info(table, colcnt, row_data, cols, + current_row_end, master_reclength, + row_end, tabledef, conv_table)); + } + for (field_ptr= begin_ptr ; field_ptr < end_ptr && *field_ptr ; ++field_ptr) { /* diff --git a/sql/rpl_record.h b/sql/rpl_record.h index db7a850cfb56..f5387bd123b7 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -26,6 +26,7 @@ typedef struct st_bitmap MY_BITMAP; #if !defined(MYSQL_CLIENT) size_t pack_row(TABLE* table, MY_BITMAP const* cols, uchar *row_data, const uchar *data); +#include "sql_base.h" #endif #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) diff --git a/sql/rpl_utility.cc b/sql/rpl_utility.cc index 97e32239bea8..6022e24ebd0b 100644 --- a/sql/rpl_utility.cc +++ b/sql/rpl_utility.cc @@ -868,14 +868,27 @@ table_def::compatible_with(THD *thd, Relay_log_info *rli, { /* We only check the initial columns for the tables. + + If column names are logged by the master, we check all + the columns present in master. */ - uint const cols_to_check= min(table->s->fields, size()); + uint cols_to_check = have_column_names() ? size() : + min(table->s->fields, size()); + TABLE *tmp_table= NULL; for (uint col= 0 ; col < cols_to_check ; ++col) { - Field *const field= table->field[col]; + Field *field = have_column_names() ? + find_field_in_table_sef(table, get_column_name(col)) : + table->field[col]; int order; + if (!field) + { + // This column is removed on slave + continue; + } + if (can_convert_field_to(field, type(col), field_metadata(col), rli, m_flags, &order)) { DBUG_PRINT("debug", ("Checking column %d -" @@ -911,7 +924,8 @@ table_def::compatible_with(THD *thd, Relay_log_info *rli, DBUG_PRINT("debug", ("Checking column %d -" " field '%s' can not be converted", col, field->field_name)); - DBUG_ASSERT(col < size() && col < table->s->fields); + DBUG_ASSERT(col < size() && (col < table->s->fields + || have_column_names())); DBUG_ASSERT(table->s->db.str && table->s->table_name.str); const char *db_name= table->s->db.str; const char *tbl_name= table->s->table_name.str; @@ -933,19 +947,24 @@ table_def::compatible_with(THD *thd, Relay_log_info *rli, if (tmp_table) { for (unsigned int col= 0; col < tmp_table->s->fields; ++col) - if (tmp_table->field[col]) + { + Field *const slave_field = have_column_names() ? + find_field_in_table_sef(table, get_column_name(col)) : + table->field[col]; + if (tmp_table->field[col] && slave_field) { char source_buf[MAX_FIELD_WIDTH]; char target_buf[MAX_FIELD_WIDTH]; String source_type(source_buf, sizeof(source_buf), &my_charset_latin1); String target_type(target_buf, sizeof(target_buf), &my_charset_latin1); tmp_table->field[col]->sql_type(source_type); - table->field[col]->sql_type(target_type); + slave_field->sql_type(target_type); DBUG_PRINT("debug", ("Field %s - conversion required." " Source type: '%s', Target type: '%s'", tmp_table->field[col]->field_name, source_type.c_ptr_safe(), target_type.c_ptr_safe())); } + } } #endif @@ -974,7 +993,8 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * min(columns@master, columns@slave) columns in the conversion table. */ - uint const cols_to_create= min(target_table->s->fields, size()); + uint const cols_to_create = have_column_names() ? size() : + min(target_table->s->fields, size()); // Default value : treat all values signed bool unsigned_flag= FALSE; @@ -989,6 +1009,10 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * for (uint col= 0 ; col < cols_to_create; ++col) { + Field * const slave_field = have_column_names() ? + find_field_in_table_sef(target_table, get_column_name(col)) : + target_table->field[col]; + Create_field *field_def= (Create_field*) alloc_root(thd->mem_root, sizeof(Create_field)); if (field_list.push_back(field_def)) @@ -1005,7 +1029,8 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * int precision; case MYSQL_TYPE_ENUM: case MYSQL_TYPE_SET: - interval= static_cast(target_table->field[col])->typelib; + if (slave_field) + interval= static_cast(slave_field)->typelib; pack_length= field_metadata(col) & 0x00ff; break; @@ -1046,7 +1071,8 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * DBUG_PRINT("debug", ("sql_type: %d, target_field: '%s', max_length: %d, decimals: %d," " maybe_null: %d, unsigned_flag: %d, pack_length: %u", - binlog_type(col), target_table->field[col]->field_name, + binlog_type(col), slave_field ? + slave_field->field_name : "", max_length, decimals, TRUE, unsigned_flag, pack_length)); field_def->init_for_tmp_table(type(col), max_length, @@ -1054,7 +1080,8 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * TRUE, // maybe_null unsigned_flag, // unsigned_flag pack_length); - field_def->charset= target_table->field[col]->charset(); + if (slave_field) + field_def->charset= slave_field->charset(); field_def->interval= interval; } @@ -1073,7 +1100,8 @@ TABLE *table_def::create_conversion_table(THD *thd, Relay_log_info *rli, TABLE * table_def::table_def(unsigned char *types, ulong size, uchar *field_metadata, int metadata_size, - uchar *null_bitmap, uint16 flags) + uchar *null_bitmap, uint16 flags, + const uchar *column_names) : m_size(size), m_type(0), m_field_metadata_size(metadata_size), m_field_metadata(0), m_null_bits(0), m_flags(flags), m_memory(NULL) @@ -1165,12 +1193,38 @@ table_def::table_def(unsigned char *types, ulong size, } if (m_size && null_bitmap) memcpy(m_null_bits, null_bitmap, (m_size + 7) / 8); + +#ifdef MYSQL_SERVER + init_dynamic_array(&m_column_names, sizeof(char*), 10, 10); + if (column_names) { + // store column names in to an array. + for (uint i= 0; i < m_size; i++) + { + uint length = (uint) *column_names++; + // memory allocated by this malloc is freed in + // the class destructor. + char* str = (char*) my_malloc(length, MYF(0)); + strncpy(str, (const char*)column_names, length); + insert_dynamic(&m_column_names, (uchar*) &str); + column_names += length; + } + } +#endif /* MYSQL_SERVER */ } table_def::~table_def() { my_free(m_memory); +#ifdef MYSQL_SERVER + for (uint i = 0; i < m_column_names.elements; ++i) + { + char **str = + dynamic_element(&m_column_names, i, char**); + my_free(*str); + } + delete_dynamic(&m_column_names); +#endif /* MYSQL_SERVER */ #ifndef DBUG_OFF m_type= 0; m_size= 0; diff --git a/sql/rpl_utility.h b/sql/rpl_utility.h index 9e66a27ec825..cc21db72b008 100644 --- a/sql/rpl_utility.h +++ b/sql/rpl_utility.h @@ -242,7 +242,8 @@ class table_def @param null_bitmap The bitmap of fields that can be null */ table_def(unsigned char *types, ulong size, uchar *field_metadata, - int metadata_size, uchar *null_bitmap, uint16 flags); + int metadata_size, uchar *null_bitmap, uint16 flags, + const uchar* column_names); ~table_def(); @@ -402,6 +403,20 @@ class table_def thread's memroot, NULL if the table could not be created */ TABLE *create_conversion_table(THD *thd, Relay_log_info *rli, TABLE *target_table) const; + + bool have_column_names() const + { + return m_column_names.elements != 0; + } + + const char* get_column_name(uint index) const + { + if (index >= m_column_names.elements) + return 0; + char **str = dynamic_element(&m_column_names, index, char**); + return *str; + } + #endif @@ -413,6 +428,9 @@ class table_def uchar *m_null_bits; uint16 m_flags; // Table flags uchar *m_memory; +#ifdef MYSQL_SERVER + DYNAMIC_ARRAY m_column_names; +#endif /* MYSQL_SERVER */ }; diff --git a/sql/sql_alter.h b/sql/sql_alter.h index 7592a63043f6..d25006e89308 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -127,6 +127,8 @@ class Alter_info // Set for ADD [COLUMN] FIRST | AFTER static const uint ALTER_COLUMN_ORDER = 1L << 26; + static const ulong ALTER_RBR_COLUMN_NAMES = 1L << 47; + enum enum_enable_or_disable { LEAVE_AS_IS, ENABLE, DISABLE }; diff --git a/sql/sql_cmd.h b/sql/sql_cmd.h index a542aa460ac5..c8cee083446b 100644 --- a/sql/sql_cmd.h +++ b/sql/sql_cmd.h @@ -93,6 +93,7 @@ enum enum_sql_command { SQLCOM_SHOW_MEMORY_STATUS, SQLCOM_FIND_GTID_POSITION, SQLCOM_GTID_EXECUTED, + SQLCOM_RBR_COLUMN_NAMES, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 97193e2e753c..5e2fadde7bff 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1560,6 +1560,7 @@ int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet, memset(&create_info, 0, sizeof(create_info)); /* Allow update_create_info to update row type */ create_info.row_type= share->row_type; + create_info.rbr_column_names = share->rbr_column_names; file->update_create_info(&create_info); primary_key= share->primary_key; @@ -1779,6 +1780,9 @@ int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet, end= longlong10_to_str(table->s->key_block_size, buff, 10); packet->append(buff, (uint) (end - buff)); } + if (create_info.rbr_column_names) + packet->append(STRING_WITH_LEN(" RBR_COLUMN_NAMES=1")); + table->file->append_create_info(packet); if (share->comment.length) { diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 0ebc2dd81891..c391c3e8a9fa 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5793,6 +5793,8 @@ static bool fill_alter_inplace_info(THD *thd, /* Check for: ALTER TABLE FORCE, ALTER TABLE ENGINE and OPTIMIZE TABLE. */ if (alter_info->flags & Alter_info::ALTER_RECREATE) ha_alter_info->handler_flags|= Alter_inplace_info::RECREATE_TABLE; + if (alter_info->flags & Alter_info::ALTER_RBR_COLUMN_NAMES) + ha_alter_info->handler_flags |= Alter_info::ALTER_RBR_COLUMN_NAMES; /* If we altering table with old VARCHAR fields we will be automatically @@ -7012,7 +7014,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, List key_parts; uint db_create_options= (table->s->db_create_options & ~(HA_OPTION_PACK_RECORD)); - uint used_fields= create_info->used_fields; + ulong used_fields= create_info->used_fields; KEY *key_info=table->key_info; List_iterator new_key_iterator; Key *key_element; @@ -7042,6 +7044,9 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, if (!(used_fields & HA_CREATE_USED_KEY_BLOCK_SIZE)) create_info->key_block_size= table->s->key_block_size; + if (!(used_fields & HA_CREATE_USED_RBR_COLUMN_NAMES)) + create_info->rbr_column_names = table->s->rbr_column_names; + if (!(used_fields & HA_CREATE_USED_STATS_SAMPLE_PAGES)) create_info->stats_sample_pages= table->s->stats_sample_pages; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index e10cb524484c..50c5a4d79e84 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1510,6 +1510,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token QUERY_SYM %token QUICK %token RANGE_SYM /* SQL-2003-R */ +%token RBR_COLUMN_NAMES_SYM %token READS_SYM /* SQL-2003-R */ %token READ_ONLY_SYM %token READ_SYM /* SQL-2003-N */ @@ -6258,6 +6259,12 @@ create_table_option: Lex->create_info.used_fields|= HA_CREATE_USED_KEY_BLOCK_SIZE; Lex->create_info.key_block_size= $3; } + | RBR_COLUMN_NAMES_SYM opt_equal ulong_num + { + Lex->create_info.used_fields |= HA_CREATE_USED_RBR_COLUMN_NAMES; + Lex->create_info.rbr_column_names = $3 ? true : false; + Lex->alter_info.flags |= Alter_info::ALTER_RBR_COLUMN_NAMES; + } ; default_charset: diff --git a/sql/table.cc b/sql/table.cc index 2c7987b85d9a..74c3c3160ae1 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1070,6 +1070,7 @@ static int open_binary_frm(THD *thd, TABLE_SHARE *share, uchar *head, share->null_field_first= 1; share->stats_sample_pages= uint2korr(head+42); share->stats_auto_recalc= static_cast(head[44]); + share->rbr_column_names = head[45] & 0x80 ? true : false; } if (!share->table_charset) { @@ -3035,6 +3036,7 @@ File create_frm(THD *thd, const char *name, const char *db, int2store(fileinfo+42, create_info->stats_sample_pages & 0xffff); fileinfo[44]= (uchar) create_info->stats_auto_recalc; fileinfo[45]= 0; + fileinfo[45] |= (create_info->rbr_column_names) ? 0x80 : 0; fileinfo[46]= 0; int4store(fileinfo+47, key_length); tmp= MYSQL_VERSION_ID; // Store to avoid warning from int4store @@ -3083,6 +3085,7 @@ void update_create_info_from_table(HA_CREATE_INFO *create_info, TABLE *table) create_info->comment= share->comment; create_info->storage_media= share->default_storage_media; create_info->tablespace= share->tablespace; + create_info->rbr_column_names = share->rbr_column_names; DBUG_VOID_RETURN; } diff --git a/sql/table.h b/sql/table.h index bfdd4eecb5a8..e0392612a6f7 100644 --- a/sql/table.h +++ b/sql/table.h @@ -753,6 +753,9 @@ struct TABLE_SHARE const File_parser *view_def; + // If true, column names for this table are logged in Table_map_log_events + bool rbr_column_names; + /* Set share's table cache key and update its db and table name appropriately.