Add primary key information to table map log event

Summary: Add primary key information to table map log event. This is useful for incremental backup tool to fetch primary key of a table without querying the server. This is also useful for the backup tool in tracking primary key change due to schema changes. Preserves the order of primary key columns in table map log events. Bitmap used for storing primary key column index in table map log events doesn't preserve the order. While this is completely fine with mysql_backup tool which generates incremental logical dump files, this doesn't work when differentialbackup is used to merge dump files since the sorted order of rows in full logical dumps and in incremental logical dumps generated by mysql_backup are different. Test Plan: mtr test to print primary key fields using mysqlbinlog. Reviewers: steaphan, jtolmer Reviewed By: jtolmer
ldonoso · Jan 24, 2017 · e236984 · e236984
1 parent 0507fa2
commit e236984
Show file tree

Hide file tree

Showing 5 changed files with 160 additions and 7 deletions.
diff --git a/mysql-test/suite/rpl/r/rpl_log_only_query_comments.result b/mysql-test/suite/rpl/r/rpl_log_only_query_comments.result
@@ -12,17 +12,17 @@ drop table t1;
 #010909  4:46:40 server id 1  end_log_pos 325 	Rows_query
 # insert into t1 values(1)
 --
-#010909  4:46:40 server id 1  end_log_pos 576 	Rows_query
+#010909  4:46:40 server id 1  end_log_pos 577 	Rows_query
 # /*Comment test */
 --
-#010909  4:46:40 server id 1  end_log_pos 847 	Rows_query
+#010909  4:46:40 server id 1  end_log_pos 849 	Rows_query
 # /*Comment test */   /* Comment test*/
 #010909  4:46:40 server id 1  end_log_pos 316 	Rows_query
 # insert into t1 values(1)
 --
-#010909  4:46:40 server id 1  end_log_pos 549 	Rows_query
+#010909  4:46:40 server id 1  end_log_pos 550 	Rows_query
 # /*Comment test */
 --
-#010909  4:46:40 server id 1  end_log_pos 802 	Rows_query
+#010909  4:46:40 server id 1  end_log_pos 804 	Rows_query
 # /*Comment test */   /* Comment test*/
 include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/r/rpl_table_map_primary_fields.result b/mysql-test/suite/rpl/r/rpl_table_map_primary_fields.result
@@ -0,0 +1,34 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+set timestamp=1000000000;
+create table t1 (i1 int);
+insert into t1 values(1);
+drop table t1;
+create table t1 (i1 int, primary key(i1));
+insert into t1 values(1);
+drop table t1;
+create table t1(i1 int, i2 int, i3 int, primary key(i1, i2, i3));
+insert into t1 values(1, 1, 1);
+drop table t1;
+create table t1(i1 int, i2 int, i3 int, primary key(i3, i2, i1));
+insert into t1 values(1, 1, 1);
+drop table t1;
+create table t1(i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int, primary key(i4, i8, i9));
+insert into t1 values(1, 1, 1, 1, 1, 1, 1, 1, 1);
+drop table t1;
+## Primary key fields of tables on master
+#Primary Key Fields: ` `
+#Primary Key Fields: ` 0 `
+#Primary Key Fields: ` 0 1 2 `
+#Primary Key Fields: ` 2 1 0 `
+#Primary Key Fields: ` 3 7 8 `
+## Primary key fields of tables on slave
+#Primary Key Fields: ` `
+#Primary Key Fields: ` 0 `
+#Primary Key Fields: ` 0 1 2 `
+#Primary Key Fields: ` 2 1 0 `
+#Primary Key Fields: ` 3 7 8 `
+include/rpl_end.inc
diff --git a/mysql-test/suite/rpl/t/rpl_table_map_primary_fields.test b/mysql-test/suite/rpl/t/rpl_table_map_primary_fields.test
@@ -0,0 +1,36 @@
+source include/mysqlbinlog_have_debug.inc;
+source include/master-slave.inc;
+source include/have_binlog_format_row.inc;
+
+set timestamp=1000000000;
+connection master;
+create table t1 (i1 int);
+insert into t1 values(1);
+drop table t1;
+
+create table t1 (i1 int, primary key(i1));
+insert into t1 values(1);
+drop table t1;
+
+create table t1(i1 int, i2 int, i3 int, primary key(i1, i2, i3));
+insert into t1 values(1, 1, 1);
+drop table t1;
+
+create table t1(i1 int, i2 int, i3 int, primary key(i3, i2, i1));
+insert into t1 values(1, 1, 1);
+drop table t1;
+
+create table t1(i1 int, i2 int, i3 int, i4 int, i5 int, i6 int, i7 int, i8 int, i9 int, primary key(i4, i8, i9));
+insert into t1 values(1, 1, 1, 1, 1, 1, 1, 1, 1);
+drop table t1;
+
+# Verify a client can understand the primary key field indexes in the binary log.
+echo ## Primary key fields of tables on master;
+let $MYSQLD_DATADIR= `select @@datadir`;
+--exec $MYSQL_BINLOG -#d,print_primary_key_fields -v -v $MYSQLD_DATADIR/master-bin.000001 | grep 'Primary Key Fields'
+sync_slave_with_master;
+
+echo ## Primary key fields of tables on slave;
+let $SLAVE_DATADIR= `select @@datadir`;
+--exec $MYSQL_BINLOG -#d,print_primary_key_fields -v -v $SLAVE_DATADIR/slave-bin.000001 | grep 'Primary Key Fields'
+source include/rpl_end.inc;
diff --git a/sql/log_event.cc b/sql/log_event.cc
@@ -12233,7 +12233,9 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl,
     m_field_metadata(0),
     m_field_metadata_size(0),
     m_null_bits(0),
-    m_meta_memory(NULL)
+    m_meta_memory(NULL),
+    m_primary_key_fields(0),
+    m_primary_key_fields_size(0)
 {
   uchar cbuf[sizeof(m_colcnt) + 1];
   uchar *cbuf_end;
@@ -12273,6 +12275,28 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl,
   */
   uint num_null_bytes= (m_table->s->fields + 7) / 8;
   m_data_size+= num_null_bytes;
+
+  KEY *pkey_info = NULL;
+  // Validate that there exists a valid primary key
+  // and calculate the space required to store primary key
+  // column indexes.
+  if (m_table->key_info && m_table->s->primary_key < MAX_KEY) {
+    pkey_info = m_table->key_info + m_table->s->primary_key;
+    // see net_store_length()
+    if (pkey_info->user_defined_key_parts < 251)
+      m_primary_key_fields_size += 1;
+    else
+      m_primary_key_fields_size += 3;
+
+    for (uint i=0; i < pkey_info->user_defined_key_parts; ++i)
+    {
+      if ((pkey_info->key_part[i].fieldnr - 1) < 251)
+        m_primary_key_fields_size += 1;
+      else
+        m_primary_key_fields_size += 3;
+    }
+  }
+
   m_meta_memory= (uchar *)my_multi_malloc(MYF(MY_WME),
                                  &m_null_bits, num_null_bytes,
                                  &m_field_metadata, (m_colcnt * 2),
@@ -12314,6 +12338,24 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl,
     if (!strcmp(db_name, ""))
       m_flags |= TM_REFERRED_FK_DB_F;
   }
+
+  if (m_primary_key_fields_size < 251)
+    m_data_size += 1;
+  else
+    m_data_size += 3;
+  m_data_size += m_primary_key_fields_size;
+
+  if (m_primary_key_fields_size)
+  {
+    m_primary_key_fields = (uchar*) my_malloc(m_primary_key_fields_size,
+                                              MYF(MY_WME));
+    uchar *ptr = m_primary_key_fields;
+    ptr = net_store_length(ptr, pkey_info->user_defined_key_parts);
+    for (uint i=0; i < pkey_info->user_defined_key_parts; ++i)
+    {
+      ptr = net_store_length(ptr, (pkey_info->key_part[i].fieldnr - 1));
+    }
+  }
 }
 #endif /* !defined(MYSQL_CLIENT) */
 
@@ -12333,7 +12375,8 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len,
     m_colcnt(0), m_coltype(0),
     m_memory(NULL), m_table_id(ULONGLONG_MAX), m_flags(0),
     m_data_size(0), m_field_metadata(0), m_field_metadata_size(0),
-    m_null_bits(0), m_meta_memory(NULL)
+    m_null_bits(0), m_meta_memory(NULL), m_primary_key_fields(0),
+    m_primary_key_fields_size(0)
 {
   unsigned int bytes_read= 0;
   DBUG_ENTER("Table_map_log_event::Table_map_log_event(const char*,uint,...)");
@@ -12420,6 +12463,20 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len,
       memcpy(m_field_metadata, ptr_after_colcnt, m_field_metadata_size);
       ptr_after_colcnt= (uchar*)ptr_after_colcnt + m_field_metadata_size;
       memcpy(m_null_bits, ptr_after_colcnt, num_null_bytes);
+      ptr_after_colcnt= (uchar*)ptr_after_colcnt + num_null_bytes;
+      if ((ptr_after_colcnt - (uchar *)buf) < event_len)
+      {
+        m_primary_key_fields_size = net_field_length(&ptr_after_colcnt);
+        if (m_primary_key_fields_size)
+        {
+          m_primary_key_fields =  (uchar*) my_malloc(m_primary_key_fields_size,
+                                                     MYF(MY_WME));
+          memcpy(m_primary_key_fields, ptr_after_colcnt,
+                 m_primary_key_fields_size);
+          ptr_after_colcnt = (uchar*)ptr_after_colcnt +
+                             m_primary_key_fields_size;
+        }
+      }
     }
   }
 
@@ -12431,6 +12488,7 @@ Table_map_log_event::~Table_map_log_event()
 {
   my_free(m_meta_memory);
   my_free(m_memory);
+  my_free(m_primary_key_fields);
 }
 
 /*
@@ -12709,6 +12767,10 @@ bool Table_map_log_event::write_data_body(IO_CACHE *file)
   uchar mbuf[sizeof(m_field_metadata_size)];
   uchar *const mbuf_end= net_store_length(mbuf, m_field_metadata_size);
 
+  uchar m_size_buf[sizeof(m_primary_key_fields_size)];
+  uchar *const m_size_buf_end = net_store_length(m_size_buf,
+                                                 m_primary_key_fields_size);
+
   return (wrapper_my_b_safe_write(file, dbuf,      sizeof(dbuf)) ||
           wrapper_my_b_safe_write(file, (const uchar*)m_dbnam,   m_dblen+1) ||
           wrapper_my_b_safe_write(file, tbuf,      sizeof(tbuf)) ||
@@ -12717,7 +12779,11 @@ bool Table_map_log_event::write_data_body(IO_CACHE *file)
           wrapper_my_b_safe_write(file, m_coltype, m_colcnt) ||
           wrapper_my_b_safe_write(file, mbuf, (size_t) (mbuf_end - mbuf)) ||
           wrapper_my_b_safe_write(file, m_field_metadata, m_field_metadata_size),
-          wrapper_my_b_safe_write(file, m_null_bits, (m_colcnt + 7) / 8));
+          wrapper_my_b_safe_write(file, m_null_bits, (m_colcnt + 7) / 8) ||
+          wrapper_my_b_safe_write(file, m_size_buf,
+                                  (size_t) (m_size_buf_end - m_size_buf)) ||
+          wrapper_my_b_safe_write(file, m_primary_key_fields,
+                                  m_primary_key_fields_size));
  }
 #endif
 
@@ -12753,6 +12819,21 @@ void Table_map_log_event::print(FILE *, PRINT_EVENT_INFO *print_event_info)
     my_b_printf(&print_event_info->head_cache,
                 "\tTable_map: `%s`.`%s` mapped to number %llu\n",
                 m_dbnam, m_tblnam, m_table_id.id());
+    DBUG_EXECUTE_IF("print_primary_key_fields", {
+                     my_b_printf(&print_event_info->head_cache,
+                                 "#Primary Key Fields: ` ");
+                     if (m_primary_key_fields)
+                     {
+                       uchar *ptr = m_primary_key_fields;
+                       uint n = net_field_length(&ptr);
+                       for (ulong i = 0; i < n; ++i) {
+                         uint field_index = net_field_length(&ptr);
+                         my_b_printf(&print_event_info->head_cache, "%u ",
+                                     field_index);
+                       }
+                     }
+                     my_b_printf(&print_event_info->head_cache, "`\n");
+                     });
     print_base64(&print_event_info->body_cache, print_event_info, TRUE);
   }
 }

diff --git a/sql/log_event.h b/sql/log_event.h
@@ -4018,6 +4018,8 @@ class Table_map_log_event : public Log_event
   ulong          m_field_metadata_size;   
   uchar         *m_null_bits;
   uchar         *m_meta_memory;
+  uchar *m_primary_key_fields;
+  uint m_primary_key_fields_size;
 };