Defragmentation support for partitioned InnoDB table

Summary:
Add defragmentation of partitioned innodb table to Facebook
defragmentation feature.

You can specify some partition name for defragmentation with PARTITION
phrase.

Without PARTITION phrase, InnoDB will defragment all partition of the
table.

And running defragmentation on partition table with async_commit option,
InnoDB will defragment all partitioned table in parallel (Not parallel
threads, just one thread will defragment all indexes). Defragmentation
of several indexes (at the same time) need a lot of disk reads and this
cause the latency increasing of user query processing.

** Usage **
ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1) INDEX PRIMARY;

ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1);

ALTER TABLE t1 DEFRAGMENT INDEX ix_lname;

ALTER TABLE t1 DEFRAGMENT;

ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1) INDEX PRIMARY async_commit;

ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1) async_commit;

ALTER TABLE t1 DEFRAGMENT INDEX ix_lname async_commit;

ALTER TABLE t1 DEFRAGMENT async_commit;

This was contributed as:
#10

Test Plan: TBA

Reviewers: steaphan, over, pengt

Reviewed By: pengt

mysql-test/suite/parts/r/partition_alter_table_defragment.result

@@ -0,0 +1,46 @@
+DROP TABLE if exists t1;
+call mtr.add_suppression("InnoDB: Warning: MySQL is trying to drop table ");
+## Create test table
+CREATE TABLE t1 (
+id INT NOT NULL AUTO_INCREMENT,
+fname CHAR(30) DEFAULT NULL,
+lname CHAR(30) NOT NULL DEFAULT '',
+PRIMARY KEY (id, lname),
+INDEX ix_lname (lname)
+) ENGINE=InnoDB
+PARTITION BY RANGE (id)
+SUBPARTITION BY KEY (lname)
+SUBPARTITIONS 2
+(PARTITION p0 VALUES LESS THAN (100) ENGINE = InnoDB,
+PARTITION p1 VALUES LESS THAN (300) ENGINE = InnoDB,
+PARTITION p2 VALUES LESS THAN (600) ENGINE = InnoDB,
+PARTITION p3 VALUES LESS THAN MAXVALUE ENGINE = InnoDB);
+CREATE TABLE t2 (
+id INT NOT NULL AUTO_INCREMENT,
+fname CHAR(30) DEFAULT NULL,
+lname CHAR(30) NOT NULL DEFAULT '',
+PRIMARY KEY (id, lname),
+INDEX ix_lname (lname)
+) ENGINE=InnoDB
+PARTITION BY RANGE (id)
+(PARTITION p0 VALUES LESS THAN (100) ENGINE = InnoDB,
+PARTITION p1 VALUES LESS THAN (300) ENGINE = InnoDB,
+PARTITION p2 VALUES LESS THAN (600) ENGINE = InnoDB,
+PARTITION p3 VALUES LESS THAN MAXVALUE ENGINE = InnoDB);
+## Populate test table
+## Delete hole rows
+## Test-1 defragment specific sub-partition and index
+ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P0sp1, P1SP1) INDEX PRIMARY;
+## Test-2 defragment specific sub-partition without index
+ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1);
+## Test-3 defragment specific partition and index
+ALTER TABLE t2 DEFRAGMENT PARTITION (p2, P3) INDEX PRIMARY;
+## Test-4 defragment specific partition without index
+ALTER TABLE t2 DEFRAGMENT PARTITION (p1);
+## Test-5 defragment specific index without partition
+ALTER TABLE t1 DEFRAGMENT INDEX ix_lname;
+## Test-6 defragment whole partitioned table
+ALTER TABLE t2 DEFRAGMENT;
+## Clean test table
+DROP TABLE t1;
+DROP TABLE t2;

mysql-test/suite/parts/t/partition_alter_table_defragment.test

@@ -0,0 +1,82 @@
+--source include/have_innodb.inc
+
+--disable_warnings
+DROP TABLE if exists t1;
+--enable_warnings
+
+call mtr.add_suppression("InnoDB: Warning: MySQL is trying to drop table ");
+
+--echo ## Create test table
+CREATE TABLE t1 (
+  id INT NOT NULL AUTO_INCREMENT,
+  fname CHAR(30) DEFAULT NULL,
+  lname CHAR(30) NOT NULL DEFAULT '',
+  PRIMARY KEY (id, lname),
+  INDEX ix_lname (lname)
+) ENGINE=InnoDB
+PARTITION BY RANGE (id)
+SUBPARTITION BY KEY (lname)
+SUBPARTITIONS 2
+(PARTITION p0 VALUES LESS THAN (100) ENGINE = InnoDB,
+ PARTITION p1 VALUES LESS THAN (300) ENGINE = InnoDB,
+ PARTITION p2 VALUES LESS THAN (600) ENGINE = InnoDB,
+ PARTITION p3 VALUES LESS THAN MAXVALUE ENGINE = InnoDB);
+
+CREATE TABLE t2 (
+  id INT NOT NULL AUTO_INCREMENT,
+  fname CHAR(30) DEFAULT NULL,
+  lname CHAR(30) NOT NULL DEFAULT '',
+  PRIMARY KEY (id, lname),
+  INDEX ix_lname (lname)
+) ENGINE=InnoDB
+PARTITION BY RANGE (id)
+(PARTITION p0 VALUES LESS THAN (100) ENGINE = InnoDB,
+ PARTITION p1 VALUES LESS THAN (300) ENGINE = InnoDB,
+ PARTITION p2 VALUES LESS THAN (600) ENGINE = InnoDB,
+ PARTITION p3 VALUES LESS THAN MAXVALUE ENGINE = InnoDB);
+
+--echo ## Populate test table
+--disable_query_log
+INSERT INTO t1 VALUES (NULL, 'matt', 'lee'), (NULL, 'lara', 'kim'), (NULL, 'seonguck', 'ryu');
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+INSERT INTO t1 SELECT NULL, fname, lname FROM t1;
+
+INSERT INTO t2 SELECT * FROM t1;
+
+--echo ## Delete hole rows
+let $delete_rows = 150;
+while($delete_rows)
+{
+    DELETE FROM t1 WHERE id = ROUND(RAND()*1000) % 768;
+    DELETE FROM t2 WHERE id = ROUND(RAND()*1000) % 768;
+    dec $delete_rows;
+}
+--enable_query_log
+
+--echo ## Test-1 defragment specific sub-partition and index
+ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P0sp1, P1SP1) INDEX PRIMARY;
+
+--echo ## Test-2 defragment specific sub-partition without index
+ALTER TABLE t1 DEFRAGMENT PARTITION (p0sp0, P1SP1);
+
+-- echo ## Test-3 defragment specific partition and index
+ALTER TABLE t2 DEFRAGMENT PARTITION (p2, P3) INDEX PRIMARY;
+
+-- echo ## Test-4 defragment specific partition without index
+ALTER TABLE t2 DEFRAGMENT PARTITION (p1);
+
+--echo ## Test-5 defragment specific index without partition
+ALTER TABLE t1 DEFRAGMENT INDEX ix_lname;
+
+--echo ## Test-6 defragment whole partitioned table
+ALTER TABLE t2 DEFRAGMENT;
+
+--echo ## Clean test table
+DROP TABLE t1;
+DROP TABLE t2;

sql/ha_partition.cc

@@ -613,6 +613,96 @@ int ha_partition::rename_table(const char *from, const char *to)
   DBUG_RETURN(del_ren_table(from, to));
 }
 
+bool ha_partition::should_defragment_partition(List<String>& defrag_parts,
+                                               const char* part_name)
+{
+  uint req_parts = defrag_parts.elements;
+  if (req_parts == 0) {
+    return true;
+  }
+  String* req_part_name;
+  List_iterator<String> req_part_it(defrag_parts);
+  while((req_part_name = req_part_it++)) {
+    const char* req_part_name_cptr = req_part_name->c_ptr();
+    if(!my_strcasecmp(system_charset_info, part_name,
+                      req_part_name_cptr)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+int ha_partition::defragment_partition(const char* table_name,
+                                       const char* part_name,
+                                       const char* subpart_name,
+                                       const char* index_name,
+                                       int part)
+{
+  char part_name_buff[FN_REFLEN];
+  if (subpart_name)
+    create_subpartition_name(part_name_buff, table_name, part_name,
+                             subpart_name, NORMAL_PART_NAME);
+  else
+    create_partition_name(part_name_buff, table_name, part_name,
+                          NORMAL_PART_NAME, true);
+
+  return m_file[part]->ha_defragment_table(part_name_buff, index_name, NULL);
+}
+
+/*
+  Defragment table
+
+  SYNOPSIS
+    defragment_table()
+    name                   Table name
+    index_name             Index name
+    alter_info             Altering info for partition selection
+
+  RETURN VALUES
+    >0                Error
+    0                 Success
+*/
+int ha_partition::defragment_table(const char* name, const char* index_name,
+                                   Alter_info* alter_info)
+{
+  int error = 0;
+  uint num_parts = m_part_info->partitions.elements;
+
+  List_iterator<partition_element> part_it(m_part_info->partitions);
+
+  DBUG_ENTER("ha_partition::defragment");
+
+  for (uint i = 0; i < num_parts; i++) {
+    partition_element *part_elem = part_it++;
+    if (m_is_sub_partitioned) {
+      List_iterator<partition_element> sub_it(part_elem->subpartitions);
+      uint num_subparts = m_part_info->num_subparts;
+      uint part;
+      for (uint j = 0; j < num_subparts; j++) {
+        partition_element *sub_elem = sub_it++;
+        part = i * num_subparts + j;
+        if (should_defragment_partition(alter_info->defrag_parts,
+                                        sub_elem->partition_name)) {
+          if ((error = defragment_partition(name, part_elem->partition_name,
+                                            sub_elem->partition_name,
+                                            index_name, part))) {
+            DBUG_RETURN(error);
+          }
+        }
+      }
+    } else {
+      if (should_defragment_partition(alter_info->defrag_parts,
+                                      part_elem->partition_name)) {
+        if ((error = defragment_partition(name, part_elem->partition_name,
+                                          NULL, index_name, i))) {
+          DBUG_RETURN(error);
+        }
+      }
+    }
+  }
+
+  DBUG_RETURN(0);
+}
 
 /*
   Create the handler file (.par-file)

sql/ha_partition.h

@@ -317,6 +317,8 @@ class ha_partition :public handler
   */
   virtual int delete_table(const char *from);
   virtual int rename_table(const char *from, const char *to);
+  virtual int defragment_table(const char* name, const char* index_name,
+                               Alter_info* alter_info);
   virtual int create(const char *name, TABLE *form,
 		     HA_CREATE_INFO *create_info);
   virtual int create_handler_files(const char *name,
@@ -342,6 +344,11 @@ class ha_partition :public handler
   virtual bool check_if_incompatible_data(HA_CREATE_INFO *create_info,
                                           uint table_changes);
 private:
+  int defragment_partition(const char* table_name, const char* part_name,
+                           const char* subpart_name, const char* index_name,
+                           int part);
+  bool should_defragment_partition(List<String>& defrag_parts,
+                                   const char*part_name);
   int copy_partitions(ulonglong * const copied, ulonglong * const deleted);
   void cleanup_new_partition(uint part_count);
   int prepare_new_partition(TABLE *table, HA_CREATE_INFO *create_info,

sql/handler.cc

@@ -4689,12 +4689,13 @@ handler::ha_drop_table(const char *name)
 */
 
 int
-handler::ha_defragment_table(const char *name, const char *index)
+handler::ha_defragment_table(const char *name, const char *index,
+                             Alter_info* alter_info)
 {
   DBUG_ASSERT(m_lock_type == F_UNLCK);
   mark_trx_read_write();
 
-  return defragment_table(name, index);
+  return defragment_table(name, index, alter_info);
 }
 
 

sql/handler.h

@@ -2116,7 +2116,8 @@ class handler :public Sql_alloc
   int ha_rename_table(const char *from, const char *to);
   int ha_delete_table(const char *name);
   void ha_drop_table(const char *name);
-  int ha_defragment_table(const char *name, const char *index);
+  int ha_defragment_table(const char *name, const char *index,
+                          Alter_info* alter_info);
 
   int ha_create(const char *name, TABLE *form, HA_CREATE_INFO *info);
 
@@ -3325,7 +3326,8 @@ class handler :public Sql_alloc
   virtual int discard_or_import_tablespace(my_bool discard)
   { return (my_errno=HA_ERR_WRONG_COMMAND); }
   virtual void drop_table(const char *name);
-  virtual int defragment_table(const char *name, const char *index)
+  virtual int defragment_table(const char *name, const char *index,
+                               Alter_info* alter_info)
   { return HA_ADMIN_NOT_IMPLEMENTED; }
   virtual int create(const char *name, TABLE *form, HA_CREATE_INFO *info)=0;
 

sql/sql_alter.cc

@@ -433,7 +433,8 @@ bool Sql_cmd_defragment_table::execute(THD *thd)
                                      first_table->alias, "", 0);
   LEX_STRING index = thd->lex->alter_info.defrag_index;
   THD_STAGE_INFO(thd, stage_alter_inplace);
-  int ret = handler->ha_defragment_table(path, index.str);
+  int ret = handler->ha_defragment_table(path, index.str,
+                                         &thd->lex->alter_info);
   DEBUG_SYNC(thd, "defragment_after_defrag");
   close_thread_tables(thd);
   if (ret == 0)

sql/sql_alter.h

@@ -194,6 +194,8 @@ class Alter_info
   enum_alter_table_lock         requested_lock;
   // Defragment index.
   LEX_STRING                    defrag_index;
+  // Defragment partition
+  List<String>                  defrag_parts;
 
   Alter_info() :
     flags(0),
@@ -210,6 +212,7 @@ class Alter_info
     alter_list.empty();
     key_list.empty();
     create_list.empty();
+    defrag_parts.empty();
     flags= 0;
     keys_onoff= LEAVE_AS_IS;
     num_parts= 0;

sql/sql_yacc.yy

@@ -7907,11 +7907,22 @@ index_defragmentation:
         ;
 
 defragment:
-          DEFRAGMENT_SYM index_defragmentation
+          DEFRAGMENT_SYM opt_use_partition index_defragmentation
           {
             THD *thd= YYTHD;
             Lex->m_sql_cmd= new (thd->mem_root)
               Sql_cmd_defragment_table();
+
+            /* $2 would be null if alter statement does not have
+            PARTITION (x, y, ...) phrase */
+            if($2)
+            {
+              List_iterator<String> it(*$2);
+              String *tmp;
+              while ((tmp= it++))
+                  Lex->alter_info.defrag_parts.push_back(tmp);
+            }
+
             if (Lex->m_sql_cmd == NULL)
               MYSQL_YYABORT;
           }

storage/innobase/btr/btr0defragment.cc

@@ -212,7 +212,7 @@ btr_defragment_remove_table(
 		btr_pcur_t* pcur = item->pcur;
 		btr_cur_t* cursor = btr_pcur_get_btr_cur(pcur);
 		dict_index_t* idx = btr_cur_get_index(cursor);
-		if (table->id == idx->table->id) {
+		if (!item->removed && table->id == idx->table->id) {
 			item->removed = true;
 		}
 	}

storage/innobase/handler/ha_innodb.cc

@@ -11195,7 +11195,8 @@ int
 ha_innobase::defragment_table(
 /*======================*/
 	const char*	name,		/*!< in: table name */
-	const char*	index_name)	/*!< in: index name */
+	const char*	index_name,	/*!< in: index name */
+	Alter_info* alter_info)		/*!< in: not used */
 {
 	char		norm_name[FN_REFLEN];
 	dict_table_t*	table;

storage/innobase/handler/ha_innodb.h

@@ -190,7 +190,8 @@ class ha_innobase: public handler
 	int truncate();
 	int delete_table(const char *name);
 	int rename_table(const char* from, const char* to);
-	int defragment_table(const char* name, const char* index_name);
+	int defragment_table(const char* name, const char* index_name,
+			     Alter_info* alter_info);
 	int check(THD* thd, HA_CHECK_OPT* check_opt);
 	char* update_table_comment(const char* comment);
 	char* get_foreign_key_create_info();