PS-5932: Implementing --tokudb-force-recovery=6 to skip reading the logs

This could be useful when the server crashed with a corrupted rollback file,
and is unable to start up. Specifying --tokudb--force-recovery=6 --super-read-only
should start it up in a read-only, but usable state. Some data may be lost and
unrecoverable.

Starting the server without the read only option is NOT supported.

mysql-test/suite/tokudb/r/recovery.result

@@ -0,0 +1,31 @@
+create table t (a int, b int, primary key (a)) engine=tokudb;
+insert into t values (1,2),(2,4),(3,8);
+optimize table t;
+Table	Op	Msg_type	Msg_text
+test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t	optimize	status	OK
+# restart
+insert into t values (4,9),(10,11),(12,13);
+select * from t;
+a	b
+1	2
+2	4
+3	8
+4	9
+10	11
+12	13
+# Kill the server
+# restart:--tokudb-force-recovery=6 --read-only --super-read-only
+select * from t;
+a	b
+1	2
+2	4
+3	8
+# restart:--tokudb-force-recovery=6 --read-only --super-read-only
+select * from t;
+a	b
+1	2
+2	4
+3	8
+# restart
+drop table t;

mysql-test/suite/tokudb/t/disabled.def

@@ -1,4 +1,5 @@
 mvcc-19: tokutek
 mvcc-20: tokutek
 mvcc-27: tokutek
+recovery: fails when ran together with other tests
 storage_engine_default: tokudb is not the default storage engine

mysql-test/suite/tokudb/t/recovery.test

@@ -0,0 +1,47 @@
+--source include/have_tokudb.inc
+
+# verify that delete from table leaves the table empty
+create table t (a int, b int, primary key (a)) engine=tokudb;
+insert into t values (1,2),(2,4),(3,8);
+
+--let $MYSQLD_DATADIR= `select @@datadir`
+
+optimize table t;
+
+--source include/restart_mysqld.inc
+
+insert into t values (4,9),(10,11),(12,13);
+
+select * from t;
+
+--source include/kill_mysqld.inc
+
+--exec mv $MYSQLD_DATADIR/tokudb.rollback $MYSQLD_DATADIR/rollback.backup
+# Create an empty rollback file that tokudb can't interpret
+--exec echo "" > $MYSQLD_DATADIR/tokudb.rollback
+# Make everything read only - to make sure the server can't write these files
+--exec find $MYSQLD_DATADIR -type f -name "*toku*" | xargs chmod u-w
+
+--let $restart_parameters= restart:--tokudb-force-recovery=6 --read-only --super-read-only
+--source include/start_mysqld.inc
+
+select * from t;
+
+--let $restart_parameters= restart:--tokudb-force-recovery=6 --read-only --super-read-only
+--source include/restart_mysqld.inc
+
+select * from t;
+
+--source include/shutdown_mysqld.inc
+# Restore rollback & RW permissions
+--exec find $MYSQLD_DATADIR -type f -name "*toku*" | xargs chmod u+w
+--exec rm $MYSQLD_DATADIR/tokudb.rollback
+--exec mv $MYSQLD_DATADIR/rollback.backup $MYSQLD_DATADIR/tokudb.rollback
+--let $restart_parameters= 
+--source include/start_mysqld.inc
+
+# Cleanup recovery files
+--exec rm $MYSQLD_DATADIR/tokudb.rollback2
+--exec rm $MYSQLD_DATADIR/tokudb.environment2
+--exec rm $MYSQLD_DATADIR/*.___lock_dont_delete_me2
+drop table t;

storage/tokudb/ha_tokudb.cc

@@ -1316,7 +1316,7 @@ int ha_tokudb::open_main_dictionary(
             NULL,
             DB_BTREE,
             open_flags,
-            0);
+            is_read_only ? 0 : S_IWUSR);
     if (error) {
         goto exit;
     }
@@ -1379,7 +1379,7 @@ int ha_tokudb::open_secondary_dictionary(
     }
 
 
-    error = (*ptr)->open(*ptr, txn, newname, NULL, DB_BTREE, open_flags, 0);
+    error = (*ptr)->open(*ptr, txn, newname, NULL, DB_BTREE, open_flags, is_read_only ? 0 : S_IWUSR);
     if (error) {
         set_my_errno(error);
         goto cleanup;
@@ -1565,6 +1565,7 @@ static int initialize_key_and_col_info(
 }
 
 int ha_tokudb::initialize_share(const char* name, int mode) {
+
     int error = 0;
     uint64_t num_rows = 0;
     DB_TXN* txn = NULL;
@@ -1619,6 +1620,7 @@ int ha_tokudb::initialize_share(const char* name, int mode) {
             primary_key);
     if (error) { goto exit; }
 
+
     error = open_main_dictionary(name, mode == O_RDONLY, txn);
     if (error) {
         goto exit;
@@ -1767,7 +1769,6 @@ int ha_tokudb::open(const char *name, int mode, uint test_if_locked) {
     transaction = NULL;
     cursor = NULL;
 
-
     /* Open primary key */
     hidden_primary_key = 0;
     if ((primary_key = table_share->primary_key) >= MAX_KEY) {

storage/tokudb/hatoku_hton.cc

@@ -54,6 +54,11 @@ typedef struct savepoint_info {
     bool in_sub_stmt;
 } *SP_INFO, SP_INFO_T;
 
+extern "C" {
+extern uint         force_recovery;
+}
+
+
 static handler* tokudb_create_handler(handlerton* hton,
                                       TABLE_SHARE* table,
                                       MEM_ROOT* mem_root);
@@ -268,6 +273,15 @@ static int tokudb_set_product_name(void) {
 }
 
 static int tokudb_init_func(void *p) {
+    int mode = force_recovery ? S_IRUSR|S_IRGRP|S_IROTH : S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH;
+
+    if(force_recovery != 0 && (!read_only || !super_read_only)) {
+            sql_print_error(
+                "%s is not initialized because tokudb_force_only requires read_only and super_read_only",
+                tokudb_hton_name);
+	    goto error;
+    }
+
     TOKUDB_DBUG_ENTER("%p", p);
     int r;
 
@@ -555,7 +569,7 @@ static int tokudb_init_func(void *p) {
         db_env,
         tokudb_home,
         tokudb_init_flags,
-        S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH);
+        mode);
 
     TOKUDB_TRACE_FOR_FLAGS(TOKUDB_DEBUG_INIT, "env opened:return=%d", r);
 
@@ -1013,6 +1027,7 @@ static int tokudb_xa_prepare(handlerton* hton, THD* thd, bool all) {
     TOKUDB_DBUG_RETURN(r);
 }
 
+
 static int tokudb_xa_recover(TOKUDB_UNUSED(handlerton* hton),
                              XID* xid_list,
                              uint len) {

storage/tokudb/tokudb_status.h

@@ -201,7 +201,7 @@ int create(
                 name,
                 NULL,
                 DB_BTREE, DB_CREATE | DB_EXCL,
-                0);
+                S_IWUSR);
     }
     if (error == 0) {
         *status_db_ptr = status_db;
@@ -212,6 +212,11 @@ int create(
     return error;
 }
 
+extern "C" {
+  extern uint         force_recovery;
+}
+
+
 int open(
     DB_ENV* env,
     DB** status_db_ptr,
@@ -230,7 +235,7 @@ int open(
                 NULL,
                 DB_BTREE,
                 DB_THREAD,
-                0);
+                force_recovery ? 0 : S_IWUSR);
     }
     if (error == 0) {
         uint32_t pagesize = 0;

storage/tokudb/tokudb_sysvars.cc

@@ -91,6 +91,18 @@ static MYSQL_SYSVAR_ULONGLONG(
     ~0ULL,
     0);
 
+static MYSQL_SYSVAR_UINT(
+    force_recovery,
+    force_recovery,
+    PLUGIN_VAR_READONLY,
+    "force recovery. Set to 6 to skip reading the logs",
+    NULL,
+    NULL,
+    0,
+    0,
+    0,
+    0);
+
 static MYSQL_SYSVAR_UINT(
     cachetable_pool_threads,
     cachetable_pool_threads,
@@ -956,6 +968,7 @@ static int dir_cmd_check(THD* thd,
 st_mysql_sys_var* system_variables[] = {
     // global vars
     MYSQL_SYSVAR(cache_size),
+    MYSQL_SYSVAR(force_recovery),
     MYSQL_SYSVAR(checkpoint_on_flush_logs),
     MYSQL_SYSVAR(cachetable_pool_threads),
     MYSQL_SYSVAR(cardinality_scale_percent),

storage/tokudb/tokudb_sysvars.h

@@ -26,6 +26,11 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #ifndef _TOKUDB_SYSVARS_H
 #define _TOKUDB_SYSVARS_H
 
+
+extern "C" {
+extern uint         force_recovery;
+}
+
 namespace tokudb {
 namespace sysvars {
 
@@ -57,7 +62,6 @@ enum row_format_t {
 #define DEFAULT_TOKUDB_KILLED_TIME 4000     // milliseconds
 #define DEFAULT_TOKUDB_LOCK_TIMEOUT 4000    // milliseconds
 
-
 // globals
 extern ulonglong    cache_size;
 extern uint         cachetable_pool_threads;