diff --git a/mysql-test/r/percona_8_0_15_dd_upgrade_encrypted.result b/mysql-test/r/percona_8_0_15_dd_upgrade_encrypted.result new file mode 100644 index 000000000000..c3b71a03c514 --- /dev/null +++ b/mysql-test/r/percona_8_0_15_dd_upgrade_encrypted.result @@ -0,0 +1,14 @@ +# Set different paths for --datadir +# Check that the file exists in the working folder. +# Unzip the zip file. +# Stop DB server which was created by MTR default +# Start the 8.0.16 server on 8.0.15 datadir +# restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=MYSQLD_DATADIR1 --keyring_file_data=MYSQLD_DATADIR1/mysecret_keyring KEYRING_PLUGIN_OPT KEYRING_PLUGIN_EARLY_LOAD +include/assert.inc [Encryption of mysql.ibd should be set to Y.] +SET SESSION DEBUG = '+d,skip_dd_table_access_check'; +include/assert.inc [dd_properties should not contain information about encryption] +include/assert.inc [All tables in mysql tablespace should have encrypt_type=Y set in options.] +# restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=MYSQLD_DATADIR1 --keyring_file_data=MYSQLD_DATADIR1/mysecret_keyring KEYRING_PLUGIN_OPT KEYRING_PLUGIN_EARLY_LOAD +# Remove copied files +# Restart the server with default options. +# restart diff --git a/mysql-test/std_data/data_8015_dd_encrypted.zip b/mysql-test/std_data/data_8015_dd_encrypted.zip new file mode 100644 index 000000000000..2234d0c8741d Binary files /dev/null and b/mysql-test/std_data/data_8015_dd_encrypted.zip differ diff --git a/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted-master.opt b/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted-master.opt new file mode 100644 index 000000000000..5df3ea9cdec1 --- /dev/null +++ b/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted-master.opt @@ -0,0 +1 @@ +--initialize --innodb_page_size=16k diff --git a/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted.test b/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted.test new file mode 100644 index 000000000000..72fbe06eedf7 --- /dev/null +++ b/mysql-test/t/percona_8_0_15_dd_upgrade_encrypted.test @@ -0,0 +1,98 @@ +# This test contains keyring file created on 64bit which is not +# portable +--source include/have_64bit.inc +--source include/no_valgrind_without_big.inc +--source include/have_util_unzip.inc + +--disable_query_log +call mtr.add_suppression("Resizing redo log from"); +call mtr.add_suppression("Upgrading redo log"); +call mtr.add_suppression("Starting to delete and rewrite log files"); +call mtr.add_suppression("New log files created"); +call mtr.add_suppression("Unknown system variable 'show_compatibility_56'"); +call mtr.add_suppression("You need to use --log-bin to make --binlog-format work"); +call mtr.add_suppression("Creating routine without parsing routine body"); +call mtr.add_suppression("Resolving dependency for the view"); +call mtr.add_suppression("references invalid"); +call mtr.add_suppression("doesn't exist"); +call mtr.add_suppression("information_schema"); +call mtr.add_suppression("Storage engine '.*' does not support system tables. \\[mysql.*\\]"); +call mtr.add_suppression("Table 'mysql.component' doesn't exist"); +call mtr.add_suppression("is expected to be transactional"); +call mtr.add_suppression("table is missing or has an incorrect definition"); +call mtr.add_suppression("ACL DDLs will not work unless mysql_upgrade is executed"); +call mtr.add_suppression("Native table .* has the wrong structure"); +call mtr.add_suppression("Column count of mysql.* is wrong"); +call mtr.add_suppression("Cannot open table mysql/version from the internal data dictionary of InnoDB though the .frm file for the table exists"); +call mtr.add_suppression("Column count of performance_schema.events_statements_summary_by_digest is wrong"); +call mtr.add_suppression("The privilege system failed to initialize correctly"); +call mtr.add_suppression("Missing system table mysql.global_grants"); +# InnoDB reports "Lock wait timeout" warnings when it tries to drop persistent +# statistics while persistent statistics table is altered during upgrade. +# This issue doesn't seem to cause any further trouble (as there is no persistent +# stats for persistent stats table anyway), so we ignore these warnings here. +call mtr.add_suppression("Unable to delete statistics for table mysql."); +# new fields were added to these tables +call mtr.add_suppression("Column count of performance_schema.replication_group_members is wrong. Expected 7, found 5"); +call mtr.add_suppression("Column count of performance_schema.replication_group_member_stats is wrong. Expected 13, found 9"); +call mtr.add_suppression("Column count of performance_schema.threads is wrong. Expected 18, found 17"); +call mtr.add_suppression("ACL table mysql.[a-zA-Z_]* missing. Some operations may fail."); +call mtr.add_suppression("Info table is not ready to be used. Table 'mysql.slave_master_info' cannot be opened"); +call mtr.add_suppression("Error in checking mysql.slave_master_info repository info type of TABLE"); +call mtr.add_suppression("Error creating master info: Error checking repositories."); +call mtr.add_suppression("Slave: Failed to initialize the master info structure for channel"); +call mtr.add_suppression("Failed to create or recover replication info repositories."); +call mtr.add_suppression("db.opt file not found for test database. Using default Character set"); +call mtr.add_suppression("Skip re-populating collations and character sets tables in InnoDB read-only mode"); +call mtr.add_suppression("Skipped updating resource group metadata in InnoDB read only mode"); +--enable_query_log + +--echo # Set different paths for --datadir +let $MYSQLD_DATADIR1 = $MYSQL_TMP_DIR/data_8015_dd_encrypted; + +--copy_file $MYSQLTEST_VARDIR/std_data/data_8015_dd_encrypted.zip $MYSQL_TMP_DIR/data_8015_dd_encrypted.zip + +--echo # Check that the file exists in the working folder. +--file_exists $MYSQL_TMP_DIR/data_8015_dd_encrypted.zip + +--echo # Unzip the zip file. +--exec unzip -qo $MYSQL_TMP_DIR/data_8015_dd_encrypted.zip -d $MYSQL_TMP_DIR + +--let $MYSQLD_DATADIR=`SELECT @@datadir` + +--echo # Stop DB server which was created by MTR default +--source include/shutdown_mysqld.inc + +--echo # Start the 8.0.16 server on 8.0.15 datadir +--let $restart_parameters = "restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=$MYSQLD_DATADIR1 --keyring_file_data=$MYSQL_TMP_DIR/data_8015_dd_encrypted/mysecret_keyring $KEYRING_PLUGIN_OPT $KEYRING_PLUGIN_EARLY_LOAD" +--replace_result $MYSQLD_DATADIR1 MYSQLD_DATADIR1 $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT KEYRING_PLUGIN_OPT $KEYRING_PLUGIN_EARLY_LOAD KEYRING_PLUGIN_EARLY_LOAD +--source include/start_mysqld.inc + +--let $assert_text = Encryption of mysql.ibd should be set to Y. +--let $assert_cond= "[SELECT ENCRYPTION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES WHERE NAME=\"mysql\"]" = "Y" +--source include/assert.inc + +SET SESSION DEBUG = '+d,skip_dd_table_access_check'; + +--let $assert_text = dd_properties should not contain information about encryption +--let $assert_cond= "[SELECT COUNT(*) FROM mysql.dd_properties WHERE properties LIKE \'%=ENCRYPTION%\']" = 0 +--source include/assert.inc + +--let $assert_text = All tables in mysql tablespace should have encrypt_type=Y set in options. +--let $assert_cond= "[SELECT COUNT(*) FROM mysql.tables WHERE tablespace_id = 1 AND options NOT LIKE \'%encrypt_type=Y%\']" = 0 +--source include/assert.inc + +# Check if server restarts correctly +--replace_result $MYSQLD_DATADIR1 MYSQLD_DATADIR1 $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT KEYRING_PLUGIN_OPT $KEYRING_PLUGIN_EARLY_LOAD KEYRING_PLUGIN_EARLY_LOAD +--source include/restart_mysqld.inc + +--source include/shutdown_mysqld.inc + +--echo # Remove copied files +--remove_file $MYSQL_TMP_DIR/data_8015_dd_encrypted.zip + +--force-rmdir $MYSQL_TMP_DIR/data_8015_dd_encrypted + +--echo # Restart the server with default options. +--let $restart_parameters= +--source include/start_mysqld.inc diff --git a/sql/dd/impl/bootstrap/bootstrapper.cc b/sql/dd/impl/bootstrap/bootstrapper.cc index 9b564821dc97..fe2ffe26a991 100644 --- a/sql/dd/impl/bootstrap/bootstrapper.cc +++ b/sql/dd/impl/bootstrap/bootstrapper.cc @@ -165,6 +165,9 @@ bool update_system_tables(THD *thd) { std::unique_ptr table_def_properties( Properties::parse_properties(def)); table_def->set_actual_table_definition(*table_def_properties); + if (bootstrap::DD_bootstrap_ctx::instance().is_dd_encrypted()) { + table_def->set_actual_encrypted(); + } } } @@ -1198,7 +1201,7 @@ bool create_dd_schema(THD *thd) { bool initialize_dd_properties(THD *thd) { // Create the dd_properties table. if (bootstrap::DD_bootstrap_ctx::instance().is_dd_encrypted()) { - dd::tables::DD_properties::instance().set_encrypted(); + dd::tables::DD_properties::instance().set_target_encrypted(); } const Object_table_definition *dd_properties_def = dd::tables::DD_properties::instance().target_table_definition(); @@ -1673,6 +1676,25 @@ bool sync_meta_data(THD *thd) { return false; } +// Helper guard used in update_properties, to be sure +// that encryption will get set back before +// update_properties exits. +struct Target_encryption_guard { + public: + Target_encryption_guard(const Object_table *object_table) + : m_object_table(object_table) + : set_encryption(object_table->is_target_encrypted()) {} + ~Target_encryption_guard() { + if (set_encryption) { + m_object_table->set_target_encrypted(); + } + } + + private: + const Object_table *m_object_table; + bool set_encryption; +}; + bool update_properties(THD *thd, const std::set *create_set, const std::set *remove_set, const String_type &target_table_schema_name) { @@ -1692,6 +1714,22 @@ bool update_properties(THD *thd, const std::set *create_set, will have a corresponding Object_table. */ DBUG_ASSERT((*it)->entity() != nullptr); + + /* + Percona Server supports mysql.ibd encryption in earlier versions than + upstream. Upstream started supporting it since 8.0.16. Upstream when + ALTER TABLESPACE mysql ENCRYPTION='Y' is issued does not update + dd_properties table that is updated here. To be in sync with upstream we + also do not want to update dd_properties. Since dd_properties are + updated based on target definition we unset the encryption from target + definition for the time of updating dd_properties. The exact field that + contains system tables properties in dd_properties is SYSTEM_TABLES. + */ + Target_encryption_guard target_encryption_guard((*it)->entity()); + if ((*it)->entity()->is_target_encrypted()) { + (*it)->entity()->unset_target_encrypted(); + } + const Object_table_definition *table_def = (*it)->entity()->target_table_definition(); diff --git a/sql/dd/impl/types/object_table_definition_impl.h b/sql/dd/impl/types/object_table_definition_impl.h index 67f5ae83f58e..b9ff442e0d92 100644 --- a/sql/dd/impl/types/object_table_definition_impl.h +++ b/sql/dd/impl/types/object_table_definition_impl.h @@ -118,6 +118,18 @@ class Object_table_definition_impl : public Object_table_definition { (*element_definitions)[element_number] = element_definition; } + void remove_element(const String_type &element_name, + Element_numbers *element_numbers, + Element_definitions *element_definitions) { + DBUG_ASSERT(element_numbers != nullptr && + element_numbers->find(element_name) != element_numbers->end() && + element_definitions->find((*element_numbers)[element_name]) != + element_definitions->end()); + + element_definitions->erase((*element_numbers)[element_name]); + element_numbers->erase(element_name); + } + int element_number(const String_type &element_name, const Element_numbers &element_numbers) const { DBUG_ASSERT(element_numbers.find(element_name) != element_numbers.end()); @@ -284,6 +296,15 @@ class Object_table_definition_impl : public Object_table_definition { &m_option_numbers, &m_option_definitions); } + virtual void remove_option(const String_type &option_name) { + remove_element(option_name, &m_option_numbers, &m_option_definitions); + } + + virtual bool has_option(int option_number) const { + return m_option_definitions.find(option_number) != + m_option_definitions.end(); + } + virtual void add_populate_statement(const String_type &statement) { m_dml_statements.push_back(statement); } diff --git a/sql/dd/impl/types/object_table_impl.cc b/sql/dd/impl/types/object_table_impl.cc index 186208be77ff..164044680658 100644 --- a/sql/dd/impl/types/object_table_impl.cc +++ b/sql/dd/impl/types/object_table_impl.cc @@ -57,9 +57,26 @@ Object_table_impl::Object_table_impl() String_type("TABLESPACE=") + String_type(MYSQL_TABLESPACE_NAME.str)); } -void Object_table_impl::set_encrypted() { - m_target_def.add_option(static_cast(Common_option::ENCRYPTION), - "ENCRYPTION", "ENCRYPTION='Y'"); +bool Object_table_impl::is_target_encrypted() const { + return m_target_def.has_option(static_cast(Common_option::ENCRYPTION)); +} + +void Object_table_impl::unset_target_encrypted() const { + m_target_def.remove_option("ENCRYPTION"); +} + +void Object_table_impl::set_target_encrypted() const { + if (!m_target_def.has_option(static_cast(Common_option::ENCRYPTION))) { + m_target_def.add_option(static_cast(Common_option::ENCRYPTION), + "ENCRYPTION", "ENCRYPTION='Y'"); + } +} + +void Object_table_impl::set_actual_encrypted() const { + if (!m_actual_def.has_option(static_cast(Common_option::ENCRYPTION))) { + m_actual_def.add_option(static_cast(Common_option::ENCRYPTION), + "ENCRYPTION", "ENCRYPTION='Y'"); + } } bool Object_table_impl::set_actual_table_definition( diff --git a/sql/dd/impl/types/object_table_impl.h b/sql/dd/impl/types/object_table_impl.h index 4238a1e64c4b..9b4a682deb81 100644 --- a/sql/dd/impl/types/object_table_impl.h +++ b/sql/dd/impl/types/object_table_impl.h @@ -35,7 +35,7 @@ namespace dd { class Object_table_impl : virtual public Object_table { protected: mutable uint m_last_dd_version; - Object_table_definition_impl m_target_def; + mutable Object_table_definition_impl m_target_def; mutable bool m_actual_present; mutable Object_table_definition_impl m_actual_def; bool m_hidden; @@ -123,7 +123,13 @@ class Object_table_impl : virtual public Object_table { virtual void set_hidden(bool hidden) { m_hidden = hidden; } - virtual void set_encrypted(); + virtual bool is_target_encrypted() const; + + virtual void unset_target_encrypted() const; + + virtual void set_target_encrypted() const; + + virtual void set_actual_encrypted() const; virtual ~Object_table_impl() {} }; diff --git a/sql/dd/types/object_table.h b/sql/dd/types/object_table.h index 39256285443b..990450af4d5f 100644 --- a/sql/dd/types/object_table.h +++ b/sql/dd/types/object_table.h @@ -175,7 +175,13 @@ class Object_table { */ virtual void set_hidden(bool hidden) = 0; - virtual void set_encrypted() = 0; + virtual bool is_target_encrypted() const = 0; + + virtual void unset_target_encrypted() const = 0; + + virtual void set_target_encrypted() const = 0; + + virtual void set_actual_encrypted() const = 0; public: virtual ~Object_table() {} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index a128b735829d..4ec68750c8ec 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -12812,10 +12812,10 @@ static bool innobase_ddse_dict_init( /* Options and tablespace are set at the SQL layer. */ if (is_dd_encrypted) { - innodb_dynamic_metadata->set_encrypted(); - innodb_table_stats->set_encrypted(); - innodb_index_stats->set_encrypted(); - innodb_ddl_log->set_encrypted(); + innodb_dynamic_metadata->set_target_encrypted(); + innodb_table_stats->set_target_encrypted(); + innodb_index_stats->set_target_encrypted(); + innodb_ddl_log->set_target_encrypted(); } tables->push_back(innodb_dynamic_metadata);