PS-4976: InnoDB undo log encryption

This commit implements undo log encryption loosely based on WL#9289 (71e656a) for MySQL 8.0.
Undo log encryption requires the use of separate undo tablepsaces with innodb_undo_tablespaces=N.

mysql-test/suite/innodb/r/undo_encrypt_basic.result

@@ -0,0 +1,31 @@
+# create bootstrap file
+# Stop the MTR default DB server
+# Run the bootstrap command with keyring
+# Search for particular string of encryption metadata, should success since it's encrypted.
+Pattern "lCB" found
+# Start the DB server with undo log encryption disabled and keyring plugin loaded. It should success.
+INSTALL PLUGIN keyring_file SONAME 'keyring_file.so';
+ERROR HY000: Function 'keyring_file' already exists
+SET GLOBAL innodb_undo_log_encrypt = ON;
+CREATE TABLE tab1(c1 INT, c2 VARCHAR(30));
+START TRANSACTION;
+INSERT INTO tab1 VALUES (100,REPEAT('a',5)),(200,REPEAT('b',5));
+SELECT * FROM tab1;
+c1	c2
+100	aaaaa
+200	bbbbb
+COMMIT;
+SET GLOBAL innodb_undo_log_encrypt = OFF;
+START TRANSACTION;
+INSERT INTO tab1 VALUES (300,REPEAT('a',5)),(400,REPEAT('b',5));
+COMMIT;
+SELECT * FROM tab1;
+c1	c2
+100	aaaaa
+200	bbbbb
+300	aaaaa
+400	bbbbb
+UNINSTALL PLUGIN keyring_file;
+DROP TABLE tab1;
+Pattern "lCB" found
+Pattern "lCB" found

mysql-test/suite/innodb/t/undo_encrypt_basic.test

@@ -0,0 +1,111 @@
+--source include/not_embedded.inc
+--source include/have_innodb.inc
+--source include/big_test.inc
+
+--disable_query_log
+call mtr.add_suppression("\\[Error\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Can't set undo log tablespace to be encrypted.");
+# See Bug #25446984: INNODB.UNDO_ENCRYPT_BOOTSTRAP HAS OCCASIONAL FAILURES ON PB2
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Can't rotate encryption on undo tablespace number");
+--enable_query_log
+
+let $MYSQLD_BASEDIR= `select @@basedir`;
+let $START_PAGE_SIZE= `select @@innodb_page_size`;
+let $LOG_FILE_SIZE= `select @@innodb_log_file_size`;
+
+# Create path for ibdata* & undo* files both DBs
+--mkdir $MYSQL_TMP_DIR/innodb_undo_data_dir
+--mkdir $MYSQL_TMP_DIR/innodb_data_home_dir
+--mkdir $MYSQL_TMP_DIR/datadir
+
+# Set path for --datadir
+let $MYSQLD_DATADIR = $MYSQL_TMP_DIR/datadir/data;
+
+# Set path for undo* files.
+let $MYSQLD_UNDO_DATADIR = $MYSQL_TMP_DIR/innodb_undo_data_dir;
+
+# Set path for ibdata* files.
+let $MYSQLD_HOME_DATA_DIR = $MYSQL_TMP_DIR/innodb_data_home_dir;
+
+let BOOTSTRAP_SQL=$MYSQL_TMP_DIR/boot.sql;
+
+--echo # create bootstrap file
+write_file $BOOTSTRAP_SQL;
+CREATE DATABASE test;
+EOF
+
+--echo # Stop the MTR default DB server
+--source include/shutdown_mysqld.inc
+
+# Remove residue files
+--force-rmdir $MYSQL_TMP_DIR/datadir
+--force-rmdir $MYSQL_TMP_DIR/innodb_data_home_dir
+--force-rmdir $MYSQL_TMP_DIR/innodb_undo_data_dir
+
+# Create path for ibdata* & undo* files both DBs
+--mkdir $MYSQL_TMP_DIR/innodb_undo_data_dir
+--mkdir $MYSQL_TMP_DIR/innodb_data_home_dir
+--mkdir $MYSQL_TMP_DIR/datadir
+
+# Test: bootstrap with undo encryption and with keyring plugin, it should
+# success, then restart with no undo encryption.
+let NEW_CMD = $MYSQLD --no-defaults --initialize-insecure --innodb_log_file_size=$LOG_FILE_SIZE --innodb_page_size=$START_PAGE_SIZE --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR --innodb_undo_directory=$MYSQLD_UNDO_DATADIR --innodb_undo_tablespaces=2 --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATADIR --init-file=$BOOTSTRAP_SQL --innodb_undo_log_encrypt=ON --secure-file-priv="" --early-plugin-load="keyring_file=$KEYRING_PLUGIN" --keyring_file_data=$MYSQL_TMP_DIR/my_key_undo5 $KEYRING_PLUGIN_OPT </dev/null>>$MYSQLTEST_VARDIR/tmp/bootstrap2.log 2>&1;
+
+--echo # Run the bootstrap command with keyring
+--exec $NEW_CMD
+
+--echo # Search for particular string of encryption metadata, should success since it's encrypted.
+let SEARCH_FILE= $MYSQLD_UNDO_DATADIR/undo001;
+let SEARCH_PATTERN= lCB;
+--source include/search_pattern.inc
+
+--echo # Start the DB server with undo log encryption disabled and keyring plugin loaded. It should success.
+--let $restart_parameters="restart: --early-plugin-load="keyring_file=$KEYRING_PLUGIN" $KEYRING_PLUGIN_OPT --keyring_file_data=$MYSQL_TMP_DIR/my_key_undo5 --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR --innodb_undo_directory=$MYSQLD_UNDO_DATADIR --datadir=$MYSQLD_DATADIR --innodb_page_size=$START_PAGE_SIZE --innodb_log_file_size=$LOG_FILE_SIZE --innodb_undo_tablespaces=2"
+--replace_result $MYSQL_TMP_DIR TMP_DIR $KEYRING_PLUGIN_OPT --plugin-dir=KEYRING_PLUGIN_PATH $MYSQLD_HOME_DATA_DIR HOME_DIR $MYSQLD_UNDO_DATADIR UNDO_DATADIR $MYSQLD_DATADIR DATADIR $START_PAGE_SIZE PAGE_SIZE $LOG_FILE_SIZE LOG_FILE_SIZE
+--source include/start_mysqld_no_echo.inc
+
+--replace_regex /\.dll/.so/
+--error ER_UDF_EXISTS
+eval INSTALL PLUGIN keyring_file SONAME '$KEYRING_PLUGIN';
+
+SET GLOBAL innodb_undo_log_encrypt = ON;
+--sleep 2
+
+CREATE TABLE tab1(c1 INT, c2 VARCHAR(30));
+START TRANSACTION;
+INSERT INTO tab1 VALUES (100,REPEAT('a',5)),(200,REPEAT('b',5));
+SELECT * FROM tab1;
+COMMIT;
+SET GLOBAL innodb_undo_log_encrypt = OFF;
+START TRANSACTION;
+INSERT INTO tab1 VALUES (300,REPEAT('a',5)),(400,REPEAT('b',5));
+COMMIT;
+SELECT * FROM tab1;
+
+UNINSTALL PLUGIN keyring_file;
+# Cleanup
+DROP TABLE tab1;
+
+--source include/shutdown_mysqld.inc
+# Search for particular string to confirm the encryption metadata is
+# stored.
+--sleep 2
+let SEARCH_FILE= $MYSQLD_UNDO_DATADIR/undo001;
+let SEARCH_PATTERN= lCB;
+--source include/search_pattern.inc
+
+let SEARCH_FILE= $MYSQLD_UNDO_DATADIR/undo002;
+let SEARCH_PATTERN= lCB;
+--source include/search_pattern.inc
+
+# restart the server with MTR default
+--let $restart_parameters=
+--source include/start_mysqld_no_echo.inc
+#--source include/restart_mysqld_no_echo.inc
+
+--remove_file $BOOTSTRAP_SQL
+
+# Remove residue files
+--force-rmdir $MYSQL_TMP_DIR/datadir
+--force-rmdir $MYSQL_TMP_DIR/innodb_data_home_dir
+--force-rmdir $MYSQL_TMP_DIR/innodb_undo_data_dir

mysql-test/suite/sys_vars/r/innodb_undo_log_encrypt_basic.result

@@ -0,0 +1,61 @@
+SET @start_global_value = @@global.innodb_undo_log_encrypt;
+SELECT @start_global_value;
+@start_global_value
+0
+select @@global.innodb_undo_log_encrypt in (0, 1);
+@@global.innodb_undo_log_encrypt in (0, 1)
+1
+select @@global.innodb_undo_log_encrypt;
+@@global.innodb_undo_log_encrypt
+0
+select @@session.innodb_undo_log_encrypt;
+ERROR HY000: Variable 'innodb_undo_log_encrypt' is a GLOBAL variable
+show global variables like 'innodb_undo_log_encrypt';
+Variable_name	Value
+innodb_undo_log_encrypt	OFF
+show session variables like 'innodb_undo_log_encrypt';
+Variable_name	Value
+innodb_undo_log_encrypt	OFF
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+set global innodb_undo_log_encrypt=1;
+select @@global.innodb_undo_log_encrypt;
+@@global.innodb_undo_log_encrypt
+0
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+set @@global.innodb_undo_log_encrypt=0;
+select @@global.innodb_undo_log_encrypt;
+@@global.innodb_undo_log_encrypt
+0
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+VARIABLE_NAME	VARIABLE_VALUE
+innodb_undo_log_encrypt	OFF
+set session innodb_undo_log_encrypt='some';
+ERROR HY000: Variable 'innodb_undo_log_encrypt' is a GLOBAL variable and should be set with SET GLOBAL
+set @@session.innodb_undo_log_encrypt='some';
+ERROR HY000: Variable 'innodb_undo_log_encrypt' is a GLOBAL variable and should be set with SET GLOBAL
+set global innodb_undo_log_encrypt=1.1;
+ERROR 42000: Incorrect argument type to variable 'innodb_undo_log_encrypt'
+set global innodb_undo_log_encrypt='foo';
+ERROR 42000: Variable 'innodb_undo_log_encrypt' can't be set to the value of 'foo'
+set global innodb_undo_log_encrypt=-2;
+set global innodb_undo_log_encrypt=1e1;
+ERROR 42000: Incorrect argument type to variable 'innodb_undo_log_encrypt'
+set global innodb_undo_log_encrypt=2;
+ERROR 42000: Variable 'innodb_undo_log_encrypt' can't be set to the value of '2'
+SET @@global.innodb_undo_log_encrypt = @start_global_value;
+SELECT @@global.innodb_undo_log_encrypt;
+@@global.innodb_undo_log_encrypt
+0

mysql-test/suite/sys_vars/t/innodb_undo_log_encrypt_basic.test

@@ -0,0 +1,65 @@
+--source include/have_innodb.inc
+
+--disable_query_log
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Can't set undo tablespaces to be encrypted, since innodb_undo_tablespaces=0");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Can't set undo tablespace number 1 to be encrypted");
+--enable_query_log
+
+SET @start_global_value = @@global.innodb_undo_log_encrypt;
+SELECT @start_global_value;
+
+#
+# exists as global only
+#
+select @@global.innodb_undo_log_encrypt in (0, 1);
+select @@global.innodb_undo_log_encrypt;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.innodb_undo_log_encrypt;
+show global variables like 'innodb_undo_log_encrypt';
+show session variables like 'innodb_undo_log_encrypt';
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+--enable_warnings
+
+#
+# show that it's writable
+#
+set global innodb_undo_log_encrypt=1;
+--sleep 2
+select @@global.innodb_undo_log_encrypt;
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+--enable_warnings
+set @@global.innodb_undo_log_encrypt=0;
+select @@global.innodb_undo_log_encrypt;
+--disable_warnings
+select * from performance_schema.global_variables where variable_name='innodb_undo_log_encrypt';
+select * from performance_schema.session_variables where variable_name='innodb_undo_log_encrypt';
+--enable_warnings
+--error ER_GLOBAL_VARIABLE
+set session innodb_undo_log_encrypt='some';
+--error ER_GLOBAL_VARIABLE
+set @@session.innodb_undo_log_encrypt='some';
+
+#
+# incorrect types
+#
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_undo_log_encrypt=1.1;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_undo_log_encrypt='foo';
+set global innodb_undo_log_encrypt=-2;
+--error ER_WRONG_TYPE_FOR_VAR
+set global innodb_undo_log_encrypt=1e1;
+--error ER_WRONG_VALUE_FOR_VAR
+set global innodb_undo_log_encrypt=2;
+
+#
+# Cleanup
+#
+
+SET @@global.innodb_undo_log_encrypt = @start_global_value;
+SELECT @@global.innodb_undo_log_encrypt;

storage/innobase/fil/fil0crypt.cc

@@ -390,25 +390,6 @@ fil_space_merge_crypt_data(
 	mutex_exit(&dst->mutex);
 }
 
-static
-ulint
-fsp_header_get_encryption_offset(
-	const page_size_t&	page_size) {
-	ulint	offset;
-#ifdef UNIV_DEBUG
-	ulint	left_size;
-#endif
-
-	offset = XDES_ARR_OFFSET + XDES_SIZE * xdes_arr_size(page_size);
-#ifdef UNIV_DEBUG
-	left_size = page_size.physical() - FSP_HEADER_OFFSET - offset
-		- FIL_PAGE_DATA_END;
-	ut_ad(left_size >= fil_get_encrypt_info_size(CRYPT_SCHEME_1_IV_LEN));
-#endif
-
-	return offset;
-}
-
 fil_space_crypt_t*
 fil_space_read_crypt_data(const page_size_t& page_size, const byte* page) {
 

storage/innobase/fil/fil0fil.cc

@@ -5916,6 +5916,13 @@ fil_io_set_encryption(
 		return;
 	}
 
+	/* For writting undo log, if encryption for undo log is disabled,
+	skip set encryption. */
+	if (fsp_is_undo_tablespace(space->id)
+	    && !srv_undo_log_encrypt && req_type.is_write()) {
+		req_type.clear_encrypted();
+		return;
+	}
 
 	/* Don't encrypt the log, page 0 of all tablespaces, all pages
 	   don't encrypt TRX_SYS_SPACE.TRX_SYS_PAGE_NO as it contains address to dblwr buffer */
@@ -8022,6 +8029,17 @@ fil_encryption_rotate()
 			continue;
 		}
 
+		/* Skip the undo tablespace when it's in default
+		key status, since it's the first server startup
+		after bootstrap, and the server uuid is not ready
+		yet. */
+		if (fsp_is_undo_tablespace(space->id)
+		    && Encryption::master_key_id ==
+			ENCRYPTION_DEFAULT_MASTER_KEY_ID) {
+			space = UT_LIST_GET_NEXT(space_list, space);
+			continue;
+		}
+
 		/* Skip the temporary tablespace when it's in default
 		key status, since it's the first server startup
 		after bootstrap, and the server uuid is not ready