PS-4556: InnoDB redo log encryption

This is based on WL#9290 in MySQL 8.0, while also adding a keyring based rotated key encryption.

The encryption setting supports two options:

* master_key, which works like the redo log encryption in MySQL 8.0
* keyring_key, which encrypts each new redo log tablespace with the latest percona_redo key from the keyring.

Related commits in upstream 8.0:

* 1a6dd57
* 8facbb0

client/mysqltest.cc

@@ -401,6 +401,7 @@ enum enum_commands {
   Q_WRITE_FILE, Q_COPY_FILE, Q_PERL, Q_DIE, Q_EXIT, Q_SKIP,
   Q_CHMOD_FILE, Q_APPEND_FILE, Q_CAT_FILE, Q_DIFF_FILES,
   Q_SEND_QUIT, Q_CHANGE_USER, Q_MKDIR, Q_RMDIR,
+  Q_FORCE_RMDIR,
   Q_LIST_FILES, Q_LIST_FILES_WRITE_FILE, Q_LIST_FILES_APPEND_FILE,
   Q_SEND_SHUTDOWN, Q_SHUTDOWN_SERVER,
   Q_RESULT_FORMAT_VERSION,
@@ -503,6 +504,7 @@ const char *command_names[]=
   "change_user",
   "mkdir",
   "rmdir",
+  "force-rmdir",
   "list_files",
   "list_files_write_file",
   "list_files_append_file",
@@ -3889,17 +3891,78 @@ void do_mkdir(struct st_command *command)
   DBUG_VOID_RETURN;
 }
 
+
+/*
+  SYNOPSIS
+  do_force_rmdir
+  command    - command handle
+  ds_dirname - pointer to dynamic string containing directory informtion
+
+  DESCRIPTION
+  force-rmdir <dir_name>
+  Remove the directory <dir_name>
+*/
+
+static void do_force_rmdir(struct st_command *command, DYNAMIC_STRING *ds_dirname)
+{
+  DBUG_ENTER("do_force_rmdir");
+
+  char dir_name[FN_REFLEN];
+  strncpy(dir_name, ds_dirname->str, sizeof(dir_name));
+
+  /* Note that my_dir sorts the list if not given any flags */
+  MY_DIR *dir_info= my_dir(ds_dirname->str, MYF(MY_DONT_SORT | MY_WANT_STAT));
+
+  if (dir_info && dir_info->number_off_files > 2)
+  {
+    /* Storing the length of the path to the file, so it can be reused */
+    size_t length= ds_dirname->length;
+
+    /* Delete the directory recursively */
+    for (uint i= 0; i < dir_info->number_off_files; i++)
+    {
+      FILEINFO *file= dir_info->dir_entry + i;
+
+      /* Skip the names "." and ".." */
+      if (!strcmp(file->name, ".") ||
+          !strcmp(file->name, ".."))
+        continue;
+
+      ds_dirname->length= length;
+      char dir_separator[2]= {FN_LIBCHAR, 0};
+      dynstr_append(ds_dirname, dir_separator);
+      dynstr_append(ds_dirname, file->name);
+
+      if (MY_S_ISDIR(file->mystat->st_mode))
+        /* It's a directory */
+        do_force_rmdir(command, ds_dirname);
+      else
+        /* It's a file */
+        my_delete(ds_dirname->str, MYF(0));
+    }
+  }
+
+  my_dirend(dir_info);
+  int error= rmdir(dir_name) != 0;
+  handle_command_error(command, error);
+
+  DBUG_VOID_RETURN;
+}
+
+
 /*
   SYNOPSIS
   do_rmdir
   command	called command
+  force         Recursively delete a directory if the value is set to true,
+                otherwise delete an empty direcory
 
   DESCRIPTION
   rmdir <dir_name>
   Remove the empty directory <dir_name>
 */
 
-void do_rmdir(struct st_command *command)
+static void do_rmdir(struct st_command *command, bool force)
 {
   int error;
   static DYNAMIC_STRING ds_dirname;
@@ -3913,8 +3976,13 @@ void do_rmdir(struct st_command *command)
                      ' ');
 
   DBUG_PRINT("info", ("removing directory: %s", ds_dirname.str));
-  error= rmdir(ds_dirname.str) != 0;
-  handle_command_error(command, error);
+  if (force)
+    do_force_rmdir(command, &ds_dirname);
+  else
+  {
+    error= rmdir(ds_dirname.str) != 0;
+    handle_command_error(command, error);
+  }
   dynstr_free(&ds_dirname);
   DBUG_VOID_RETURN;
 }
@@ -9545,7 +9613,8 @@ int main(int argc, char **argv)
       case Q_REMOVE_FILE: do_remove_file(command); break;
       case Q_REMOVE_FILES_WILDCARD: do_remove_files_wildcard(command); break;
       case Q_MKDIR: do_mkdir(command); break;
-      case Q_RMDIR: do_rmdir(command); break;
+      case Q_RMDIR: do_rmdir(command, 0); break;
+      case Q_FORCE_RMDIR: do_rmdir(command, 1); break;
       case Q_LIST_FILES: do_list_files(command); break;
       case Q_LIST_FILES_WRITE_FILE:
         do_list_files_write_file_command(command, FALSE);

include/system_key.h

@@ -21,6 +21,7 @@
 
 C_MODE_START
 #define PERCONA_BINLOG_KEY_NAME "percona_binlog"
+#define PERCONA_REDO_KEY_NAME "percona_redo"
 extern const size_t valid_percona_system_keys_size;
 extern const char* valid_percona_system_keys[];
 

mysql-test/include/log_encrypt_1.inc

@@ -0,0 +1,118 @@
+# WL#9290 InnoDB: Support transparent tablespace data encryption for redo log
+# This test case will test basic redo log encryption support features.
+
+--source include/no_valgrind_without_big.inc
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+--source include/have_innodb_max_16k.inc
+
+call mtr.add_suppression("\\[Error\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded.");
+
+# eNable redo log encryption, should report error in server log, since keyring is not loaded.
+eval SET GLOBAL innodb_redo_log_encrypt = $redo_log_mode;
+
+# Create a table with encryption, should fail since keyring is not
+# loaded.
+--error ER_CANNOT_FIND_KEY_IN_KEYRING
+CREATE TABLE t1(c1 INT, c2 char(20)) ENCRYPTION="Y" ENGINE = InnoDB;
+
+CREATE TABLE t1(c1 INT, c2 char(20)) ENGINE = InnoDB;
+
+--error ER_CANNOT_FIND_KEY_IN_KEYRING
+ALTER TABLE t1 ENCRYPTION="Y", algorithm=copy;
+
+let $restart_parameters = restart: $KEYRING_PARAMS --general-log --log-output=FILE --general_log_file=$MYSQL_TMP_DIR/keyring_query_log $KEYRING_PLUGIN_OPT;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT --plugin-dir=KEYRING_PLUGIN_PATH
+--replace_regex /\.dll/.so/
+--source include/restart_mysqld_no_echo.inc
+
+#Enable redo log encryption
+eval SET GLOBAL innodb_redo_log_encrypt = $redo_log_mode;
+
+SHOW CREATE TABLE t1;
+
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+
+SELECT * FROM t1 ORDER BY c1 LIMIT 10;
+
+# Restart to confirm the encryption info can be retrieved properly.
+let $restart_parameters = restart: $KEYRING_PARAMS --general-log --log-output=FILE --general_log_file=$MYSQL_TMP_DIR/keyring_query_log --innodb_redo_log_encrypt=$redo_log_mode;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT --plugin-dir=KEYRING_PLUGIN_PATH
+--replace_regex /\.dll/.so/
+--source include/restart_mysqld_no_echo.inc
+
+SELECT * FROM t1 ORDER BY c1 LIMIT 10;
+
+DROP TABLE t1;
+
+# Crash/recovery test.
+CREATE TABLE t1(c1 INT, c2 char(20)) ENGINE = InnoDB;
+
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+
+# Restart to confirm the encryption info can be retrieved properly.
+--let $restart_parameters = restart: $KEYRING_PARAMS
+--source include/restart_mysqld_no_echo.inc
+
+SELECT * FROM t1 ORDER BY c1 LIMIT 10;
+DELETE FROM t1;
+
+START TRANSACTION;
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+
+# Restart to confirm the encryption info can be retrieved properly.
+let $restart_parameters = restart: $KEYRING_PARAMS --general-log --log-output=FILE --general_log_file=$MYSQL_TMP_DIR/keyring_query_log --innodb_redo_log_encrypt=$redo_log_mode;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT --plugin-dir=KEYRING_PLUGIN_PATH
+--replace_regex /\.dll/.so/
+--source include/restart_mysqld_no_echo.inc
+
+SELECT * FROM t1 ORDER BY c1 LIMIT 10;
+
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+
+SELECT * FROM t1 ORDER BY c1 LIMIT 10;
+
+# Cleanup
+DROP TABLE t1;

mysql-test/include/log_encrypt_2.inc

@@ -0,0 +1,129 @@
+# InnoDB transparent tablespace data encryption
+# This test case will test basic encryption support features.
+
+--source include/no_valgrind_without_big.inc
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+CREATE TABLE t1(c1 int) ENGINE=InnoDB ENCRYPTION="Y";
+
+DROP TABLE t1;
+
+# Restart the server with keyring loaded
+--let restart_parameters="restart:$KEYRING_PARAMS"
+--source include/restart_mysqld_no_echo.inc
+
+let $innodb_file_per_table = `SELECT @@innodb_file_per_table`;
+
+SET GLOBAL innodb_file_per_table = 1;
+SELECT @@innodb_file_per_table;
+
+# Create a table with encryption
+CREATE TABLE t1(c1 INT, c2 char(20)) ENCRYPTION="Y" ENGINE = InnoDB;
+
+SHOW CREATE TABLE t1;
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+
+SELECT * FROM t1 LIMIT 10;
+
+# Restart to confirm the encryption info can be retrieved properly.
+--let restart_parameters="restart:$KEYRING_PARAMS"
+--source include/restart_mysqld_no_echo.inc
+
+SELECT * FROM t1 LIMIT 10;
+
+# Key rotation.
+ALTER INSTANCE ROTATE INNODB MASTER KEY;
+
+DROP TABLE t1;
+
+# Crash/recovery test.
+CREATE TABLE t1(c1 INT, c2 char(20)) ENCRYPTION="Y" ENGINE = InnoDB;
+
+INSERT INTO t1 VALUES(0, "aaaaa");
+INSERT INTO t1 VALUES(1, "bbbbb");
+INSERT INTO t1 VALUES(2, "ccccc");
+INSERT INTO t1 VALUES(3, "ddddd");
+INSERT INTO t1 VALUES(4, "eeeee");
+INSERT INTO t1 VALUES(5, "fffff");
+INSERT INTO t1 VALUES(6, "ggggg");
+INSERT INTO t1 VALUES(7, "hhhhh");
+INSERT INTO t1 VALUES(8, "iiiii");
+INSERT INTO t1 VALUES(9, "jjjjj");
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+INSERT INTO t1 select * from t1;
+
+# Restart to confirm the encryption info can be retrieved properly.
+--let restart_parameters="restart:$KEYRING_PARAMS"
+--let $restart_hide_args = 1
+--source include/kill_and_restart_mysqld.inc
+
+SELECT * FROM t1 LIMIT 10;
+DROP TABLE t1;
+
+let $restart_parameters = restart: $KEYRING_PARAMS --general-log --log-output=FILE --general_log_file=$MYSQL_TMP_DIR/keyring_query_log;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR $KEYRING_PLUGIN_OPT --plugin-dir=KEYRING_PLUGIN_PATH
+--replace_regex /\.dll/.so/
+--source include/restart_mysqld_no_echo.inc
+
+# Check no effect of block_encryption_mode = 'aes-256-cbc' variable on table encryption
+SET block_encryption_mode = 'aes-256-cbc';
+# Test encryption .
+CREATE DATABASE tde_db;
+CREATE TABLE tde_db.t1(c1 INT PRIMARY KEY, c2 char(50)) ENCRYPTION = 'Y' ENGINE = InnoDB;
+
+INSERT INTO tde_db.t1 VALUES(0, 'abc');
+INSERT INTO tde_db.t1 VALUES(1, 'xyz');
+INSERT INTO tde_db.t1 VALUES(2, null);
+INSERT INTO tde_db.t1 VALUES(3, null);
+SELECT * FROM tde_db.t1 LIMIT 10;
+ALTER INSTANCE ROTATE INNODB MASTER KEY;
+SELECT * FROM tde_db.t1 LIMIT 10;
+--echo # Mysqldump output
+--exec $MYSQL_DUMP --compact --skip-comments --databases tde_db
+--echo # Redirecting mysqlpump output to MYSQL_TMP_DIR/mysqlpump_encrypt.sql
+--exec $MYSQL_PUMP --default-parallelism=1 --databases tde_db > $MYSQL_TMP_DIR/mysqlpump_encrypt.sql
+DROP DATABASE tde_db;
+
+--let SEARCH_FILE=$MYSQL_TMP_DIR/keyring_query_log
+let SEARCH_PATTERN= ALTER INSTANCE ROTATE INNODB MASTER KEY;
+--source include/search_pattern.inc
+
+--echo # Loading tables from mysqlpump_encrypt.sql
+--exec $MYSQL --skip-comments < $MYSQL_TMP_DIR/mysqlpump_encrypt.sql
+SELECT * FROM tde_db.t1 LIMIT 10;
+INSERT INTO tde_db.t1 VALUES(4, null);
+SELECT * FROM tde_db.t1 LIMIT 10;
+DROP DATABASE tde_db;
+#
+
+# Cleanup
+--remove_file $MYSQL_TMP_DIR/keyring_query_log
+--remove_file $MYSQL_TMP_DIR/mysecret_keyring2
+--remove_file $MYSQL_TMP_DIR/mysqlpump_encrypt.sql
+eval SET GLOBAL innodb_file_per_table=$innodb_file_per_table;