PS-3889: Native Partitioning for TokuDB and RocksDB for 5.7

TokuDB and MyRocks native partitioning is implemented as a copy of partition
storage engine in 5.7 with corresponding changes.

The base class - Partition_base does the most of the work.

The general differences from ha_partition class are the following:

1) all code which works with .par files was removed;
2) the code to read partition information from .frm files was added;
3) the responsibility for creating new handlers for separate partitions is moved
to Partition_base class descendants;
4) the initialization sequence for native and non-native partitioning handlers
is different, that is why create() and open() functions are changed
5) delete and rename table functions are changed both for Partition_base and
it's descendants(ha_tokupart and ha_rockspart) because it's necessary to
determine if the table is partitioned before calling the corresponding
functionality.

The changes in the server code are the following:

1) remove .frm file after a table has been removed to let handler the ability
to read partition info from .frm file during "delete table" execution;
2) remove "static" keyword from some functions signature to use them outside of
their module.

include/my_base.h

@@ -501,7 +501,8 @@ is the global server default. */
 #define HA_ERR_NOT_ALLOWED_COMMAND     200  /* Operation is not allowed */
 #define HA_ERR_COMPUTE_FAILED          201  /* Compute generated column value failed */
 #define HA_ERR_DEST_SCHEMA_NOT_EXIST   202  /* Destination schema does not exist */
-#define HA_ERR_LAST                    202  /* Copy of last error nr */
+#define HA_ERR_CANNOT_INITIALIZE_PARTITIONING 203 /* Partitioning can't be initialized */
+#define HA_ERR_LAST                    203  /* Copy of last error nr */
 
 /* Number of different errors */
 #define HA_ERR_ERRORS            (HA_ERR_LAST - HA_ERR_FIRST + 1)

mysql-test/include/table_files_replace_pattern.inc

@@ -1 +1 @@
---replace_regex  /[a-z0-9]+_[a-z0-9]+_[a-z0-9]+(_[BP]_[a-z0-9]+){0,1}\./id./ /sqlx_[a-z0-9]+_[a-z0-9]+_/sqlx_nnnn_nnnn_/ /sqlx-[a-z0-9]+_[a-z0-9]+/sqlx-nnnn_nnnn/ /#p#/#P#/ /#sp#/#SP#/ /#tmp#/#TMP#/ $ADDITIONAL_REGEX
+--replace_regex  /_[a-z0-9]+_[a-z0-9]+_[a-z0-9]+(_[BP]_[a-z0-9]+){0,1}\.tokudb/_id.tokudb/ /sqlx_[a-z0-9]+_[a-z0-9]+_/sqlx_nnnn_nnnn_/ /sqlx-[a-z0-9]+_[a-z0-9]+/sqlx-nnnn_nnnn/ /#p#/#P#/ /#sp#/#SP#/ /#tmp#/#TMP#/ /_([0-9]+)\.sdi/_id.sdi/ $ADDITIONAL_REGEX

mysql-test/suite/parts/inc/update_parts_from_prev_ver_suppressions.inc

@@ -0,0 +1,28 @@
+--disable_query_log
+call mtr.add_suppression("Resizing redo log from");
+call mtr.add_suppression("Upgrading redo log");
+call mtr.add_suppression("Starting to delete and rewrite log files");
+call mtr.add_suppression("New log files created");
+call mtr.add_suppression("You need to use --log-bin to make --binlog-format work");
+call mtr.add_suppression("Creating routine without parsing routine body");
+call mtr.add_suppression("Storage engine '.*' does not support system tables. \\[mysql.*\\]");
+call mtr.add_suppression("Table 'mysql.component' doesn't exist");
+call mtr.add_suppression("is expected to be transactional");
+call mtr.add_suppression("table is missing or has an incorrect definition");
+call mtr.add_suppression("ACL DDLs will not work unless mysql_upgrade is executed");
+call mtr.add_suppression(".* Native table .* has the wrong structure");
+call mtr.add_suppression("Column count of mysql.* is wrong");
+call mtr.add_suppression(".*Missing system table mysql.global_grants.*");
+call mtr.add_suppression("ACL table mysql.[a-z_]* missing. Some operations may fail.");
+call mtr.add_suppression("Info table is not ready to be used. Table 'mysql.slave_master_info' cannot be opened");
+call mtr.add_suppression("Error in checking mysql.slave_master_info repository info type of TABLE");
+call mtr.add_suppression("Error creating master info: Error checking repositories.");
+call mtr.add_suppression("Slave: Failed to initialize the master info structure for channel");
+call mtr.add_suppression("Failed to create or recover replication info repositories.");
+# 5.6 suppresions
+call mtr.add_suppression("Column count of performance_schema.* is wrong");
+call mtr.add_suppression(".* has no `.*` column at position *");
+call mtr.add_suppression("Failed to open optimizer cost constant tables");
+call mtr.add_suppression(".* table is not ready to be used. Table '.*' cannot be opened.");
+call mtr.add_suppression("InnoDB: Cannot open '.*' for reading: No such file or directory");
+--enable_query_log

mysql-test/suite/parts/inc/upgrade_parts_from_prev_ver_alter.inc

@@ -0,0 +1,39 @@
+--echo #########
+--echo # Test for "ALTER TABLE ... UPGRADE PARTITIONING"
+--echo ###
+
+--echo # Unzip the zip file.
+--file_exists $DATA_ARCH_PATH
+--exec unzip -qo $DATA_ARCH_PATH -d $MYSQL_TMP_DIR
+
+--echo # Create a bootstrap file in temp location
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+--exec echo $MYSQL_TMP_DIR/bootstrap.log
+
+--let $restart_parameters = "restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=$MYSQLD_DATADIR1 $ADDITIONAL_OPTS"
+--let $restart_hide_args = 1
+--source include/start_mysqld.inc
+
+--echo # Check test table before upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Upgrade
+ALTER TABLE test.t1 UPGRADE PARTITIONING;
+
+--echo # Check test table after upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Stop the server
+--source include/shutdown_mysqld.inc
+
+--let $restart_parameters =
+--let $restart_hide_args =
+
+--echo # Remove data directory
+--exec rm -rf $MYSQLD_DATADIR1

mysql-test/suite/parts/inc/upgrade_parts_from_prev_ver_mysql_upgrade.inc

@@ -0,0 +1,32 @@
+--echo #########
+--echo # Test for upgrade partitioning with mysql_upgrade
+--echo ###
+
+--echo # Unzip the zip file.
+--file_exists $DATA_ARCH_PATH
+--exec unzip -qo $DATA_ARCH_PATH -d $MYSQL_TMP_DIR
+
+--let $restart_parameters = "restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=$MYSQLD_DATADIR1 $ADDITIONAL_OPTS"
+--let $restart_hide_args = 1
+--source include/start_mysqld.inc
+
+--echo # Execute mysql_upgrade
+--source include/mysql_upgrade_preparation.inc
+--exec $MYSQL_UPGRADE --skip-verbose --force 2>&1
+--source include/mysql_upgrade_cleanup.inc
+
+--echo # Check test table after upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Stop the server
+--source include/shutdown_mysqld.inc
+
+--let $restart_parameters =
+--let $restart_hide_args =
+
+--echo # Remove copied files
+--remove_file $DATA_ARCH_PATH
+--exec rm -rf $MYSQLD_DATADIR1

mysql-test/suite/rocksdb.rpl/r/rpl_rocksdb_extra_col_slave.result

@@ -503,11 +503,7 @@ c1	c2	c3	c4	c5	c6	c7
 3	3.00	If is does not, I will open a bug	b1b1b1b1b1b1b1b1	QA	1	CURRENT_TIMESTAMP
 *** Add Partition on master ***
 ALTER TABLE t16 PARTITION BY KEY(c1) PARTITIONS 4;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t16', is deprecated and will be removed in a future release. Please use native partitioning instead.
 INSERT INTO t16 () VALUES(4,1.00,'Replication Rocks',@b1,'Omer');
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t16', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SHOW CREATE TABLE t16;
 Table	Create Table
 t16	CREATE TABLE `t16` (
@@ -520,8 +516,6 @@ t16	CREATE TABLE `t16` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (c1)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t16', is deprecated and will be removed in a future release. Please use native partitioning instead.
 *** Show table on Slave ****
 SHOW CREATE TABLE t16;
 Table	Create Table
@@ -537,8 +531,6 @@ t16	CREATE TABLE `t16` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (c1)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t16', is deprecated and will be removed in a future release. Please use native partitioning instead.
 *** DROP TABLE t16 ***
 DROP TABLE t16;
 *** Alter Master End ***

mysql-test/suite/rocksdb.rpl/r/rpl_skip_trx_api_binlog_format.result

@@ -31,11 +31,7 @@ set @@global.read_only=@orig_read_only;
 set @@global.rocksdb_rpl_skip_tx_api=ON;
 set rocksdb_unsafe_for_binlog=1;
 create table t1(a int, primary key (a)) engine=rocksdb partition by key(a) partitions 2;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 insert into t1 values(1), (2), (3), (4);
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 include/wait_for_slave_sql_error.inc [errno=1593]
 Last_SQL_Error = 'Fatal error: Masters binlog format is not ROW and storage engine can not handle non-ROW events at this time. Table: 'test.t1' Query: 'insert into t1 values(1), (2), (3), (4)''
 "Table after error"

mysql-test/suite/rocksdb.sys_vars/r/rocksdb_update_cf_options_basic.result

@@ -21,8 +21,6 @@ PARTITION custom_p5 VALUES IN (0x6789A),
 PARTITION custom_p6 VALUES IN (0x789AB),
 PARTITION custom_p7 VALUES IN (0x89ABC)
 );
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SELECT @@global.rocksdb_update_cf_options;
 @@global.rocksdb_update_cf_options
 NULL

mysql-test/suite/rocksdb/r/add_index_inplace.result

@@ -246,16 +246,8 @@ a	b
 3	7
 DROP TABLE t1;
 CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 DROP INDEX kij ON t1;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -267,8 +259,6 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (i)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SELECT * FROM t1 ORDER BY i LIMIT 10;
 i	j	k
 1	1	1

mysql-test/suite/rocksdb/r/add_index_inplace_crash.result

@@ -21,10 +21,6 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 DROP TABLE t1;
 CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 # crash_during_index_creation_partition
 flush logs;
 SET SESSION debug="+d,crash_during_index_creation_partition";
@@ -42,11 +38,7 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (i)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SELECT * FROM t1 ORDER BY i LIMIT 10;
 i	j	k
 1	1	1
@@ -64,10 +56,6 @@ COUNT(*)
 100
 DROP TABLE t1;
 CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 # crash_during_index_creation_partition
 flush logs;
 SET SESSION debug="+d,myrocks_simulate_index_create_rollback";
@@ -85,11 +73,7 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (i)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE;
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -102,8 +86,6 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1
 /*!50100 PARTITION BY KEY (i)
 PARTITIONS 4 */
-Warnings:
-Warning	1287	The partition engine, used by table 'test.t1', is deprecated and will be removed in a future release. Please use native partitioning instead.
 SELECT COUNT(*) FROM t1;
 COUNT(*)
 100