PS-3889: Native Partitioning for TokuDB and RocksDB for 5.7

TokuDB and MyRocks native partitioning is implemented as a copy of partition
storage engine in 5.7 with corresponding changes.

The base class - Partition_base does the most of the work.

The general differences from ha_partition class are the following:

1) all code which works with .par files was removed;
2) the code to read partition information from .frm files was added;
3) the responsibility for creating new handlers for separate partitions is moved
to Partition_base class descendants;
4) the initialization sequence for native and non-native partitioning handlers
is different, that is why create() and open() functions are changed
5) delete and rename table functions are changed both for Partition_base and
it's descendants(ha_tokupart and ha_rockspart) because it's necessary to
determine if the table is partitioned before calling the corresponding
functionality.

The changes in the server code are the following:

1) remove .frm file after a table has been removed to let handler the ability
to read partition info from .frm file during "delete table" execution;
2) remove "static" keyword from some functions signature to use them outside of
their module.

Two new system variables were added:

tokudb_enable_native_patition -
  enable native partitioning for TokuDB, read-only, default value is 'off',
rocksdb_enable_native_patition -
  enable native partitioning for RocksDB, read-only, default value is 'off'.

The ability to have custom result directory for the certain options combination
was added to MTR. To use it just add option "mtr-result-dir" in the certain
options group in 'combinations' file.

The ability to add only the certain list of tests for the certain option
combination is added to MTR. For this purpose the new "mtr-tests-list" option
is introduced. This option contains comma-separated list of tests in the
current suite, which will be executed for the current options combination. If
the option is missed then all tests from the suite will be executed for
the combination.

As an example of the new options see changed and newly created "combination"
files of this commit.

The above features are used to test TokuDB and RocksDB native and non-native
partitioning, that is why the corresponding test suites have two directories
for result files for one directory of tests.

include/my_base.h

@@ -501,7 +501,8 @@ is the global server default. */
 #define HA_ERR_NOT_ALLOWED_COMMAND     200  /* Operation is not allowed */
 #define HA_ERR_COMPUTE_FAILED          201  /* Compute generated column value failed */
 #define HA_ERR_DEST_SCHEMA_NOT_EXIST   202  /* Destination schema does not exist */
-#define HA_ERR_LAST                    202  /* Copy of last error nr */
+#define HA_ERR_CANNOT_INITIALIZE_PARTITIONING 203 /* Partitioning can't be initialized */
+#define HA_ERR_LAST                    203  /* Copy of last error nr */
 
 /* Number of different errors */
 #define HA_ERR_ERRORS            (HA_ERR_LAST - HA_ERR_FIRST + 1)

mysql-test/include/table_files_replace_pattern.inc

@@ -1 +1 @@
---replace_regex  /[a-z0-9]+_[a-z0-9]+_[a-z0-9]+(_[BP]_[a-z0-9]+){0,1}\./id./ /sqlx_[a-z0-9]+_[a-z0-9]+_/sqlx_nnnn_nnnn_/ /sqlx-[a-z0-9]+_[a-z0-9]+/sqlx-nnnn_nnnn/ /#p#/#P#/ /#sp#/#SP#/ /#tmp#/#TMP#/ $ADDITIONAL_REGEX
+--replace_regex  /_[a-z0-9]+_[a-z0-9]+_[a-z0-9]+(_[BP]_[a-z0-9]+){0,1}\.tokudb/_id.tokudb/ /sqlx_[a-z0-9]+_[a-z0-9]+_/sqlx_nnnn_nnnn_/ /sqlx-[a-z0-9]+_[a-z0-9]+/sqlx-nnnn_nnnn/ /#p#/#P#/ /#sp#/#SP#/ /#tmp#/#TMP#/ /_([0-9]+)\.sdi/_id.sdi/ $ADDITIONAL_REGEX

mysql-test/lib/mtr_cases.pm

@@ -545,7 +545,17 @@ sub collect_one_suite($)
 	my $comb= {};
 	$comb->{name}= $group->name();
         foreach my $option ( $group->options() ) {
-	  push(@{$comb->{comb_opt}}, $option->option());
+        my $option_string = $option->option();
+        if ($option_string =~ m/--mtr[-_]result[-_]dir\s*=\s*(.+)/) {
+          mtr_verbose("The result dir was changed to $1");
+          $comb->{result_dir} = $1;
+        } elsif ($option_string =~ m/--mtr[-_]tests[-_]list\s*=\s*(.+)/) {
+          my @comb_tests_array = split (/,/, $1);
+          my %comb_tests_set = map { $_ => 1 } @comb_tests_array;
+          $comb->{tests_set} = \%comb_tests_set;
+        } else {
+          push(@{$comb->{comb_opt}}, $option->option());
+        }
 	}
 	push(@combinations, $comb);
       }
@@ -564,6 +574,8 @@ sub collect_one_suite($)
 
 	  next if ( $test->{'skip'} );
 
+    next if ($comb->{tests_set} and not
+             $comb->{tests_set}->{$test->{shortname}});
 	  # Skip this combination if the values it provides
 	  # already are set in master_opt or slave_opt
 	  if (My::Options::is_set($test->{master_opt}, $comb->{comb_opt}) &&
@@ -585,6 +597,16 @@ sub collect_one_suite($)
 	  push(@{$new_test->{master_opt}}, @{$comb->{comb_opt}});
 	  push(@{$new_test->{slave_opt}}, @{$comb->{comb_opt}});
 
+    if ($comb->{result_dir}) {
+      my $result_file = "$suitedir/$comb->{result_dir}/".
+                        "$new_test->{shortname}.result";
+      if (-f $result_file) {
+        $new_test->{result_file} = $result_file;
+      } else {
+        $new_test->{record_file} =  $result_file;
+      }
+    }
+
 	  # Add combination name short name
 	  $new_test->{combination}= $comb->{name};
 

mysql-test/suite/parts/inc/disable_native_partitioning.inc

@@ -0,0 +1,27 @@
+--let $MYSQLD_DATADIR=`select @@datadir`
+
+--let $restart_parameters = "restart: --loose-$disable_native_partition_opt=0"
+--source include/restart_mysqld.inc
+
+eval CREATE TABLE t1 (a INT UNSIGNED NOT NULL, PRIMARY KEY(a)) ENGINE = $engine
+PARTITION BY RANGE (a) (
+    PARTITION p0 VALUES LESS THAN (100),
+    PARTITION p1 VALUES LESS THAN (200)
+);
+
+--list_files $MYSQLD_DATADIR/test *.par
+
+DROP TABLE t1;
+
+--let $restart_parameters = "restart: "
+--source include/restart_mysqld.inc
+
+eval CREATE TABLE t1 (a INT UNSIGNED NOT NULL, PRIMARY KEY(a)) ENGINE = $engine
+PARTITION BY RANGE (a) (
+    PARTITION p0 VALUES LESS THAN (100),
+    PARTITION p1 VALUES LESS THAN (200)
+);
+
+--list_files $MYSQLD_DATADIR/test *.par
+
+DROP TABLE t1;

mysql-test/suite/parts/inc/update_parts_from_prev_ver_suppressions.inc

@@ -0,0 +1,28 @@
+--disable_query_log
+call mtr.add_suppression("Resizing redo log from");
+call mtr.add_suppression("Upgrading redo log");
+call mtr.add_suppression("Starting to delete and rewrite log files");
+call mtr.add_suppression("New log files created");
+call mtr.add_suppression("You need to use --log-bin to make --binlog-format work");
+call mtr.add_suppression("Creating routine without parsing routine body");
+call mtr.add_suppression("Storage engine '.*' does not support system tables. \\[mysql.*\\]");
+call mtr.add_suppression("Table 'mysql.component' doesn't exist");
+call mtr.add_suppression("is expected to be transactional");
+call mtr.add_suppression("table is missing or has an incorrect definition");
+call mtr.add_suppression("ACL DDLs will not work unless mysql_upgrade is executed");
+call mtr.add_suppression(".* Native table .* has the wrong structure");
+call mtr.add_suppression("Column count of mysql.* is wrong");
+call mtr.add_suppression(".*Missing system table mysql.global_grants.*");
+call mtr.add_suppression("ACL table mysql.[a-z_]* missing. Some operations may fail.");
+call mtr.add_suppression("Info table is not ready to be used. Table 'mysql.slave_master_info' cannot be opened");
+call mtr.add_suppression("Error in checking mysql.slave_master_info repository info type of TABLE");
+call mtr.add_suppression("Error creating master info: Error checking repositories.");
+call mtr.add_suppression("Slave: Failed to initialize the master info structure for channel");
+call mtr.add_suppression("Failed to create or recover replication info repositories.");
+# 5.6 suppresions
+call mtr.add_suppression("Column count of performance_schema.* is wrong");
+call mtr.add_suppression(".* has no `.*` column at position *");
+call mtr.add_suppression("Failed to open optimizer cost constant tables");
+call mtr.add_suppression(".* table is not ready to be used. Table '.*' cannot be opened.");
+call mtr.add_suppression("InnoDB: Cannot open '.*' for reading: No such file or directory");
+--enable_query_log

mysql-test/suite/parts/inc/upgrade_parts_from_prev_ver_alter.inc

@@ -0,0 +1,39 @@
+--echo #########
+--echo # Test for "ALTER TABLE ... UPGRADE PARTITIONING"
+--echo ###
+
+--echo # Unzip the zip file.
+--file_exists $DATA_ARCH_PATH
+--exec unzip -qo $DATA_ARCH_PATH -d $MYSQL_TMP_DIR
+
+--echo # Create a bootstrap file in temp location
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+--exec echo $MYSQL_TMP_DIR/bootstrap.log
+
+--let $restart_parameters = "restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=$MYSQLD_DATADIR1 $ADDITIONAL_OPTS"
+--let $restart_hide_args = 1
+--source include/start_mysqld.inc
+
+--echo # Check test table before upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Upgrade
+ALTER TABLE test.t1 UPGRADE PARTITIONING;
+
+--echo # Check test table after upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Stop the server
+--source include/shutdown_mysqld.inc
+
+--let $restart_parameters =
+--let $restart_hide_args =
+
+--echo # Remove data directory
+--exec rm -rf $MYSQLD_DATADIR1

mysql-test/suite/parts/inc/upgrade_parts_from_prev_ver_mysql_upgrade.inc

@@ -0,0 +1,32 @@
+--echo #########
+--echo # Test for upgrade partitioning with mysql_upgrade
+--echo ###
+
+--echo # Unzip the zip file.
+--file_exists $DATA_ARCH_PATH
+--exec unzip -qo $DATA_ARCH_PATH -d $MYSQL_TMP_DIR
+
+--let $restart_parameters = "restart: --loose-skip-log-bin --skip-log-slave-updates --datadir=$MYSQLD_DATADIR1 $ADDITIONAL_OPTS"
+--let $restart_hide_args = 1
+--source include/start_mysqld.inc
+
+--echo # Execute mysql_upgrade
+--source include/mysql_upgrade_preparation.inc
+--exec $MYSQL_UPGRADE --skip-verbose --force 2>&1
+--source include/mysql_upgrade_cleanup.inc
+
+--echo # Check test table after upgrade
+SHOW CREATE TABLE test.t1;
+SELECT * FROM test.t1 ORDER BY a;
+--source include/table_files_replace_pattern.inc
+--list_files $MYSQLD_DATADIR1/test/
+
+--echo # Stop the server
+--source include/shutdown_mysqld.inc
+
+--let $restart_parameters =
+--let $restart_hide_args =
+
+--echo # Remove copied files
+--remove_file $DATA_ARCH_PATH
+--exec rm -rf $MYSQLD_DATADIR1

mysql-test/suite/rocksdb.rpl/combinations

@@ -12,3 +12,25 @@ binlog-format=statement
 
 [mix]
 binlog-format=mixed
+
+[row-write-committed-native-partitioning]
+mtr-result-dir = r-native-partitioning
+mtr-tests-list = rpl_rocksdb_extra_col_slave
+binlog-format=row
+loose-rocksdb_write_policy=write_committed
+loose-rocksdb-enable-native-partition=on
+
+[row-write-prepared-native-partitioning]
+mtr-result-dir = r-native-partitioning
+mtr-tests-list = rpl_rocksdb_extra_col_slave
+binlog-format=row
+loose-rocksdb_write_policy=write_prepared
+loose-rocksdb_commit_time_batch_for_recovery=on
+loose-rocksdb-enable-native-partition=on
+
+[stmt-native-partitioning]
+mtr-result-dir = r-native-partitioning
+mtr-tests-list = rpl_skip_trx_api_binlog_format
+binlog-format=statement
+loose-rocksdb-enable-native-partition=on
+