binlogdata.proto

/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file contains all the types and servers necessary to make
// RPC calls to VtTablet for the binlog protocol, used by filtered
// replication only.

syntax = "proto3";
option go_package = "vitess.io/vitess/go/vt/proto/binlogdata";

package binlogdata;

import "vtrpc.proto";
import "query.proto";
import "topodata.proto";

// Charset is the per-statement charset info from a QUERY_EVENT binlog entry.
message Charset {
  // @@session.character_set_client
  int32 client = 1;
  // @@session.collation_connection
  int32 conn = 2;
  // @@session.collation_server
  int32 server = 3;
}

// BinlogTransaction describes a transaction inside the binlogs.
// It is streamed by vttablet for filtered replication, used during resharding.
message BinlogTransaction {
  message Statement {
    enum Category {
      BL_UNRECOGNIZED = 0;
      BL_BEGIN = 1;
      BL_COMMIT = 2;
      BL_ROLLBACK = 3;
      // BL_DML is deprecated.
      BL_DML_DEPRECATED = 4;
      BL_DDL = 5;
      BL_SET = 6;
      BL_INSERT = 7;
      BL_UPDATE = 8;
      BL_DELETE = 9;
    }

    // what type of statement is this?
    Category category = 1;

    // charset of this statement, if different from pre-negotiated default.
    Charset charset = 2;

    // the sql
    bytes sql = 3;
  }

  // the statements in this transaction
  repeated Statement statements = 1;

  // DEPRECATED (replaced by event_token): the timestamp of the statements.
  // int64 timestamp = 2;
  reserved 2;

  // DEPRECATED (replaced by event_token): the Transaction ID after
  // this statement was applied.
  // string transaction_id = 3;
  reserved 3;

  // The Event Token for this event.
  query.EventToken event_token = 4;
}

// StreamKeyRangeRequest is the payload to StreamKeyRange
message StreamKeyRangeRequest {
  // where to start
  string position = 1;

  // what to get
  topodata.KeyRange key_range = 2;

  // default charset on the player side
  Charset charset = 3;
}

// StreamKeyRangeResponse is the response from StreamKeyRange
message StreamKeyRangeResponse{
  BinlogTransaction binlog_transaction = 1;
}

// StreamTablesRequest is the payload to StreamTables
message StreamTablesRequest {
  // where to start
  string position = 1;

  // what to get
  repeated string tables = 2;

  // default charset on the player side
  Charset charset = 3;
}

// StreamTablesResponse is the response from StreamTables
message StreamTablesResponse {
  BinlogTransaction binlog_transaction = 1;
}

// CharsetConversion represent a conversion of text from one charset to another
message CharsetConversion {
  // FromCharset is the charset name from which we convert the text (e.g. latin1)
  string from_charset = 1;
  // ToCharset is the charset name to which we convert the text (e.g. utf8mb4)
  string to_charset = 2;
}

// Rule represents one rule in a Filter.
message Rule {
  // Match can be a table name or a regular expression.
  // If it starts with a '/', it's a regular expression.
  // For example, "t" matches a table named "t", whereas
  // "/t.*" matches all tables that begin with 't'.
  string match = 1;
  // Filter: If empty, all columns and rows of the matching tables
  // are sent. If it's a keyrange like "-80", only rows that
  // match the keyrange are sent.
  // If Match is a table name instead of a regular expression,
  // the Filter can also be a select expression like this:
  // "select * from t", same as an empty Filter, or
  // "select * from t where in_keyrange('-80')", same as "-80", or
  // "select col1, col2 from t where in_keyrange(col1, 'hash', '-80'), or
  // What is allowed in a select expression depends on whether
  // it's a vstreamer or vreplication request. For more details,
  // please refer to the specific package documentation.
  // On the vreplication side, Filter can also accept a special
  // "exclude" value, which will cause the matched tables
  // to be excluded.
  // TODO(sougou): support this on vstreamer side also.
  string filter = 2;
  // ConvertEnumToText: optional, list per enum column name, the list of textual values.
  // When reading the binary log, all enum values are numeric. But sometimes it
  // is useful/needed to know what the textual mapping are.
  // Online DDL provides such use case.

  // Example: key="color", value="'red','green','blue'"
  map<string, string> convert_enum_to_text = 3;

  // ConvertCharset: optional mapping, between column name and a CharsetConversion.
  // This hints to vreplication that columns are encoded from/to non-trivial charsets
  // The map is only populated when either "from" or "to" charset of a column are non-trivial
  // trivial charsets are utf8 and ascii variants.
  map<string, CharsetConversion> convert_charset = 4;

  // SourceUniqueKeyColumns represents the ordered columns in the index used by rowstreamer to iterate the table
  // It is comma delimited, as in col1,col2,col3 (tokens are escaped via net/url)
  string source_unique_key_columns = 5;

  // TargetUniqueKeyColumns represents the ordered columns in that index used by vcopier and vplayer to apply rows
  // It is comma delimited, as in col1,col2,col3 (tokens are escaped via net/url)
  string target_unique_key_columns = 6;

  // SourceUniqueKeyTargetColumns represents the names of columns in target table, mapped from the chosen unique
  // key on source tables (some columns may be renamed from source to target)
  string source_unique_key_target_columns = 7;

  // ConvertIntToEnum lists any columns that are converted from an integral value into an enum.
  // such columns need to have special transofrmation of the data, from an integral format into a
  // string format. e.g. the value 0 needs to be converted to '0'.
  map<string, bool> convert_int_to_enum = 8;

   // ForceUniqueKey gives vtreamer a hint for `FORCE INDEX (...)` usage.
   string force_unique_key = 9;
}

// Filter represents a list of ordered rules. The first
// match wins.
message Filter {
  repeated Rule rules = 1;
  enum FieldEventMode {
    ERR_ON_MISMATCH = 0;
    BEST_EFFORT = 1;
  }
  // FieldEventMode specifies the behavior if there is a mismatch
  // between the current schema and the fields in the binlog. This
  // can happen if the binlog position is before a DDL that would
  // cause the fields to change. If vstreamer detects such 
  // an inconsistency, the behavior depends on the FieldEventMode.
  // If the value is ERR_ON_MISMATCH (default), then it errors out.
  // If it's BEST_EFFORT, it sends a field event with fake column
  // names as "@1", "@2", etc.
  FieldEventMode field_event_mode = 2;

  int64 workflow_type = 3;
  string workflow_name = 4;
}

// OnDDLAction lists the possible actions for DDLs.
enum OnDDLAction {
  IGNORE = 0;
  STOP = 1;
  EXEC = 2;
  EXEC_IGNORE = 3;
}

// VReplicationWorkflowType define types of vreplication workflows.
enum VReplicationWorkflowType {
  Materialize = 0;
  MoveTables = 1;
  CreateLookupIndex = 2;
  Migrate = 3;
  Reshard = 4;
  OnlineDDL = 5;
}

// VReplicationWorkflowSubType define types of vreplication workflows.
enum VReplicationWorkflowSubType {
  None = 0;
  Partial = 1;
  AtomicCopy = 2;
}

// VReplicationWorklfowState defines the valid states that a workflow can be in.
enum VReplicationWorkflowState {
  Unknown = 0;
  Init = 1;
  Stopped = 2;
  Copying = 3;
  Running = 4;
  Error = 5;
  Lagging = 6;
}

// BinlogSource specifies the source  and filter parameters for
// Filtered Replication. KeyRange and Tables are legacy. Filter
// is the new way to specify the filtering rules.
message BinlogSource {
  // the source keyspace
  string keyspace = 1;

  // the source shard
  string shard = 2;

  // the source tablet type
  topodata.TabletType tablet_type = 3;

  // KeyRange is set if the request is for a keyrange
  topodata.KeyRange key_range = 4;

  // Tables is set if the request is for a list of tables
  repeated string tables = 5;

  // Filter is set if we're using the generalized representation
  // for the filter.
  Filter filter = 6;

  // OnDdl specifies the action to be taken when a DDL is encountered.
  OnDDLAction on_ddl = 7;

  // Source is an external mysql. This attribute should be set to the username
  // to use in the connection
  string external_mysql = 8;

  // StopAfterCopy specifies if vreplication should be stopped
  // after copying is done.
  bool stop_after_copy = 9;

  // ExternalCluster is the name of the mounted cluster which has the source keyspace/db for this workflow
  // it is of the type <cluster_type.cluster_name>
  string external_cluster = 10;

  // SourceTimeZone is the time zone in which datetimes on the source were stored, provided as an option in MoveTables
  string source_time_zone = 11;
  // TargetTimeZone is not currently specifiable by the user, defaults to UTC for the forward workflows
  // and to the SourceTimeZone in reverse workflows
  string target_time_zone = 12;
}

// VEventType enumerates the event types. Many of these types
// will not be encountered in RBR mode.
enum VEventType {
  UNKNOWN = 0;
  GTID = 1;
  BEGIN = 2;
  COMMIT = 3;
  ROLLBACK = 4;
  DDL = 5;
  // INSERT, REPLACE, UPDATE, DELETE and SET will not be seen in RBR mode.
  INSERT = 6;
  REPLACE = 7;
  UPDATE = 8;
  DELETE = 9;
  SET = 10;
  // OTHER is a dummy event. If encountered, the current GTID must be
  // recorded by the client to be able to resume.
  OTHER = 11;
  ROW = 12;
  FIELD = 13;
  // HEARTBEAT is sent if there is inactivity. If a client does not
  // receive events beyond the hearbeat interval, it can assume that it's
  // lost connection to the vstreamer.
  HEARTBEAT = 14;
  // VGTID is generated by VTGate's VStream that combines multiple
  // GTIDs.
  VGTID = 15;
  JOURNAL = 16;
  VERSION = 17;
  LASTPK = 18;
  SAVEPOINT = 19;
  // COPY_COMPLETED is sent when VTGate's VStream copy operation is done.
  // If a client experiences some disruptions before receiving the event,
  // the client should restart the copy operation.
  COPY_COMPLETED = 20;
}


// RowChange represents one row change.
// If Before is set and not After, it's a delete.
// If After is set and not Before, it's an insert.
// If both are set, it's an update.
message RowChange {
  message Bitmap {
    int64 count = 1;
    bytes cols = 2;
  }
  query.Row before = 1;
  query.Row after = 2;
  // DataColumns is a bitmap of all columns: bit is set if column is
  // present in the after image.
  Bitmap data_columns = 3;
  // JsonPartialValues is a bitmap of any JSON columns, where the bit
  // is set if the value in the AFTER image is a partial JSON value
  // that is represented as an expression of
  // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is
  // used to add/update/remove a path in the JSON document. When the
  // value is used the fmt directive must be replaced by the actual
  // column name of the JSON field.
  Bitmap json_partial_values = 4;
}

// RowEvent represent row events for one table.
message RowEvent {
  string table_name = 1;
  repeated RowChange row_changes = 2;
  string keyspace = 3;
  string shard = 4;
  uint32 flags = 5; // https://dev.mysql.com/doc/dev/mysql-server/latest/classbinary__log_1_1Rows__event.html
  bool is_internal_table = 6; // set for sidecardb tables
}

// FieldEvent represents the field info for a table.
message FieldEvent {
  string table_name = 1;
  repeated query.Field fields = 2;
  string keyspace = 3;
  string shard = 4;

  // Field numbers in the gap between shard (4) and enum_set_string_values
  // (25) are NOT reserved and can be used.

  // Are ENUM and SET field values already mapped to strings in the ROW
  // events? This allows us to transition VTGate VStream consumers from
  // the pre v20 behavior of having to do this mapping themselves to the
  // v20+ behavior of not having to do this anymore and to expect string
  // values directly.
  // NOTE: because this is the use case, this is ONLY ever set today in
  // vstreams managed by the vstreamManager.
  bool enum_set_string_values = 25;
  bool is_internal_table = 26; // set for sidecardb tables

  // Add new members in the field number gap between shard (4) and
  // enum_set_string_values (25).
}

// ShardGtid contains the GTID position for one shard.
// It's used in a request for requesting a starting position.
// It's used in a response to transmit the current position
// of a shard. It's also used in a Journal to indicate the
// list of targets and shard positions to migrate to.
message ShardGtid {
  string keyspace = 1;
  string shard = 2;
  string gtid = 3;
  repeated TableLastPK table_p_ks = 4;
}

// A VGtid is a list of ShardGtids.
message VGtid {
  repeated ShardGtid shard_gtids = 1;
}

// KeyspaceShard represents a keyspace and shard.
message KeyspaceShard {
  string keyspace = 1;
  string shard = 2;
}

// MigrationType specifies the type of migration for the Journal.
enum MigrationType {
  TABLES = 0;
  SHARDS = 1;
}

// Journal contains the metadata for a journal event.
// The commit of a journal event indicates the point of no return
// for a migration.
message Journal {
  // Id represents a unique journal id.
  int64 id = 1;
  MigrationType migration_type = 2;
  // Tables is set if the journal represents a TABLES migration.
  repeated string tables = 3;
  // LocalPosition is the source position at which the migration happened.
  string local_position = 4;
  // ShardGtids is the list of targets to which the migration took place.
  repeated ShardGtid shard_gtids = 5;
  // Participants is the list of source participants for a migration.
  // Every participant is expected to have an identical journal entry.
  // While streaming, the client must wait for the journal entry to
  // be received from all pariticipants, and then replace them with new
  // streams specified by ShardGtid.
  // If a stream does not have all participants, a consistent migration
  // is not possible.
  repeated KeyspaceShard participants = 6;
  // SourceWorkflows is the list of workflows in the source shard that
  // were migrated to the target. If a migration fails after a Journal
  // is committed, this information is used to start the target streams
  // that were created prior to the creation of the journal.
  repeated string source_workflows = 7;
}

// VEvent represents a vstream event.
// A FieldEvent is sent once for every table, just before
// the first event for that table. The client is expected
// to cache this information and match it against the RowEvent
// which contains the table name.
// A GTID event always precedes a commitable event, which can be
// COMMIT, DDL or OTHER.
// OTHER events are non-material events that have no additional metadata.
message VEvent {
  VEventType type = 1;
  // Timestamp is the binlog timestamp in seconds.
  // The value should be ignored if 0.
  int64 timestamp = 2;
  // Gtid is set if the event type is GTID.
  string gtid = 3;
  // Statement is set if the event type is DDL, DML or SAVEPOINT.
  string statement = 4;
  // RowEvent is set if the event type is ROW.
  RowEvent row_event = 5;
  // FieldEvent is set if the event type is FIELD.
  FieldEvent field_event = 6;
  // Vgtid is set if the event type is VGTID.
  // This event is only generated by VTGate's VStream function.
  VGtid vgtid = 7;
  // Journal is set if the event type is JOURNAL. 
  Journal journal = 8;
  // Dml is set if the event type is INSERT, REPLACE, UPDATE or DELETE.
  string dml = 9;
  // CurrentTime specifies the current time when the message was sent.
  // This can be used to compenssate for clock skew.
  int64 current_time = 20;
  // LastPK is the last PK for a table
  LastPKEvent last_p_k_event = 21;
  // the source keyspace
  string keyspace = 22;
  // the source shard
  string shard = 23;
  // indicate that we are being throttled right now
  bool throttled = 24;
  // ThrottledReason is a human readable string that explains why the stream is throttled
  string throttled_reason = 25;
}

message MinimalTable {
  string name = 1;
  repeated query.Field fields = 2;
  repeated int64 p_k_columns = 3;
  // This will be PRIMARY when the actual primary key is used and it
  // will be the name of the Primary Key equivalent if one is used
  // instead. Otherwise it will be empty.
  string p_k_index_name = 4;
}

message MinimalSchema {
  repeated MinimalTable tables = 1;
}

message VStreamOptions {
  repeated string internal_tables = 1;
  map<string, string> config_overrides = 2;
}

// VStreamRequest is the payload for VStreamer
message VStreamRequest {
  vtrpc.CallerID effective_caller_id = 1;
  query.VTGateCallerID immediate_caller_id = 2;
  query.Target target = 3;

  string position = 4;
  Filter filter = 5;
  repeated TableLastPK table_last_p_ks = 6;

  VStreamOptions options = 7;
}

// VStreamResponse is the response from VStreamer
message VStreamResponse {
  repeated VEvent events = 1;
}

// VStreamRowsRequest is the payload for VStreamRows
message VStreamRowsRequest {
  vtrpc.CallerID effective_caller_id = 1;
  query.VTGateCallerID immediate_caller_id = 2;
  query.Target target = 3;

  string query = 4;
  query.QueryResult lastpk = 5;
  VStreamOptions options = 6;
}

// VStreamRowsResponse is the response from VStreamRows
message VStreamRowsResponse {
  repeated query.Field fields = 1;
  repeated query.Field pkfields = 2;
  string gtid = 3;
  repeated query.Row rows = 4;
  query.Row lastpk = 5;
  // Throttled indicates that rowstreamer is being throttled right now
  bool throttled = 6;
  // Heartbeat indicates that this is a heartbeat message
  bool heartbeat = 7;
  // ThrottledReason is a human readable string that explains why the stream is throttled
  string throttled_reason = 8;
}


// VStreamTablesRequest is the payload for VStreamTables
message VStreamTablesRequest {
  vtrpc.CallerID effective_caller_id = 1;
  query.VTGateCallerID immediate_caller_id = 2;
  query.Target target = 3;
  VStreamOptions options = 4;
}

// VStreamTablesResponse is the response from VStreamTables
message VStreamTablesResponse {
  string table_name = 1;
  repeated query.Field fields = 2;
  repeated query.Field pkfields = 3;
  string gtid = 4;
  repeated query.Row rows =  5;
  query.Row lastpk = 6;
}

message LastPKEvent {
  TableLastPK table_last_p_k = 1;
  bool completed = 2;
}

message TableLastPK {
  string table_name = 1;
  query.QueryResult lastpk = 3;
}

// VStreamResultsRequest is the payload for VStreamResults
// The ids match VStreamRows, in case we decide to merge the two.
// The ids match VStreamRows, in case we decide to merge the two.
message VStreamResultsRequest {
  vtrpc.CallerID effective_caller_id = 1;
  query.VTGateCallerID immediate_caller_id = 2;
  query.Target target = 3;

  string query = 4;
}

// VStreamResultsResponse is the response from VStreamResults
// The ids match VStreamRows, in case we decide to merge the two.
message VStreamResultsResponse {
  repeated query.Field fields = 1;
  string gtid = 3;
  repeated query.Row rows = 4;
}