ENG-2703: Co-partition table syntax support

Summary: Implemented co-partition in the yql layer. Throws feature not suppoted error after performing initial checks. Test Plan: Jenkins, ql-create-table-test.cc Reviewers: robert, neil, mihnea Reviewed By: mihnea Subscribers: kannan, yql Differential Revision: https://phabricator.dev.yugabyte.com/D3853
yugabyte · Jan 12, 2018 · 290d886 · 290d886
1 parent fbd1caf
commit 290d886
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 11 deletions.
diff --git a/src/yb/yql/cql/ql/exec/executor.cc b/src/yb/yql/cql/ql/exec/executor.cc
@@ -261,7 +261,8 @@ Status Executor::ExecPTNode(const PTCreateTable *tnode) {
   }
 
   TableProperties table_properties;
-  if (!tnode->ToTableProperties(&table_properties).ok()) {
+  s = tnode->ToTableProperties(&table_properties);
+  if (!s.ok()) {
     return exec_context_->Error(tnode->columns().front(), s, ErrorCode::INVALID_TABLE_DEFINITION);
   }
 

diff --git a/src/yb/yql/cql/ql/kwlist.h b/src/yb/yql/cql/ql/kwlist.h
@@ -368,6 +368,7 @@ PG_KEYWORD("rows", ROWS, UNRESERVED_KEYWORD)
 PG_KEYWORD("rule", RULE, UNRESERVED_KEYWORD)
 PG_KEYWORD("savepoint", SAVEPOINT, UNRESERVED_KEYWORD)
 PG_KEYWORD("schema", SCHEMA, RESERVED_KEYWORD)
+PG_KEYWORD("scheme", SCHEME, UNRESERVED_KEYWORD)
 PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD)
 PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD)
 PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD)

diff --git a/src/yb/yql/cql/ql/parser/parser_gram.y b/src/yb/yql/cql/ql/parser/parser_gram.y
@@ -589,9 +589,9 @@ using namespace yb::ql;
                           OBJECT_P OF OFF OFFSET OIDS ON ONLY OPERATOR OPTION OPTIONS OR ORDER
                           ORDINALITY OUT_P OUTER_P OVER OVERLAPS OVERLAY OWNED OWNER
 
-                          PARSER PARTIAL PARTITION PASSING PASSWORD PLACING PLANS POLICY
-                          POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY PRIOR
-                          PRIVILEGES PROCEDURAL PROCEDURE PROGRAM
+                          PARSER PARTIAL PARTITION PASSING PASSWORD PLACING PLANS POLICY POSITION
+                          PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY PRIOR PRIVILEGES
+                          PROCEDURAL PROCEDURE PROGRAM
 
                           QUOTE
 
@@ -600,11 +600,11 @@ using namespace yb::ql;
                           RESTART RESTRICT RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
                           ROW ROWS RULE
 
-                          SAVEPOINT SCHEMA SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE
+                          SAVEPOINT SCHEMA SCHEME SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE
                           SEQUENCES SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF
-                          SHARE SHOW SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE
-                          STANDALONE_P START STATEMENT STATIC STATISTICS STDIN STDOUT STORAGE
-                          STRICT_P STRIP_P SUBSTRING SYMMETRIC SYSID SYSTEM_P
+                          SHARE SHOW SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P
+                          STABLE STANDALONE_P START STATEMENT STATIC STATISTICS STDIN STDOUT
+                          STORAGE STRICT_P STRIP_P SUBSTRING SYMMETRIC SYSID SYSTEM_P
 
                           TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P
                           THEN TIME TIMESTAMP TIMEUUID TINYINT TO TOKEN TRAILING TRANSACTION
@@ -1450,6 +1450,10 @@ table_property:
   | COMPACT STORAGE {
     $$ = MAKE_NODE(@1, PTTablePropertyListNode);
   }
+  | PARTITION SCHEME OF qualified_name {
+    PTTableProperty::SharedPtr pt_table_property = MAKE_NODE(@4, PTTableProperty, $4);
+    $$ = MAKE_NODE(@1, PTTablePropertyListNode, pt_table_property);
+  }
 ;
 
 property_map:
@@ -4949,6 +4953,7 @@ unreserved_keyword:
   | ROWS { $$ = $1; }
   | RULE { $$ = $1; }
   | SAVEPOINT { $$ = $1; }
+  | SCHEME { $$ = $1; }
   | SCROLL { $$ = $1; }
   | SEARCH { $$ = $1; }
   | SECOND_P { $$ = $1; }

diff --git a/src/yb/yql/cql/ql/ptree/pt_table_property.cc b/src/yb/yql/cql/ql/ptree/pt_table_property.cc
@@ -51,8 +51,7 @@ PTTableProperty::PTTableProperty(MemoryContext *memctx,
                    const MCSharedPtr<MCString>& lhs,
                    const PTExpr::SharedPtr& rhs)
     : PTProperty(memctx, loc, lhs, rhs),
-      property_type_(PropertyType::kTableProperty) {
-}
+      property_type_(PropertyType::kTableProperty) {}
 
 PTTableProperty::PTTableProperty(MemoryContext *memctx,
                                  YBLocation::SharedPtr loc,
@@ -61,6 +60,12 @@ PTTableProperty::PTTableProperty(MemoryContext *memctx,
     : PTProperty(memctx, loc), name_(name), direction_(direction),
       property_type_(PropertyType::kClusteringOrder) {}
 
+PTTableProperty::PTTableProperty(MemoryContext *memctx,
+                                 YBLocation::SharedPtr loc,
+                                 const PTQualifiedName::SharedPtr tname)
+    : PTProperty(memctx, loc), property_type_(PropertyType::kCoPartitionTable),
+      copartition_table_name_(tname) {}
+
 PTTableProperty::PTTableProperty(MemoryContext *memctx,
                                  YBLocation::SharedPtr loc)
     : PTProperty(memctx, loc) {
@@ -102,6 +107,41 @@ Status PTTableProperty::AnalyzeSpeculativeRetry(const string &val) {
 
 CHECKED_STATUS PTTableProperty::Analyze(SemContext *sem_context) {
 
+  if (property_type_ == PropertyType::kCoPartitionTable) {
+    RETURN_NOT_OK(copartition_table_name_->AnalyzeName(sem_context, OBJECT_TABLE));
+
+    bool is_system; // ignored
+    int num_key_cols, num_hash_key_cols;
+    MCVector<ColumnDesc> copartition_table_columns(sem_context->PTempMem());
+    RETURN_NOT_OK(sem_context->LookupTable(copartition_table_name_->ToTableName(),
+                                           copartition_table_name_->loc(), /* write_table = */ true,
+                                           &copartition_table_, &is_system,
+                                           &copartition_table_columns, &num_key_cols,
+                                           &num_hash_key_cols));
+    if (sem_context->current_create_table_stmt()->hash_columns().size() != num_hash_key_cols) {
+      return sem_context->Error(this, Substitute("The number of hash keys in the current table "
+                                "differ from the number of hash keys in '$0'.",
+                                copartition_table_name_->ToTableName().table_name()).c_str(),
+                                ErrorCode::INCOMPATIBLE_COPARTITION_SCHEMA);
+    }
+
+    int index = 0;
+    for (auto hash_col : sem_context->current_create_table_stmt()->hash_columns()) {
+      auto type = hash_col->ql_type();
+      auto base_type = copartition_table_columns[index].ql_type();
+      if (type != base_type) {
+        return sem_context->Error(this, Substitute("The hash key '$0' in the current table has a "
+                                  "different datatype from the corresponding hash key in '$1'",
+                                  hash_col->yb_name(),
+                                  copartition_table_name_->ToTableName().table_name()).c_str(),
+                                  ErrorCode::INCOMPATIBLE_COPARTITION_SCHEMA);
+      }
+      index++;
+    }
+
+    return Status::OK();
+  }
+
   // Verify we have a valid property name in the lhs.
   const auto& table_property_name = lhs_->c_str();
   auto iterator = kPropertyDataTypes.find(table_property_name);
@@ -207,6 +247,9 @@ std::ostream& operator<<(ostream& os, const PropertyType& property_type) {
     case PropertyType::kTablePropertyMap:
       os << "kTablePropertyMap";
       break;
+    case PropertyType::kCoPartitionTable:
+      os << "kCoPartitionTable";
+      break;
   }
   return os;
 }
@@ -237,6 +280,10 @@ CHECKED_STATUS PTTablePropertyListNode::Analyze(SemContext *sem_context) {
         table_properties.insert(table_property_name);
         break;
       }
+      case PropertyType ::kCoPartitionTable: {
+        RETURN_NOT_OK(tnode->Analyze(sem_context));
+        break;
+      }
       case PropertyType::kClusteringOrder: {
         const auto &column_name = tnode->name().c_str();
         const PTColumnDefinition *col = sem_context->GetColumnDefinition(tnode->name());
@@ -293,11 +340,16 @@ CHECKED_STATUS PTTablePropertyListNode::Analyze(SemContext *sem_context) {
 Status PTTableProperty::SetTableProperty(yb::TableProperties *table_property) const {
   // TODO: Also reject properties that cannot be changed during alter table (like clustering order)
 
-  // Clustering order not handled here.
+  // Clustering order and copartitioning not handled here.
   if (property_type_ == PropertyType::kClusteringOrder) {
     return Status::OK();
   }
 
+  if(property_type_ == PropertyType::kCoPartitionTable) {
+    // TODO: This should be enabled once the YBase layer changes are finished
+    return STATUS(NotSupported, "Co-partitioning is not implemented yet");
+  }
+
   string table_property_name;
   ToLowerCase(lhs_->c_str(), &table_property_name);
   auto iterator = kPropertyDataTypes.find(table_property_name);

diff --git a/src/yb/yql/cql/ql/ptree/pt_table_property.h b/src/yb/yql/cql/ql/ptree/pt_table_property.h
@@ -31,6 +31,7 @@ enum class PropertyType : int {
   kTableProperty = 0,
   kClusteringOrder,
   kTablePropertyMap,
+  kCoPartitionTable,
 };
 
 class PTTableProperty : public PTProperty {
@@ -73,6 +74,11 @@ class PTTableProperty : public PTProperty {
                   const MCSharedPtr<MCString>& name,
                   const PTOrderBy::Direction direction);
 
+  // Constructor for PropertyType::kCoPartitionTable
+  PTTableProperty(MemoryContext *memctx,
+                  YBLocation::SharedPtr loc,
+                  const PTQualifiedName::SharedPtr tname);
+
   PTTableProperty(MemoryContext *memctx,
                   YBLocation::SharedPtr loc);
 
@@ -104,6 +110,11 @@ class PTTableProperty : public PTProperty {
     return direction_;
   }
 
+  PTQualifiedName table_name() const {
+    DCHECK_EQ(property_type_, PropertyType::kCoPartitionTable);
+    return *copartition_table_name_;
+  }
+
  protected:
   bool IsValidProperty(const string& property_name) {
     return kPropertyDataTypes.find(property_name) != kPropertyDataTypes.end();
@@ -112,6 +123,8 @@ class PTTableProperty : public PTProperty {
   MCSharedPtr<MCString> name_;
   PTOrderBy::Direction direction_;
   PropertyType property_type_;
+  PTQualifiedName::SharedPtr copartition_table_name_;
+  std::shared_ptr<client::YBTable> copartition_table_;
 
  private:
   CHECKED_STATUS AnalyzeSpeculativeRetry(const string &val);

diff --git a/src/yb/yql/cql/ql/test/ql-create-table-test.cc b/src/yb/yql/cql/ql/test/ql-create-table-test.cc
@@ -249,5 +249,76 @@ TEST_F(TestQLCreateTable, TestQLCreateTableWithClusteringOrderBy) {
       "Invalid Table Property. Not a clustering key colum");
 }
 
+TEST_F(TestQLCreateTable, TestQLCreateTableWithPartitionScemeOf) {
+  // Init the simulated cluster.
+  ASSERT_NO_FATALS(CreateSimulatedCluster());
+
+  // Get an available processor.
+  TestQLProcessor *processor = GetQLProcessor();
+
+  const string table1 = "devices(supplier_id INT, device_id DOUBLE, model_year INT, "
+      "device_name TEXT, PRIMARY KEY((supplier_id, device_id), model_year));";
+  const string table2 = "descriptions1(supplier_id INT, device_id DOUBLE, description TEXT, "
+      "PRIMARY KEY((supplier_id, device_id))) with partition scheme of devices;";
+  const string table3 = "descriptions2(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, dev_id), image_id)) "
+      "with partition scheme of devices;";
+  const string table4 = "descriptions3(supplier_id INT, device_id DOUBLE, description TEXT, "
+      "PRIMARY KEY(supplier_id, device_id)) with partition scheme of devices;";
+  const string table5 = "descriptions4(supplier_id INT, device_id DOUBLE, model_year INT, "
+      "description TEXT, PRIMARY KEY((supplier_id, device_id, model_year))) "
+      "with partition scheme of devices;";
+  const string table6 = "descriptions5(supplier_id INT, device_id DOUBLE, description TEXT, "
+      "PRIMARY KEY((supplier_id, device_id))) with partition scheme of non_existing_table;";
+  const string table7 = "descriptions6(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, description))) with partition scheme of devices;";
+  const string table8 = "descriptions7(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((dev_id, supp_id))) with partition scheme of devices;";
+  const string table9 = "descriptions8(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, image_id))) with partition scheme of devices;";
+  const string table10 = "descriptions9(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((description, image_id))) with partition scheme of devices;";
+  const string table11 = "descriptions10(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, dev_id))) with partition scheme devices;";
+  const string table12 = "descriptions11(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, dev_id))) with partition schema of devices;";
+  const string table13 = "descriptions12(supp_id INT, dev_id DOUBLE, description TEXT, "
+      "image_id INT, PRIMARY KEY((supp_id, dev_id))) with partitioning scheme of devices;";
+
+  // Create the devices tables.
+  EXEC_VALID_STMT(CreateStmt(table1));
+
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table2),
+                                 "Co-partitioning is not implemented yet");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table3),
+                                 "Co-partitioning is not implemented yet");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table4),
+                                 "The number of hash keys in the current table "
+                                     "differ from the number of hash keys in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table5),
+                                 "The number of hash keys in the current table "
+                                     "differ from the number of hash keys in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table6),
+                                 "Table Not Found");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table7),
+                                 "The hash key 'description' in the current table has a different "
+                                     "datatype from the corresponding hash key in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table8),
+                                 "The hash key 'dev_id' in the current table has a different "
+                                     "datatype from the corresponding hash key in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table9),
+                                 "The hash key 'image_id' in the current table has a different "
+                                     "datatype from the corresponding hash key in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table10),
+                                 "The hash key 'description' in the current table has a different "
+                                     "datatype from the corresponding hash key in 'devices'");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table11),
+                                 "syntax error");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table12),
+                                 "syntax error");
+  EXEC_INVALID_TABLE_CREATE_STMT(CreateStmt(table13),
+                                 "syntax error");
+}
+
 } // namespace ql
 } // namespace yb
diff --git a/src/yb/yql/cql/ql/util/errcodes.cc b/src/yb/yql/cql/ql/util/errcodes.cc
@@ -87,6 +87,7 @@ const std::unordered_map<ErrorCode, const char*, util::EnumHash> kQLErrorMessage
   { ErrorCode::INVALID_COUNTING_EXPR, "Counters can only be incremented or decremented" },
   { ErrorCode::DUPLICATE_TYPE, "Duplicate Type" },
   { ErrorCode::DUPLICATE_TYPE_FIELD, "Duplicate Type Field" },
+  { ErrorCode::INCOMPATIBLE_COPARTITION_SCHEMA, "Incompatible Copartition Schema" },
 
   //------------------------------------------------------------------------------------------------
   // Execution errors [-300, x).

diff --git a/src/yb/yql/cql/ql/util/errcodes.h b/src/yb/yql/cql/ql/util/errcodes.h
@@ -97,6 +97,7 @@ enum class ErrorCode : int64_t {
   DUPLICATE_TYPE = -218,
   DUPLICATE_TYPE_FIELD = -219,
   ALTER_KEY_COLUMN = -220,
+  INCOMPATIBLE_COPARTITION_SCHEMA = -221,
 
   //------------------------------------------------------------------------------------------------
   // Execution errors [-300, x).