From 007907460d72a3aa82b222567c08589c62efb614 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Mon, 7 Mar 2016 20:43:31 -0800 Subject: [PATCH] Address comments from #11408 + fix style --- .../apache/spark/sql/execution/SparkQl.scala | 43 +- .../command/AlterTableCommandParser.scala | 677 +++++++++--------- .../spark/sql/execution/command/ddl.scala | 96 +-- .../{DDLSuite.scala => DDLCommandSuite.scala} | 193 ++--- 4 files changed, 473 insertions(+), 536 deletions(-) rename sql/core/src/test/scala/org/apache/spark/sql/execution/command/{DDLSuite.scala => DDLCommandSuite.scala} (92%) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala index 7d782a4d8fb6d..6a456b67340f1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala @@ -29,20 +29,27 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly import ParserUtils._ /** Check if a command should not be explained. */ - protected def isNoExplainCommand(command: String): Boolean = + protected def isNoExplainCommand(command: String): Boolean = { "TOK_DESCTABLE" == command || "TOK_ALTERTABLE" == command + } protected def extractProps( node: ASTNode, - firstLevelNodeStr: String, - secondLevelNodeStr: String): Seq[(String, String)] = node match { - case Token(firstLevelNodeStr, options) => - options.map { - case Token(secondLevelNodeStr, keysAndValue) => - val key = keysAndValue.init.map(x => unquoteString(x.text)).mkString(".") - val value = unquoteString(keysAndValue.last.text) - (key, value) - } + firstLevelProp: String, + secondLevelProp: String): Seq[(String, String)] = { + node match { + case Token(x, options) if x == firstLevelProp => + options.map { + case Token(y, keysAndValue) if y == secondLevelProp => + val key = keysAndValue.init.map(x => unquoteString(x.text)).mkString(".") + val value = unquoteString(keysAndValue.last.text) + (key, value) + case _ => + throw new AnalysisException(s"Expected property '$secondLevelProp' in '${node.text}'") + } + case _ => + throw new AnalysisException(s"Expected property '$firstLevelProp' in '${node.text}'") + } } protected override def nodeToPlan(node: ASTNode): LogicalPlan = { @@ -100,7 +107,7 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly propList.flatMap(extractProps(_, "TOK_DBPROPLIST", "TOK_TABLEPROPERTY")) }.toMap - CreateDataBase(databaseName, allowExisting.isDefined, location, comment, props)(node.source) + CreateDatabase(databaseName, allowExisting.isDefined, location, comment, props)(node.source) case Token("TOK_CREATEFUNCTION", func :: as :: createFuncArgs) => val funcName = func.map(x => unquoteString(x.text)).mkString(".") @@ -137,12 +144,12 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly Some(Token("TOK_TABLEPROVIDER", providerNameParts)), tableOpts, tableAs) = getClauses(Seq( - "TEMPORARY", - "TOK_IFNOTEXISTS", - "TOK_TABNAME", "TOK_TABCOLLIST", - "TOK_TABLEPROVIDER", - "TOK_TABLEOPTIONS", - "TOK_QUERY"), createTableArgs) + "TEMPORARY", + "TOK_IFNOTEXISTS", + "TOK_TABNAME", "TOK_TABCOLLIST", + "TOK_TABLEPROVIDER", + "TOK_TABLEOPTIONS", + "TOK_QUERY"), createTableArgs) val tableIdent: TableIdentifier = extractTableIdent(tabName) @@ -156,7 +163,7 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly val options: Map[String, String] = tableOpts.toSeq.flatMap(extractProps(_, "TOK_TABLEOPTIONS", "TOK_TABLEOPTION")).toMap - val asClause = tableAs.map(nodeToPlan(_)) + val asClause = tableAs.map(nodeToPlan) if (temp.isDefined && allowExisting.isDefined) { throw new AnalysisException( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala index b99753b4ddabd..58ff8de6c52d4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommandParser.scala @@ -27,9 +27,28 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.types.StructType + +/** + * Helper object to parse alter table commands. + */ object AlterTableCommandParser { import ParserUtils._ + /** + * Parse the given node assuming it is an alter table command. + */ + def parse(v1: ASTNode): LogicalPlan = { + v1.children match { + case (tabName @ Token("TOK_TABNAME", _)) :: restNodes => + val tableIdent: TableIdentifier = extractTableIdent(tabName) + val partitionSpec = getClauseOption("TOK_PARTSPEC", v1.children) + val partition = partitionSpec.flatMap(parsePartitionSpec) + matchAlterTableCommands(v1, restNodes, tableIdent, partition) + case _ => + throw new AnalysisException(s"Could not parse alter table command: '${v1.text}'") + } + } + private def cleanAndUnquoteString(s: String): String = { cleanIdentifier(unquoteString(s)) } @@ -48,348 +67,344 @@ object AlterTableCommandParser { } } - private def extractTableProps(node: ASTNode): Map[String, Option[String]] = node match { - case Token("TOK_TABLEPROPERTIES", propsList) => - propsList.flatMap { - case Token("TOK_TABLEPROPLIST", props) => - props.map { - case Token("TOK_TABLEPROPERTY", key :: Token("TOK_NULL", Nil) :: Nil) => - val k = cleanAndUnquoteString(key.text) - (k, None) - case Token("TOK_TABLEPROPERTY", key :: value :: Nil) => - val k = cleanAndUnquoteString(key.text) - val v = cleanAndUnquoteString(value.text) - (k, Some(v)) - } - }.toMap - } - - def parse(v1: ASTNode): LogicalPlan = v1.children match { - case (tabName @ Token("TOK_TABNAME", _)) :: restNodes => - val tableIdent: TableIdentifier = extractTableIdent(tabName) - val partitionSpec = getClauseOption("TOK_PARTSPEC", v1.children) - val partition = partitionSpec.flatMap(parsePartitionSpec) - matchAlterTableCommands(v1, restNodes, tableIdent, partition) - case _ => - throw new NotImplementedError(v1.text) + private def extractTableProps(node: ASTNode): Map[String, Option[String]] = { + node match { + case Token("TOK_TABLEPROPERTIES", propsList) => + propsList.flatMap { + case Token("TOK_TABLEPROPLIST", props) => + props.map { + case Token("TOK_TABLEPROPERTY", key :: Token("TOK_NULL", Nil) :: Nil) => + val k = cleanAndUnquoteString(key.text) + (k, None) + case Token("TOK_TABLEPROPERTY", key :: value :: Nil) => + val k = cleanAndUnquoteString(key.text) + val v = cleanAndUnquoteString(value.text) + (k, Some(v)) + } + }.toMap + case _ => + throw new AnalysisException( + s"Expected table properties in alter table command: '${node.text}'") + } } + // TODO: This method is massive. Break it down. Also, add some comments... private def matchAlterTableCommands( node: ASTNode, nodes: Seq[ASTNode], tableIdent: TableIdentifier, - partition: Option[Map[String, Option[String]]]): LogicalPlan = nodes match { - case rename @ Token("TOK_ALTERTABLE_RENAME", renameArgs) :: _ => - val renamedTable = getClause("TOK_TABNAME", renameArgs) - val renamedTableIdent: TableIdentifier = extractTableIdent(renamedTable) - AlterTableRename(tableIdent, renamedTableIdent)(node.source) - - case Token("TOK_ALTERTABLE_PROPERTIES", args) :: _ => - val setTableProperties = extractTableProps(args.head) - AlterTableSetProperties( - tableIdent, - setTableProperties)(node.source) - - case Token("TOK_ALTERTABLE_DROPPROPERTIES", args) :: _ => - val dropTableProperties = extractTableProps(args.head) - val allowExisting = getClauseOption("TOK_IFEXISTS", args) - AlterTableDropProperties( - tableIdent, - dropTableProperties, allowExisting.isDefined)(node.source) - - case Token("TOK_ALTERTABLE_SERIALIZER", Token(serdeClassName, Nil) :: serdeArgs) :: _ => - // When SET SERDE serde_classname WITH SERDEPROPERTIES, this is None - val serdeProperties: Option[Map[String, Option[String]]] = - serdeArgs.headOption.map(extractTableProps) - - AlterTableSerDeProperties( - tableIdent, - Some(cleanAndUnquoteString(serdeClassName)), - serdeProperties, - partition)(node.source) - - case Token("TOK_ALTERTABLE_SERDEPROPERTIES", args) :: _ => - val serdeProperties: Map[String, Option[String]] = extractTableProps(args.head) - - AlterTableSerDeProperties( - tableIdent, - None, - Some(serdeProperties), - partition)(node.source) - - case Token("TOK_ALTERTABLE_CLUSTER_SORT", clusterAndSoryByArgs :: Nil) :: _ => - val (buckets, noClustered, noSorted) = clusterAndSoryByArgs match { - case Token("TOK_ALTERTABLE_BUCKETS", bucketArgsHead :: bucketArgs) => - val bucketCols = bucketArgsHead.children.map(_.text) - - val (sortCols, sortDirections, numBuckets) = { - if (bucketArgs.head.text == "TOK_TABCOLNAME") { - val (cols, directions) = bucketArgs.head.children.map { - case Token("TOK_TABSORTCOLNAMEASC", Token(colName, Nil) :: Nil) => - (colName, Ascending) - case Token("TOK_TABSORTCOLNAMEDESC", Token(colName, Nil) :: Nil) => - (colName, Descending) - }.unzip - (cols, directions, bucketArgs.last.text.toInt) - } else { - (Nil, Nil, bucketArgs.head.text.toInt) - } - } - - (Some(BucketSpec(numBuckets, bucketCols, sortCols, sortDirections)), - false, false) - case Token("TOK_NOT_CLUSTERED", Nil) => - (None, true, false) - case Token("TOK_NOT_SORTED", Nil) => - (None, false, true) - } - - AlterTableStoreProperties( - tableIdent, - buckets, - noClustered, - noSorted)(node.source) - - case Token("TOK_ALTERTABLE_BUCKETS", Token(bucketNum, Nil) :: Nil) :: _ => - val num = bucketNum.toInt - val buckets = Some(BucketSpec(num, Nil, Nil, Nil)) - AlterTableStoreProperties( - tableIdent, - buckets, - false, - false)(node.source) - - case Token("TOK_ALTERTABLE_SKEWED", Nil) :: _ => - // ALTER TABLE table_name NOT SKEWED - AlterTableSkewed(tableIdent, Nil, Nil, false, true, false)(node.source) - - case Token("TOK_ALTERTABLE_SKEWED", Token("TOK_STOREDASDIRS", Nil) :: Nil) => - // ALTER TABLE table_name NOT STORED AS DIRECTORIES - AlterTableSkewed(tableIdent, Nil, Nil, false, false, true)(node.source) - - case (tableSkewed @ Token("TOK_ALTERTABLE_SKEWED", _)) :: _ => - val skewedArgs = tableSkewed match { - case Token("TOK_ALTERTABLE_SKEWED", skewedArgs :: Nil) => - skewedArgs match { - case Token("TOK_TABLESKEWED", skewedCols :: skewedValues :: stored) => - val cols = skewedCols.children.map(n => cleanAndUnquoteString(n.text)) - val values = skewedValues match { - case Token("TOK_TABCOLVALUE", values) => - Seq(values.map(n => cleanAndUnquoteString(n.text))) - case Token("TOK_TABCOLVALUE_PAIR", pairs) => - pairs.map { - case Token("TOK_TABCOLVALUES", values :: Nil) => - values match { - case Token("TOK_TABCOLVALUE", vals) => - vals.map(n => cleanAndUnquoteString(n.text)) - } - } - } - - val storedAsDirs = stored match { - case Token("TOK_STOREDASDIRS", Nil) :: Nil => true - case _ => false + partition: Option[Map[String, Option[String]]]): LogicalPlan = { + nodes match { + case rename @ Token("TOK_ALTERTABLE_RENAME", renameArgs) :: _ => + val renamedTable = getClause("TOK_TABNAME", renameArgs) + val renamedTableIdent: TableIdentifier = extractTableIdent(renamedTable) + AlterTableRename(tableIdent, renamedTableIdent)(node.source) + + case Token("TOK_ALTERTABLE_PROPERTIES", args) :: _ => + val setTableProperties = extractTableProps(args.head) + AlterTableSetProperties( + tableIdent, + setTableProperties)(node.source) + + case Token("TOK_ALTERTABLE_DROPPROPERTIES", args) :: _ => + val dropTableProperties = extractTableProps(args.head) + val allowExisting = getClauseOption("TOK_IFEXISTS", args) + AlterTableDropProperties( + tableIdent, + dropTableProperties, allowExisting.isDefined)(node.source) + + case Token("TOK_ALTERTABLE_SERIALIZER", Token(serdeClassName, Nil) :: serdeArgs) :: _ => + // When SET SERDE serde_classname WITH SERDEPROPERTIES, this is None + val serdeProperties: Option[Map[String, Option[String]]] = + serdeArgs.headOption.map(extractTableProps) + + AlterTableSerDeProperties( + tableIdent, + Some(cleanAndUnquoteString(serdeClassName)), + serdeProperties, + partition)(node.source) + + case Token("TOK_ALTERTABLE_SERDEPROPERTIES", args) :: _ => + val serdeProperties: Map[String, Option[String]] = extractTableProps(args.head) + + AlterTableSerDeProperties( + tableIdent, + None, + Some(serdeProperties), + partition)(node.source) + + case Token("TOK_ALTERTABLE_CLUSTER_SORT", clusterAndSoryByArgs :: Nil) :: _ => + val (buckets, noClustered, noSorted) = clusterAndSoryByArgs match { + case Token("TOK_ALTERTABLE_BUCKETS", bucketArgsHead :: bucketArgs) => + val bucketCols = bucketArgsHead.children.map(_.text) + val (sortCols, sortDirections, numBuckets) = { + if (bucketArgs.head.text == "TOK_TABCOLNAME") { + val (cols, directions) = bucketArgs.head.children.map { + case Token("TOK_TABSORTCOLNAMEASC", Token(colName, Nil) :: Nil) => + (colName, Ascending) + case Token("TOK_TABSORTCOLNAMEDESC", Token(colName, Nil) :: Nil) => + (colName, Descending) + }.unzip + (cols, directions, bucketArgs.last.text.toInt) + } else { + (Nil, Nil, bucketArgs.head.text.toInt) } + } + val bucketSpec = BucketSpec(numBuckets, bucketCols, sortCols, sortDirections) + (Some(bucketSpec), false, false) + case Token("TOK_NOT_CLUSTERED", Nil) => + (None, true, false) + case Token("TOK_NOT_SORTED", Nil) => + (None, false, true) + } + AlterTableStoreProperties( + tableIdent, + buckets, + noClustered, + noSorted)(node.source) + + case Token("TOK_ALTERTABLE_BUCKETS", Token(bucketNum, Nil) :: Nil) :: _ => + val num = bucketNum.toInt + val buckets = Some(BucketSpec(num, Nil, Nil, Nil)) + AlterTableStoreProperties( + tableIdent, + buckets, + noClustered = false, + noSorted = false)(node.source) + + case Token("TOK_ALTERTABLE_SKEWED", Nil) :: _ => + // ALTER TABLE table_name NOT SKEWED + AlterTableSkewed( + tableIdent, + Nil, + Nil, + storedAsDirs = false, + notSkewed = true, + notStoredAsDirs = false)(node.source) + + case Token("TOK_ALTERTABLE_SKEWED", Token("TOK_STOREDASDIRS", Nil) :: Nil) => + // ALTER TABLE table_name NOT STORED AS DIRECTORIES + AlterTableSkewed( + tableIdent, + Nil, + Nil, + storedAsDirs = false, + notSkewed = false, + notStoredAsDirs = true)(node.source) + + case (tableSkewed @ Token("TOK_ALTERTABLE_SKEWED", _)) :: _ => + val skewedArgs = tableSkewed match { + case Token("TOK_ALTERTABLE_SKEWED", args :: Nil) => + args match { + case Token("TOK_TABLESKEWED", skewedCols :: skewedValues :: stored) => + val cols = skewedCols.children.map(n => cleanAndUnquoteString(n.text)) + val values = skewedValues match { + case Token("TOK_TABCOLVALUE", colVal) => + Seq(colVal.map(n => cleanAndUnquoteString(n.text))) + case Token("TOK_TABCOLVALUE_PAIR", pairs) => + pairs.map { + case Token("TOK_TABCOLVALUES", colVals :: Nil) => + colVals match { + case Token("TOK_TABCOLVALUE", vals) => + vals.map(n => cleanAndUnquoteString(n.text)) + } + } + } + + val storedAsDirs = stored match { + case Token("TOK_STOREDASDIRS", Nil) :: Nil => true + case _ => false + } + + (cols, values, storedAsDirs) + } + } + val (cols, values, storedAsDirs) = skewedArgs + AlterTableSkewed( + tableIdent, + cols, + values, + storedAsDirs, + notSkewed = false, + notStoredAsDirs = false)(node.source) + + case Token("TOK_ALTERTABLE_SKEWED_LOCATION", + Token("TOK_SKEWED_LOCATIONS", + Token("TOK_SKEWED_LOCATION_LIST", locationMaps) :: Nil) :: Nil) :: _ => + val skewedMaps = locationMaps.map { + case Token("TOK_SKEWED_LOCATION_MAP", key :: value :: Nil) => + val k = key match { + case Token(const, Nil) => Seq(cleanAndUnquoteString(const)) + case Token("TOK_TABCOLVALUES", values :: Nil) => + values match { + case Token("TOK_TABCOLVALUE", vals) => + vals.map(n => cleanAndUnquoteString(n.text)) + } + } + (k, cleanAndUnquoteString(value.text)) + }.toMap + AlterTableSkewedLocation(tableIdent, skewedMaps)(node.source) - (cols, values, storedAsDirs) - } - } - - val (cols, values, storedAsDirs) = skewedArgs - - AlterTableSkewed( - tableIdent, - cols, - values, - storedAsDirs, - notSkewed = false, - notStoredAsDirs = false)(node.source) - - case Token("TOK_ALTERTABLE_SKEWED_LOCATION", - Token("TOK_SKEWED_LOCATIONS", - Token("TOK_SKEWED_LOCATION_LIST", locationMaps) :: Nil) :: Nil) :: _ => - val skewedMaps = locationMaps.map { - case Token("TOK_SKEWED_LOCATION_MAP", key :: value :: Nil) => - val k = key match { - case Token(const, Nil) => Seq(cleanAndUnquoteString(const)) - case Token("TOK_TABCOLVALUES", values :: Nil) => - values match { - case Token("TOK_TABCOLVALUE", vals) => - vals.map(n => cleanAndUnquoteString(n.text)) - } - } - (k, cleanAndUnquoteString(value.text)) - }.toMap - AlterTableSkewedLocation(tableIdent, skewedMaps)(node.source) - - case Token("TOK_ALTERTABLE_ADDPARTS", addPartsArgs) :: _ => - val (allowExisting, parts) = addPartsArgs match { - case Token("TOK_IFNOTEXISTS", Nil) :: others => (true, others) - case _ => (false, addPartsArgs) - } - - val partitions: ArrayBuffer[(Map[String, Option[String]], Option[String])] = - new ArrayBuffer() - var currentPart: Map[String, Option[String]] = null - parts.map { - case t @ Token("TOK_PARTSPEC", partArgs) => - if (currentPart != null) { - partitions += ((currentPart, None)) - } - currentPart = parsePartitionSpec(t).get - case Token("TOK_PARTITIONLOCATION", loc :: Nil) => - val location = unquoteString(loc.text) - if (currentPart != null) { - partitions += ((currentPart, Some(location))) - currentPart = null - } else { - // We should not reach here - throw new AnalysisException("Partition location must follow a partition spec.") - } - } - - if (currentPart != null) { - partitions += ((currentPart, None)) - } - AlterTableAddPartition(tableIdent, partitions, allowExisting)(node.source) - - case Token("TOK_ALTERTABLE_RENAMEPART", partArg :: Nil) :: _ => - val Some(newPartition) = parsePartitionSpec(partArg) - AlterTableRenamePartition(tableIdent, partition.get, newPartition)(node.source) - - case Token("TOK_ALTERTABLE_EXCHANGEPARTITION", - (p @ Token("TOK_PARTSPEC", _)) :: (t @ Token("TOK_TABNAME", _)) :: Nil) :: _ => - val Some(partition) = parsePartitionSpec(p) - val fromTableIdent = extractTableIdent(t) - AlterTableExchangePartition(tableIdent, fromTableIdent, partition)(node.source) - - case Token("TOK_ALTERTABLE_DROPPARTS", args) :: _ => - val parts = args.collect { - case Token("TOK_PARTSPEC", partitions) => - partitions.map { - case Token("TOK_PARTVAL", ident :: op :: constant :: Nil) => - (cleanAndUnquoteString(ident.text), - op.text, cleanAndUnquoteString(constant.text)) + case Token("TOK_ALTERTABLE_ADDPARTS", addPartsArgs) :: _ => + val (allowExisting, parts) = addPartsArgs match { + case Token("TOK_IFNOTEXISTS", Nil) :: others => (true, others) + case _ => (false, addPartsArgs) + } + val partitions: ArrayBuffer[(Map[String, Option[String]], Option[String])] = + new ArrayBuffer() + var currentPart: Map[String, Option[String]] = null + parts.map { + case t @ Token("TOK_PARTSPEC", partArgs) => + if (currentPart != null) { + partitions += ((currentPart, None)) + } + currentPart = parsePartitionSpec(t).get + case Token("TOK_PARTITIONLOCATION", loc :: Nil) => + val location = unquoteString(loc.text) + if (currentPart != null) { + partitions += ((currentPart, Some(location))) + currentPart = null + } else { + // We should not reach here + throw new AnalysisException("Partition location must follow a partition spec.") + } + } + if (currentPart != null) { + partitions += ((currentPart, None)) + } + AlterTableAddPartition(tableIdent, partitions, allowExisting)(node.source) + + case Token("TOK_ALTERTABLE_RENAMEPART", partArg :: Nil) :: _ => + val Some(newPartition) = parsePartitionSpec(partArg) + AlterTableRenamePartition(tableIdent, partition.get, newPartition)(node.source) + + case Token("TOK_ALTERTABLE_EXCHANGEPARTITION", + (p @ Token("TOK_PARTSPEC", _)) :: (t @ Token("TOK_TABNAME", _)) :: Nil) :: _ => + val Some(partition) = parsePartitionSpec(p) + val fromTableIdent = extractTableIdent(t) + AlterTableExchangePartition(tableIdent, fromTableIdent, partition)(node.source) + + case Token("TOK_ALTERTABLE_DROPPARTS", args) :: _ => + val parts = args.collect { + case Token("TOK_PARTSPEC", partitions) => + partitions.map { + case Token("TOK_PARTVAL", ident :: op :: constant :: Nil) => + (cleanAndUnquoteString(ident.text), + op.text, cleanAndUnquoteString(constant.text)) + } + } + val allowExisting = getClauseOption("TOK_IFEXISTS", args).isDefined + val purge = getClauseOption("PURGE", args) + val replication = getClauseOption("TOK_REPLICATION", args).map { + case Token("TOK_REPLICATION", replId :: metadata) => + (cleanAndUnquoteString(replId.text), metadata.nonEmpty) + } + AlterTableDropPartition( + tableIdent, + parts, + allowExisting, + purge.isDefined, + replication)(node.source) + + case Token("TOK_ALTERTABLE_ARCHIVE", partArg :: Nil) :: _ => + val Some(partition) = parsePartitionSpec(partArg) + AlterTableArchivePartition(tableIdent, partition)(node.source) + + case Token("TOK_ALTERTABLE_UNARCHIVE", partArg :: Nil) :: _ => + val Some(partition) = parsePartitionSpec(partArg) + AlterTableUnarchivePartition(tableIdent, partition)(node.source) + + case Token("TOK_ALTERTABLE_FILEFORMAT", args) :: _ => + val Seq(fileFormat, genericFormat) = + getClauses(Seq("TOK_TABLEFILEFORMAT", "TOK_FILEFORMAT_GENERIC"), + args) + val fFormat = fileFormat.map(_.children.map(n => cleanAndUnquoteString(n.text))) + val gFormat = genericFormat.map(f => cleanAndUnquoteString(f.children(0).text)) + AlterTableSetFileFormat(tableIdent, partition, fFormat, gFormat)(node.source) + + case Token("TOK_ALTERTABLE_LOCATION", Token(loc, Nil) :: Nil) :: _ => + AlterTableSetLocation(tableIdent, partition, cleanAndUnquoteString(loc))(node.source) + + case Token("TOK_ALTERTABLE_TOUCH", args) :: _ => + val part = getClauseOption("TOK_PARTSPEC", args).flatMap(parsePartitionSpec) + AlterTableTouch(tableIdent, part)(node.source) + + case Token("TOK_ALTERTABLE_COMPACT", Token(compactType, Nil) :: Nil) :: _ => + AlterTableCompact(tableIdent, partition, cleanAndUnquoteString(compactType))(node.source) + + case Token("TOK_ALTERTABLE_MERGEFILES", _) :: _ => + AlterTableMerge(tableIdent, partition)(node.source) + + case Token("TOK_ALTERTABLE_RENAMECOL", args) :: _ => + val oldName = args(0).text + val newName = args(1).text + val dataType = nodeToDataType(args(2)) + val afterPos = + getClauseOption("TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION", args) + val afterPosCol = afterPos.map { ap => + ap.children match { + case Token(col, Nil) :: Nil => col + case _ => null } - } - - val allowExisting = getClauseOption("TOK_IFEXISTS", args).isDefined - - val purge = getClauseOption("PURGE", args) - - val replication = getClauseOption("TOK_REPLICATION", args).map { - case Token("TOK_REPLICATION", replId :: metadata) => - (cleanAndUnquoteString(replId.text), metadata.nonEmpty) - } - - AlterTableDropPartition( - tableIdent, - parts, - allowExisting, - purge.isDefined, - replication)(node.source) - - case Token("TOK_ALTERTABLE_ARCHIVE", partArg :: Nil) :: _ => - val Some(partition) = parsePartitionSpec(partArg) - AlterTableArchivePartition(tableIdent, partition)(node.source) - - case Token("TOK_ALTERTABLE_UNARCHIVE", partArg :: Nil) :: _ => - val Some(partition) = parsePartitionSpec(partArg) - AlterTableUnarchivePartition(tableIdent, partition)(node.source) - - case Token("TOK_ALTERTABLE_FILEFORMAT", args) :: _ => - val Seq(fileFormat, genericFormat) = - getClauses(Seq("TOK_TABLEFILEFORMAT", "TOK_FILEFORMAT_GENERIC"), - args) - val fFormat = fileFormat.map(_.children.map(n => cleanAndUnquoteString(n.text))) - val gFormat = genericFormat.map(f => cleanAndUnquoteString(f.children(0).text)) - AlterTableSetFileFormat(tableIdent, partition, fFormat, gFormat)(node.source) - - case Token("TOK_ALTERTABLE_LOCATION", Token(loc, Nil) :: Nil) :: _ => - AlterTableSetLocation(tableIdent, partition, cleanAndUnquoteString(loc))(node.source) - - case Token("TOK_ALTERTABLE_TOUCH", args) :: _ => - val part = getClauseOption("TOK_PARTSPEC", args).flatMap(parsePartitionSpec) - AlterTableTouch(tableIdent, part)(node.source) - - case Token("TOK_ALTERTABLE_COMPACT", Token(compactType, Nil) :: Nil) :: _ => - AlterTableCompact(tableIdent, partition, cleanAndUnquoteString(compactType))(node.source) - - case Token("TOK_ALTERTABLE_MERGEFILES", _) :: _ => - AlterTableMerge(tableIdent, partition)(node.source) - - case Token("TOK_ALTERTABLE_RENAMECOL", args) :: _ => - val oldName = args(0).text - val newName = args(1).text - val dataType = nodeToDataType(args(2)) - val afterPos = - getClauseOption("TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION", args) - val afterPosCol = afterPos.map { ap => - ap.children match { - case Token(col, Nil) :: Nil => col - case _ => null } - } - - val restrict = getClauseOption("TOK_RESTRICT", args) - val cascade = getClauseOption("TOK_CASCADE", args) - - val comment = if (args.size > 3) { - args(3) match { - case Token(commentStr, Nil) - if commentStr != "TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION" && - commentStr != "TOK_RESTRICT" && commentStr != "TOK_CASCADE" => + val restrict = getClauseOption("TOK_RESTRICT", args) + val cascade = getClauseOption("TOK_CASCADE", args) + val comment = if (args.size > 3) { + args(3) match { + case Token(commentStr, Nil) + if commentStr != "TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION" && + commentStr != "TOK_RESTRICT" && commentStr != "TOK_CASCADE" => Some(cleanAndUnquoteString(commentStr)) - case _ => - None + case _ => + None + } + } else { + None } - } else { - None - } - - AlterTableChangeCol( - tableIdent, - partition, - oldName, - newName, - dataType, - comment, - afterPos.isDefined, - afterPosCol, - restrict.isDefined, - cascade.isDefined)(node.source) - - case Token("TOK_ALTERTABLE_ADDCOLS", args) :: _ => - val tableCols = getClause("TOK_TABCOLLIST", args) - val columns = tableCols match { - case Token("TOK_TABCOLLIST", fields) => StructType(fields.map(nodeToStructField)) - } - - val restrict = getClauseOption("TOK_RESTRICT", args) - val cascade = getClauseOption("TOK_CASCADE", args) - - AlterTableAddCol( - tableIdent, - partition, - columns, - restrict.isDefined, - cascade.isDefined)(node.source) - - case Token("TOK_ALTERTABLE_REPLACECOLS", args) :: _ => - val tableCols = getClause("TOK_TABCOLLIST", args) - val columns = tableCols match { - case Token("TOK_TABCOLLIST", fields) => StructType(fields.map(nodeToStructField)) - } - - val restrict = getClauseOption("TOK_RESTRICT", args) - val cascade = getClauseOption("TOK_CASCADE", args) - - AlterTableReplaceCol( - tableIdent, - partition, - columns, - restrict.isDefined, - cascade.isDefined)(node.source) + AlterTableChangeCol( + tableIdent, + partition, + oldName, + newName, + dataType, + comment, + afterPos.isDefined, + afterPosCol, + restrict.isDefined, + cascade.isDefined)(node.source) + + case Token("TOK_ALTERTABLE_ADDCOLS", args) :: _ => + val tableCols = getClause("TOK_TABCOLLIST", args) + val columns = tableCols match { + case Token("TOK_TABCOLLIST", fields) => StructType(fields.map(nodeToStructField)) + } + val restrict = getClauseOption("TOK_RESTRICT", args) + val cascade = getClauseOption("TOK_CASCADE", args) + AlterTableAddCol( + tableIdent, + partition, + columns, + restrict.isDefined, + cascade.isDefined)(node.source) + + case Token("TOK_ALTERTABLE_REPLACECOLS", args) :: _ => + val tableCols = getClause("TOK_TABCOLLIST", args) + val columns = tableCols match { + case Token("TOK_TABCOLLIST", fields) => StructType(fields.map(nodeToStructField)) + } + val restrict = getClauseOption("TOK_RESTRICT", args) + val cascade = getClauseOption("TOK_CASCADE", args) + AlterTableReplaceCol( + tableIdent, + partition, + columns, + restrict.isDefined, + cascade.isDefined)(node.source) + + case _ => + throw new AnalysisException( + s"Unexpected children nodes in alter table command: '${node.text}'") + } } -} +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index ec31c7f058cfb..4ccb7c340ed5c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -30,54 +30,61 @@ import org.apache.spark.sql.types._ * query text. */ abstract class NativeDDLCommands(val sql: String) extends RunnableCommand { + override def run(sqlContext: SQLContext): Seq[Row] = { sqlContext.runNativeSql(sql) } - override val output: Seq[Attribute] = + override val output: Seq[Attribute] = { Seq(AttributeReference("result", StringType, nullable = false)()) + } + } -case class CreateDataBase( +case class CreateDatabase( databaseName: String, allowExisting: Boolean, path: Option[String], comment: Option[String], - props: Map[String, String])(sql: String) extends NativeDDLCommands(sql) with Logging + props: Map[String, String])(sql: String) + extends NativeDDLCommands(sql) with Logging case class CreateFunction( functionName: String, - asName: String, + alias: String, resourcesMap: Map[String, String], - isTemp: Boolean)(sql: String) extends NativeDDLCommands(sql) with Logging + isTemp: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableRename( tableName: TableIdentifier, - renameTableName: TableIdentifier)(sql: String) extends NativeDDLCommands(sql) - with Logging + renameTableName: TableIdentifier)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSetProperties( tableName: TableIdentifier, - setProperties: Map[String, Option[String]])(sql: String) extends NativeDDLCommands(sql) - with Logging + setProperties: Map[String, Option[String]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableDropProperties( tableName: TableIdentifier, dropProperties: Map[String, Option[String]], - allowExisting: Boolean)(sql: String) extends NativeDDLCommands(sql) with Logging + allowExisting: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSerDeProperties( tableName: TableIdentifier, serdeClassName: Option[String], serdeProperties: Option[Map[String, Option[String]]], - partition: Option[Map[String, Option[String]]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Option[Map[String, Option[String]]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableStoreProperties( tableName: TableIdentifier, buckets: Option[BucketSpec], noClustered: Boolean, - noSorted: Boolean)(sql: String) extends NativeDDLCommands(sql) with Logging + noSorted: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSkewed( tableName: TableIdentifier, @@ -85,76 +92,79 @@ case class AlterTableSkewed( skewedValues: Seq[Seq[String]], storedAsDirs: Boolean, notSkewed: Boolean, - // TODO: wtf? - notStoredAsDirs: Boolean)(sql: String) extends NativeDDLCommands(sql) with Logging + // TODO: what?? + notStoredAsDirs: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSkewedLocation( tableName: TableIdentifier, - skewedMap: Map[Seq[String], String])(sql: String) extends NativeDDLCommands(sql) with Logging + skewedMap: Map[Seq[String], String])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableAddPartition( tableName: TableIdentifier, partitionsAndLocs: Seq[(Map[String, Option[String]], Option[String])], - allowExisting: Boolean)(sql: String) extends NativeDDLCommands(sql) with Logging + allowExisting: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableRenamePartition( tableName: TableIdentifier, oldPartition: Map[String, Option[String]], - newPartition: Map[String, Option[String]])(sql: String) extends NativeDDLCommands(sql) - with Logging + newPartition: Map[String, Option[String]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableExchangePartition( tableName: TableIdentifier, fromTableName: TableIdentifier, - partition: Map[String, Option[String]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Map[String, Option[String]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableDropPartition( tableName: TableIdentifier, partitions: Seq[Seq[(String, String, String)]], allowExisting: Boolean, purge: Boolean, - replication: Option[(String, Boolean)])(sql: String) extends NativeDDLCommands(sql) - with Logging + replication: Option[(String, Boolean)])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableArchivePartition( tableName: TableIdentifier, - partition: Map[String, Option[String]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Map[String, Option[String]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableUnarchivePartition( tableName: TableIdentifier, - partition: Map[String, Option[String]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Map[String, Option[String]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSetFileFormat( tableName: TableIdentifier, partition: Option[Map[String, Option[String]]], fileFormat: Option[Seq[String]], - genericFormat: Option[String])(sql: String) extends NativeDDLCommands(sql) - with Logging + genericFormat: Option[String])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableSetLocation( tableName: TableIdentifier, partition: Option[Map[String, Option[String]]], - location: String)(sql: String) extends NativeDDLCommands(sql) - with Logging + location: String)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableTouch( tableName: TableIdentifier, - partition: Option[Map[String, Option[String]]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Option[Map[String, Option[String]]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableCompact( tableName: TableIdentifier, partition: Option[Map[String, Option[String]]], - compactType: String)(sql: String) extends NativeDDLCommands(sql) - with Logging + compactType: String)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableMerge( tableName: TableIdentifier, - partition: Option[Map[String, Option[String]]])(sql: String) extends NativeDDLCommands(sql) - with Logging + partition: Option[Map[String, Option[String]]])(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableChangeCol( tableName: TableIdentifier, @@ -166,21 +176,21 @@ case class AlterTableChangeCol( afterPos: Boolean, afterPosCol: Option[String], restrict: Boolean, - cascade: Boolean)(sql: String) extends NativeDDLCommands(sql) - with Logging + cascade: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableAddCol( tableName: TableIdentifier, partition: Option[Map[String, Option[String]]], columns: StructType, restrict: Boolean, - cascade: Boolean)(sql: String) extends NativeDDLCommands(sql) - with Logging + cascade: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging case class AlterTableReplaceCol( tableName: TableIdentifier, partition: Option[Map[String, Option[String]]], columns: StructType, restrict: Boolean, - cascade: Boolean)(sql: String) extends NativeDDLCommands(sql) - with Logging + cascade: Boolean)(sql: String) + extends NativeDDLCommands(sql) with Logging diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala similarity index 92% rename from sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala rename to sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index 049060d88f191..27ff583ca5de3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -24,8 +24,8 @@ import org.apache.spark.sql.execution.SparkQl import org.apache.spark.sql.execution.datasources.BucketSpec import org.apache.spark.sql.types._ -class DDLSuite extends PlanTest { - val parser = new SparkQl() +class DDLCommandSuite extends PlanTest { + private val parser = new SparkQl test("create database") { val sql = @@ -34,16 +34,13 @@ class DDLSuite extends PlanTest { |COMMENT 'database_comment' LOCATION '/home/user/db' |WITH DBPROPERTIES ('a'='a', 'b'='b', 'c'='c') """.stripMargin - val parsed = parser.parsePlan(sql) - - val expected = CreateDataBase( + val expected = CreateDatabase( "database_name", - true, + allowExisting = true, Some("/home/user/db"), Some("database_comment"), Map("a" -> "a", "b" -> "b", "c" -> "c"))(sql) - comparePlans(parsed, expected) } @@ -54,15 +51,12 @@ class DDLSuite extends PlanTest { |'com.matthewrathbone.example.SimpleUDFExample' USING JAR '/path/to/jar', |FILE 'path/to/file' """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = CreateFunction( "helloworld", "com.matthewrathbone.example.SimpleUDFExample", Map("jar" -> "/path/to/jar", "file" -> "path/to/file"), - true)(sql) - + isTemp = true)(sql) comparePlans(parsed, expected) } @@ -80,25 +74,20 @@ class DDLSuite extends PlanTest { "'comment' = 'new_comment')" val sql2 = "ALTER TABLE table_name UNSET TBLPROPERTIES ('comment', 'test')" val sql3 = "ALTER TABLE table_name UNSET TBLPROPERTIES IF EXISTS ('comment', 'test')" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) - val expected1 = AlterTableSetProperties( TableIdentifier("table_name", None), Map("test" -> Some("test"), "comment" -> Some("new_comment")))(sql1) - val expected2 = AlterTableDropProperties( TableIdentifier("table_name", None), Map("comment" -> None, "test" -> None), - false)(sql2) - + allowExisting = false)(sql2) val expected3 = AlterTableDropProperties( TableIdentifier("table_name", None), Map("comment" -> None, "test" -> None), - true)(sql3) - + allowExisting = true)(sql3) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -111,62 +100,52 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name SET SERDE 'org.apache.class' |WITH SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',') """.stripMargin - val sql3 = """ |ALTER TABLE table_name SET SERDEPROPERTIES ('columns'='foo,bar', |'field.delim' = ',') """.stripMargin - val sql4 = """ |ALTER TABLE table_name PARTITION (test, dt='2008-08-08', |country='us') SET SERDE 'org.apache.class' WITH SERDEPROPERTIES ('columns'='foo,bar', |'field.delim' = ',') """.stripMargin - val sql5 = """ |ALTER TABLE table_name PARTITION (test, dt='2008-08-08', |country='us') SET SERDEPROPERTIES ('columns'='foo,bar', 'field.delim' = ',') """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) val parsed4 = parser.parsePlan(sql4) val parsed5 = parser.parsePlan(sql5) - val expected1 = AlterTableSerDeProperties( TableIdentifier("table_name", None), Some("org.apache.class"), None, None)(sql1) - val expected2 = AlterTableSerDeProperties( TableIdentifier("table_name", None), Some("org.apache.class"), Some(Map("columns" -> Some("foo,bar"), "field.delim" -> Some(","))), None)(sql2) - val expected3 = AlterTableSerDeProperties( TableIdentifier("table_name", None), None, Some(Map("columns" -> Some("foo,bar"), "field.delim" -> Some(","))), None)(sql3) - val expected4 = AlterTableSerDeProperties( TableIdentifier("table_name", None), Some("org.apache.class"), Some(Map("columns" -> Some("foo,bar"), "field.delim" -> Some(","))), Some(Map("test" -> None, "dt" -> Some("2008-08-08"), "country" -> Some("us"))))(sql4) - val expected5 = AlterTableSerDeProperties( TableIdentifier("table_name", None), None, Some(Map("columns" -> Some("foo,bar"), "field.delim" -> Some(","))), Some(Map("test" -> None, "dt" -> Some("2008-08-08"), "country" -> Some("us"))))(sql5) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -176,51 +155,42 @@ class DDLSuite extends PlanTest { test("alter table: storage properties") { val sql1 = "ALTER TABLE table_name CLUSTERED BY (dt, country) INTO 10 BUCKETS" - val sql2 = "ALTER TABLE table_name CLUSTERED BY (dt, country) SORTED BY " + "(dt, country DESC) INTO 10 BUCKETS" - val sql3 = "ALTER TABLE table_name INTO 20 BUCKETS" val sql4 = "ALTER TABLE table_name NOT CLUSTERED" val sql5 = "ALTER TABLE table_name NOT SORTED" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) val parsed4 = parser.parsePlan(sql4) val parsed5 = parser.parsePlan(sql5) - val expected1 = AlterTableStoreProperties( TableIdentifier("table_name", None), Some(BucketSpec(10, List("dt", "country"), List(), List())), - false, - false)(sql1) - + noClustered = false, + noSorted = false)(sql1) val expected2 = AlterTableStoreProperties( TableIdentifier("table_name", None), Some(BucketSpec(10, List("dt", "country"), List("dt", "country"), List(Ascending, Descending))), - false, - false)(sql2) - + noClustered = false, + noSorted = false)(sql2) val expected3 = AlterTableStoreProperties( TableIdentifier("table_name", None), Some(BucketSpec(20, List(), List(), List())), - false, - false)(sql3) - + noClustered = false, + noSorted = false)(sql3) val expected4 = AlterTableStoreProperties( TableIdentifier("table_name", None), None, - true, - false)(sql4) - + noClustered = true, + noSorted = false)(sql4) val expected5 = AlterTableStoreProperties( TableIdentifier("table_name", None), None, - false, - true)(sql5) - + noClustered = false, + noSorted = true)(sql5) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -234,47 +204,40 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name SKEWED BY (dt, country) ON |(('2008-08-08', 'us'), ('2009-09-09', 'uk')) STORED AS DIRECTORIES """.stripMargin - val sql2 = """ |ALTER TABLE table_name SKEWED BY (dt, country) ON |('2008-08-08', 'us') STORED AS DIRECTORIES """.stripMargin - val sql3 = """ |ALTER TABLE table_name SKEWED BY (dt, country) ON |(('2008-08-08', 'us'), ('2009-09-09', 'uk')) """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) - val expected1 = AlterTableSkewed( TableIdentifier("table_name", None), Seq("dt", "country"), Seq(List("2008-08-08", "us"), List("2009-09-09", "uk")), - true, - false, - false)(sql1) - + storedAsDirs = true, + notSkewed = false, + notStoredAsDirs = false)(sql1) val expected2 = AlterTableSkewed( TableIdentifier("table_name", None), Seq("dt", "country"), Seq(List("2008-08-08", "us")), - true, - false, - false)(sql2) - + storedAsDirs = true, + notSkewed = false, + notStoredAsDirs = false)(sql2) val expected3 = AlterTableSkewed( TableIdentifier("table_name", None), Seq("dt", "country"), Seq(List("2008-08-08", "us"), List("2009-09-09", "uk")), - false, - false, - false)(sql3) - + storedAsDirs = false, + notSkewed = false, + notStoredAsDirs = false)(sql3) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -286,24 +249,19 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name SET SKEWED LOCATION |('123'='location1', 'test'='location2') """.stripMargin - val sql2 = """ |ALTER TABLE table_name SET SKEWED LOCATION |(('2008-08-08', 'us')='location1', 'test'='location2') """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableSkewedLocation( TableIdentifier("table_name", None), Map(List("123") -> "location1", List("test") -> "location2"))(sql1) - val expected2 = AlterTableSkewedLocation( TableIdentifier("table_name", None), Map(List("2008-08-08", "us") -> "location1", List("test") -> "location2"))(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } @@ -315,16 +273,13 @@ class DDLSuite extends PlanTest { |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION |(dt='2009-09-09', country='uk') """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableAddPartition( TableIdentifier("table_name", None), Seq( (Map("dt" -> Some("2008-08-08"), "country" -> Some("us")), Some("location1")), (Map("dt" -> Some("2009-09-09"), "country" -> Some("uk")), None)), - true)(sql) - + allowExisting = true)(sql) comparePlans(parsed, expected) } @@ -334,14 +289,11 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') |RENAME TO PARTITION (dt='2008-09-09', country='uk') """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableRenamePartition( TableIdentifier("table_name", None), Map("dt" -> Some("2008-08-08"), "country" -> Some("us")), Map("dt" -> Some("2008-09-09"), "country" -> Some("uk")))(sql) - comparePlans(parsed, expected) } @@ -351,14 +303,11 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name_1 EXCHANGE PARTITION |(dt='2008-08-08', country='us') WITH TABLE table_name_2 """.stripMargin - val parsed = parser.parsePlan(sql) - val expected = AlterTableExchangePartition( TableIdentifier("table_name_1", None), TableIdentifier("table_name_2", None), Map("dt" -> Some("2008-08-08"), "country" -> Some("us")))(sql) - comparePlans(parsed, expected) } @@ -368,35 +317,30 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name DROP IF EXISTS PARTITION |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') """.stripMargin - val sql2 = """ |ALTER TABLE table_name DROP IF EXISTS PARTITION |(dt='2008-08-08', country='us'), PARTITION (dt='2009-09-09', country='uk') |PURGE FOR METADATA REPLICATION ('test') """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableDropPartition( TableIdentifier("table_name", None), Seq( List(("dt", "=", "2008-08-08"), ("country", "=", "us")), List(("dt", "=", "2009-09-09"), ("country", "=", "uk"))), - true, - false, + allowExisting = true, + purge = false, None)(sql1) - val expected2 = AlterTableDropPartition( TableIdentifier("table_name", None), Seq( List(("dt", "=", "2008-08-08"), ("country", "=", "us")), List(("dt", "=", "2009-09-09"), ("country", "=", "uk"))), - true, - true, + allowExisting = true, + purge = true, Some(("test", true)))(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } @@ -404,22 +348,18 @@ class DDLSuite extends PlanTest { test("alter table: archive partition") { val sql = "ALTER TABLE table_name ARCHIVE PARTITION (dt='2008-08-08', country='us')" val parsed = parser.parsePlan(sql) - val expected = AlterTableArchivePartition( TableIdentifier("table_name", None), Map("dt" -> Some("2008-08-08"), "country" -> Some("us")))(sql) - comparePlans(parsed, expected) } test("alter table: unarchive partition") { val sql = "ALTER TABLE table_name UNARCHIVE PARTITION (dt='2008-08-08', country='us')" val parsed = parser.parsePlan(sql) - val expected = AlterTableUnarchivePartition( TableIdentifier("table_name", None), Map("dt" -> Some("2008-08-08"), "country" -> Some("us")))(sql) - comparePlans(parsed, expected) } @@ -429,35 +369,28 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name SET FILEFORMAT INPUTFORMAT 'test' |OUTPUTFORMAT 'test' SERDE 'test' INPUTDRIVER 'test' OUTPUTDRIVER 'test' """.stripMargin - val sql2 = "ALTER TABLE table_name SET FILEFORMAT INPUTFORMAT 'test' " + "OUTPUTFORMAT 'test' SERDE 'test'" - val sql3 = "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') " + "SET FILEFORMAT PARQUET" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) - val expected1 = AlterTableSetFileFormat( TableIdentifier("table_name", None), None, Some(List("test", "test", "test", "test", "test")), None)(sql1) - val expected2 = AlterTableSetFileFormat( TableIdentifier("table_name", None), None, Some(List("test", "test", "test")), None)(sql2) - val expected3 = AlterTableSetFileFormat( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))), None, Some("PARQUET"))(sql3) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -467,20 +400,16 @@ class DDLSuite extends PlanTest { val sql1 = "ALTER TABLE table_name SET LOCATION 'new location'" val sql2 = "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') " + "SET LOCATION 'new location'" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableSetLocation( TableIdentifier("table_name", None), None, "new location")(sql1) - val expected2 = AlterTableSetLocation( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))), "new location")(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } @@ -488,18 +417,14 @@ class DDLSuite extends PlanTest { test("alter table: touch") { val sql1 = "ALTER TABLE table_name TOUCH" val sql2 = "ALTER TABLE table_name TOUCH PARTITION (dt='2008-08-08', country='us')" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableTouch( TableIdentifier("table_name", None), None)(sql1) - val expected2 = AlterTableTouch( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))))(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } @@ -511,20 +436,16 @@ class DDLSuite extends PlanTest { |ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') |COMPACT 'MAJOR' """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableCompact( TableIdentifier("table_name", None), None, "compaction_type")(sql1) - val expected2 = AlterTableCompact( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))), "MAJOR")(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } @@ -532,41 +453,33 @@ class DDLSuite extends PlanTest { test("alter table: concatenate") { val sql1 = "ALTER TABLE table_name CONCATENATE" val sql2 = "ALTER TABLE table_name PARTITION (dt='2008-08-08', country='us') CONCATENATE" - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val expected1 = AlterTableMerge( TableIdentifier("table_name", None), None)(sql1) - val expected2 = AlterTableMerge( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))))(sql2) - comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } test("alter table: change column name/type/position/comment") { val sql1 = "ALTER TABLE table_name CHANGE col_old_name col_new_name INT" - val sql2 = """ |ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT |COMMENT 'col_comment' FIRST CASCADE """.stripMargin - val sql3 = """ |ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT |COMMENT 'col_comment' AFTER column_name RESTRICT """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) val parsed3 = parser.parsePlan(sql3) - val expected1 = AlterTableChangeCol( TableIdentifier("table_name", None), None, @@ -574,11 +487,10 @@ class DDLSuite extends PlanTest { "col_new_name", IntegerType, None, - false, + afterPos = false, None, - false, - false)(sql1) - + restrict = false, + cascade = false)(sql1) val expected2 = AlterTableChangeCol( TableIdentifier("table_name", None), None, @@ -586,11 +498,10 @@ class DDLSuite extends PlanTest { "col_new_name", IntegerType, Some("col_comment"), - false, + afterPos = false, None, - false, - true)(sql2) - + restrict = false, + cascade = true)(sql2) val expected3 = AlterTableChangeCol( TableIdentifier("table_name", None), None, @@ -598,11 +509,10 @@ class DDLSuite extends PlanTest { "col_new_name", IntegerType, Some("col_comment"), - true, + afterPos = true, Some("column_name"), - true, - false)(sql3) - + restrict = true, + cascade = false)(sql3) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) comparePlans(parsed3, expected3) @@ -615,38 +525,33 @@ class DDLSuite extends PlanTest { |ADD COLUMNS (new_col1 INT COMMENT 'test_comment', new_col2 LONG |COMMENT 'test_comment2') CASCADE """.stripMargin - val sql2 = """ |ALTER TABLE table_name REPLACE COLUMNS (new_col1 INT |COMMENT 'test_comment', new_col2 LONG COMMENT 'test_comment2') RESTRICT """.stripMargin - val parsed1 = parser.parsePlan(sql1) val parsed2 = parser.parsePlan(sql2) - val meta1 = new MetadataBuilder().putString("comment", "test_comment").build() val meta2 = new MetadataBuilder().putString("comment", "test_comment2").build() - val expected1 = AlterTableAddCol( TableIdentifier("table_name", None), Some(Map("dt" -> Some("2008-08-08"), "country" -> Some("us"))), StructType(Seq( - StructField("new_col1", IntegerType, true, meta1), - StructField("new_col2", LongType, true, meta2))), - false, - true)(sql1) - + StructField("new_col1", IntegerType, nullable = true, meta1), + StructField("new_col2", LongType, nullable = true, meta2))), + restrict = false, + cascade = true)(sql1) val expected2 = AlterTableReplaceCol( TableIdentifier("table_name", None), None, StructType(Seq( - StructField("new_col1", IntegerType, true, meta1), - StructField("new_col2", LongType, true, meta2))), - true, - false)(sql2) - + StructField("new_col1", IntegerType, nullable = true, meta1), + StructField("new_col2", LongType, nullable = true, meta2))), + restrict = true, + cascade = false)(sql2) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) } + }