Skip to content

Commit

Permalink
[SPARK-37304][SQL] Allow ANSI intervals in v2 `ALTER TABLE .. REPLACE…
Browse files Browse the repository at this point in the history
… COLUMNS`

### What changes were proposed in this pull request?
In the PR, I propose to allow ANSI intervals: year-month and day-time intervals in the `ALTER TABLE .. REPLACE COLUMNS` command for tables in v2 catalogs (v1 catalogs don't support the command). Also added unified test suite to migrate related tests in the future.

### Why are the changes needed?
To improve user experience with Spark SQL. After the changes, users can replace columns with ANSI intervals instead of removing and adding such columns.

### Does this PR introduce _any_ user-facing change?
In some sense, yes. After the changes, the command doesn't output any error message.

### How was this patch tested?
By running new test suite:
```
$ build/sbt "test:testOnly *AlterTableReplaceColumnsSuite"
```

Closes apache#34571 from MaxGekk/add-replace-ansi-interval-col.

Authored-by: Max Gekk <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
  • Loading branch information
MaxGekk committed Nov 12, 2021
1 parent 68a0ab5 commit 71f4ee3
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ case class ReplaceColumns(
table: LogicalPlan,
columnsToAdd: Seq[QualifiedColType]) extends AlterTableCommand {
columnsToAdd.foreach { c =>
TypeUtils.failWithIntervalType(c.dataType)
TypeUtils.failWithIntervalType(c.dataType, forbidAnsiIntervals = false)
}

override lazy val resolved: Boolean = table.resolved && columnsToAdd.forall(_.resolved)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.spark.sql.execution.command

import java.time.{Duration, Period}

import org.apache.spark.sql.{QueryTest, Row}

/**
* This base suite contains unified tests for the `ALTER TABLE .. REPLACE COLUMNS` command that
* check the V2 table catalog. The tests that cannot run for all supported catalogs are
* located in more specific test suites:
*
* - V2 table catalog tests:
* `org.apache.spark.sql.execution.command.v2.AlterTableReplaceColumnsSuite`
*/
trait AlterTableReplaceColumnsSuiteBase extends QueryTest with DDLCommandTestUtils {
override val command = "ALTER TABLE .. REPLACE COLUMNS"

test("SPARK-37304: Replace columns by ANSI intervals") {
withNamespaceAndTable("ns", "tbl") { t =>
sql(s"CREATE TABLE $t (ym INTERVAL MONTH, dt INTERVAL HOUR, data STRING) $defaultUsing")
// TODO(SPARK-37303): Uncomment the command below after REPLACE COLUMNS is fixed
// sql(s"INSERT INTO $t SELECT INTERVAL '1' MONTH, INTERVAL '2' HOUR, 'abc'")
sql(
s"""
|ALTER TABLE $t REPLACE COLUMNS (
| new_ym INTERVAL YEAR,
| new_dt INTERVAL MINUTE,
| new_data INT)""".stripMargin)
sql(s"INSERT INTO $t SELECT INTERVAL '3' YEAR, INTERVAL '4' MINUTE, 5")

checkAnswer(
sql(s"SELECT new_ym, new_dt, new_data FROM $t"),
Seq(
Row(Period.ofYears(3), Duration.ofMinutes(4), 5)))
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.spark.sql.execution.command.v2

import org.apache.spark.sql.execution.command

/**
* The class contains tests for the `ALTER TABLE .. REPLACE COLUMNS` command
* to check V2 table catalogs.
*/
class AlterTableReplaceColumnsSuite
extends command.AlterTableReplaceColumnsSuiteBase
with CommandSuiteBase

0 comments on commit 71f4ee3

Please sign in to comment.