Skip to content

Commit

Permalink
DDL and write support API followup.
Browse files Browse the repository at this point in the history
  • Loading branch information
yhuai committed Feb 6, 2015
1 parent 2a6213a commit af9e9b3
Show file tree
Hide file tree
Showing 17 changed files with 561 additions and 157 deletions.
34 changes: 34 additions & 0 deletions sql/core/src/main/java/org/apache/spark/sql/sources/SaveModes.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.sql.sources;

public class SaveModes {

/**
* Gets the Append object.
*/
public static final SaveMode Append = Append$.MODULE$;
/**
* Gets the Overwrite object.
*/
public static final SaveMode Overwrite = Overwrite$.MODULE$;

/**
* Gets the ErrorIfExists object.
*/
public static final SaveMode ErrorIfExists = ErrorIfExists$.MODULE$;
}
78 changes: 51 additions & 27 deletions sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.spark.sql

import org.apache.spark.sql.sources.SaveMode

import scala.reflect.ClassTag

import org.apache.spark.annotation.{DeveloperApi, Experimental}
Expand Down Expand Up @@ -532,39 +534,57 @@ trait DataFrame extends RDDApi[Row] {

/**
* :: Experimental ::
* Creates a table from the the contents of this DataFrame with a set of options.
* It will use the default data source
* configured in spark.sql.source.default. This will fail if the table already exists.
* Creates a table from the the contents of this DataFrame.
* It will use the default data source configured in spark.sql.sources.default.
* This will fail if the table already exists.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
* an RDD out to a parquet file, and then register that file as a table. This "table" can then
* be the target of an `insertInto`.
*/
@Experimental
def saveAsTable(tableName: String): Unit

/**
* :: Experimental ::
* Creates a table from the the contents of this DataFrame.
* It will use the default data source configured in spark.sql.sources.default.
* If appendIfExists is true and the table already exists,
* it will append contents of this DataFrame to the table.
* If appendIfExists is false and the table already exists, this will fail.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
* an RDD out to a parquet file, and then register that file as a table. This "table" can then
* be the target of an `insertInto`.
*/
@Experimental
def saveAsTable(tableName: String, options: (String, String)*): Unit
def saveAsTable(tableName: String, appendIfExists: Boolean): Unit

/**
* :: Experimental ::
* Creates a table from the the contents of this DataFrame based on a set of options.
* It will use the default data source configured in `spark.sql.source.default`.
* Creates a table at the given path from the the contents of this DataFrame
* based on a given data source and a set of options.
* This will fail if the table already exists.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
* an RDD out to a parquet file, and then register that file as a table. This "table" can then
* be the target of an insertInto`.
* be the target of an `insertInto`.
*/
@Experimental
def saveAsTable(
tableName: String,
options: java.util.Map[String, String]): Unit
dataSourceName: String): Unit

/**
* :: Experimental ::
* Creates a table at the given path from the the contents of this DataFrame
* based on a given data source and a set of options.
* This will fail if the table already exists.
* If appendIfExists is true and the table already exists,
* it will append contents of this DataFrame to the table.
* If appendIfExists is false and the table already exists, this will fail.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
Expand All @@ -575,13 +595,16 @@ trait DataFrame extends RDDApi[Row] {
def saveAsTable(
tableName: String,
dataSourceName: String,
options: (String, String)*): Unit
appendIfExists: Boolean): Unit

/**
* :: Experimental ::
* Creates a table at the given path from the the contents of this DataFrame
* based on a given data source and a set of options.
* This will fail if the table already exists.
* If appendIfExists is true and the table already exists,
* it will append contents of this DataFrame to the table.
* If appendIfExists is false and the table already exists,
* This will fail.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
Expand All @@ -592,12 +615,17 @@ trait DataFrame extends RDDApi[Row] {
def saveAsTable(
tableName: String,
dataSourceName: String,
appendIfExists: Boolean,
options: java.util.Map[String, String]): Unit

/**
* :: Experimental ::
* Creates a table from the the contents of this DataFrame based on a given data source
* and a set of options. This will fail if the table already exists.
* and a set of options.
* If appendIfExists is true and the table already exists,
* it will append contents of this DataFrame to the table.
* If appendIfExists is false and the table already exists,
* This will fail.
*
* Note that this currently only works with DataFrames that are created from a HiveContext as
* there is no notion of a persisted catalog in a standard SQL context. Instead you can write
Expand All @@ -608,44 +636,38 @@ trait DataFrame extends RDDApi[Row] {
def saveAsTable(
tableName: String,
dataSourceName: String,
appendIfExists: Boolean,
options: Map[String, String]): Unit

/**
* :: Experimental ::
* Saves the contents of this DataFrame to the given path.
* It will use the default data source configured in spark.sql.source.default.
* It will use the default data source configured in spark.sql.sources.default.
*/
@Experimental
def save(path: String): Unit

/**
* :: Experimental ::
* Saves the contents of this DataFrame to the given path based on the given data source
* and a set of options.
* Saves the contents of this DataFrame to the given path.
* It will use the default data source configured in spark.sql.sources.default.
*/
@Experimental
def save(path: String, dataSourceName: String, options: (String, String)*): Unit
def save(path: String, mode: SaveMode): Unit

/**
* :: Experimental ::
* Saves the contents of this DataFrame to the given path based on the given data source
* and a set of options.
* Saves the contents of this DataFrame to the given path based on the given data source.
*/
@Experimental
def save(
path: String,
dataSourceName: String,
options: java.util.Map[String, String]): Unit
def save(path: String, dataSourceName: String): Unit

/**
* :: Experimental ::
* Saves the contents of this DataFrame based on the given data source and a set of options.
* Saves the contents of this DataFrame to the given path based on the given data source.
*/
@Experimental
def save(
dataSourceName: String,
option: (String, String),
options: (String, String)*): Unit
def save(path: String, dataSourceName: String, mode: SaveMode): Unit

/**
* :: Experimental ::
Expand All @@ -654,6 +676,7 @@ trait DataFrame extends RDDApi[Row] {
@Experimental
def save(
dataSourceName: String,
mode: SaveMode,
options: java.util.Map[String, String]): Unit

/**
Expand All @@ -663,6 +686,7 @@ trait DataFrame extends RDDApi[Row] {
@Experimental
def save(
dataSourceName: String,
mode: SaveMode,
options: Map[String, String]): Unit

/**
Expand Down
Loading

0 comments on commit af9e9b3

Please sign in to comment.