Skip to content

Commit

Permalink
Support for writing CSV with a single function call
Browse files Browse the repository at this point in the history
  • Loading branch information
HyukjinKwon committed Feb 26, 2016
1 parent 318bf41 commit a97a0a8
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 2 deletions.
29 changes: 29 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,12 @@ final class DataFrameWriter private[sql](df: DataFrame) {
* format("parquet").save(path)
* }}}
*
* You can set the following JSON-specific options for writing JSON files:
* <li>`compression` or `codec` (default `null`): compression codec to use when saving to file.
* This should be the fully qualified name of a class implementing
* [[org.apache.hadoop.io.compress.CompressionCodec]] or one of the known case-insensitive
* shorten names(`bzip2`, `gzip`, `lz4`, and `snappy`). </li>
*
* @since 1.4.0
*/
def parquet(path: String): Unit = format("parquet").save(path)
Expand Down Expand Up @@ -492,10 +498,33 @@ final class DataFrameWriter private[sql](df: DataFrame) {
* df.write().text("/path/to/output")
* }}}
*
* You can set the following options for writing text files:
* <li>`compression` or `codec` (default `null`): compression codec to use when saving to file.
* This should be the fully qualified name of a class implementing
* [[org.apache.hadoop.io.compress.CompressionCodec]] or one of the known case-insensitive
* shorten names(`bzip2`, `gzip`, `lz4`, and `snappy`). </li>
*
* @since 1.6.0
*/
def text(path: String): Unit = format("text").save(path)

/**
* Saves the content of the [[DataFrame]] in CSV format at the specified path.
* This is equivalent to:
* {{{
* format("csv").save(path)
* }}}
*
* You can set the following CSV-specific options for writing CSV files:
* <li>`compression` or `codec` (default `null`): compression codec to use when saving to file.
* This should be the fully qualified name of a class implementing
* [[org.apache.hadoop.io.compress.CompressionCodec]] or one of the known case-insensitive
* shorten names(`bzip2`, `gzip`, `lz4`, and `snappy`). </li>
*
* @since 2.0.0
*/
def csv(path: String): Unit = format("csv").save(path)

///////////////////////////////////////////////////////////////////////////////////////
// Builder pattern config options
///////////////////////////////////////////////////////////////////////////////////////
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -268,9 +268,8 @@ class CSVSuite extends QueryTest with SharedSQLContext with SQLTestUtils {
.load(testFile(carsFile))

cars.coalesce(1).write
.format("csv")
.option("header", "true")
.save(csvDir)
.csv(csvDir)

val carsCopy = sqlContext.read
.format("csv")
Expand Down

0 comments on commit a97a0a8

Please sign in to comment.