Skip to content

Commit

Permalink
[SPARKR-244] Fix test failure after integration of subtract() and sub…
Browse files Browse the repository at this point in the history
…tractByKey() for RDD.
  • Loading branch information
Sun Rui authored and Davies Liu committed Apr 14, 2015
1 parent 7e8caa3 commit 07d0dbc
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 14 deletions.
1 change: 1 addition & 0 deletions R/pkg/NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ exportMethods("columns",
"show",
"showDF",
"sortDF",
"except",
"toJSON",
"toRDD",
"unionAll",
Expand Down
21 changes: 11 additions & 10 deletions R/pkg/R/DataFrame.R
Original file line number Diff line number Diff line change
Expand Up @@ -1141,32 +1141,33 @@ setMethod("intersect",
dataFrame(intersected)
})

#' Subtract
#' except
#'
#' Return a new DataFrame containing rows in this DataFrame
#' but not in another DataFrame. This is equivalent to `EXCEPT` in SQL.
#'
#' @param x A Spark DataFrame
#' @param y A Spark DataFrame
#' @return A DataFrame containing the result of the subtract operation.
#' @rdname subtract
#' @return A DataFrame containing the result of the except operation.
#' @rdname except
#' @export
#' @examples
#'\dontrun{
#' sc <- sparkR.init()
#' sqlCtx <- sparkRSQL.init(sc)
#' df1 <- jsonFile(sqlCtx, path)
#' df2 <- jsonFile(sqlCtx, path2)
#' subtractDF <- subtract(df, df2)
#' exceptDF <- except(df, df2)
#' }
setGeneric("except", function(x, y) { standardGeneric("except") })

#' @rdname subtract
#' @rdname except
#' @export
setMethod("subtract",
signature(x = "DataFrame", other = "DataFrame"),
function(x, other) {
subtracted <- callJMethod(x@sdf, "except", other@sdf)
dataFrame(subtracted)
setMethod("except",
signature(x = "DataFrame", y = "DataFrame"),
function(x, y) {
excepted <- callJMethod(x@sdf, "except", y@sdf)
dataFrame(excepted)
})

#' Save the contents of the DataFrame to a data source
Expand Down
8 changes: 4 additions & 4 deletions R/pkg/inst/tests/test_sparkSQL.R
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,7 @@ test_that("isLocal()", {
expect_false(isLocal(df))
})

test_that("unionAll(), subtract(), and intersect() on a DataFrame", {
test_that("unionAll(), except(), and intersect() on a DataFrame", {
df <- jsonFile(sqlCtx, jsonPath)

lines <- c("{\"name\":\"Bob\", \"age\":24}",
Expand All @@ -659,10 +659,10 @@ test_that("unionAll(), subtract(), and intersect() on a DataFrame", {
expect_true(count(unioned) == 6)
expect_true(first(unioned)$name == "Michael")

subtracted <- sortDF(subtract(df, df2), desc(df$age))
excepted <- sortDF(except(df, df2), desc(df$age))
expect_true(inherits(unioned, "DataFrame"))
expect_true(count(subtracted) == 2)
expect_true(first(subtracted)$name == "Justin")
expect_true(count(excepted) == 2)
expect_true(first(excepted)$name == "Justin")

intersected <- sortDF(intersect(df, df2), df$age)
expect_true(inherits(unioned, "DataFrame"))
Expand Down

0 comments on commit 07d0dbc

Please sign in to comment.