Merge pull request apache#439 from palantir/revert-431-juang/merge-ea…

…sy-upstream Revert "Merge partial upstream"
mccheah · Nov 13, 2018 · a0555b5 · a0555b5
2 parents c54a754 + 8009f21
commit a0555b5
Show file tree

Hide file tree

Showing 618 changed files with 21,265 additions and 18,339 deletions.
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
@@ -28,8 +28,9 @@ importFrom("utils", "download.file", "object.size", "packageVersion", "tail", "u
 
 # S3 methods exported
 export("sparkR.session")
-export("sparkR.session.stop")
+export("sparkR.init")
 export("sparkR.stop")
+export("sparkR.session.stop")
 export("sparkR.conf")
 export("sparkR.version")
 export("sparkR.uiWebUrl")
@@ -41,6 +42,9 @@ export("sparkR.callJStatic")
 
 export("install.spark")
 
+export("sparkRSQL.init",
+       "sparkRHive.init")
+
 # MLlib integration
 exportMethods("glm",
               "spark.glm",
@@ -66,8 +70,7 @@ exportMethods("glm",
               "spark.svmLinear",
               "spark.fpGrowth",
               "spark.freqItemsets",
-              "spark.associationRules",
-              "spark.findFrequentSequentialPatterns")
+              "spark.associationRules")
 
 # Job group lifecycle management methods
 export("setJobGroup",
@@ -147,13 +150,15 @@ exportMethods("arrange",
               "printSchema",
               "randomSplit",
               "rbind",
+              "registerTempTable",
               "rename",
               "repartition",
               "repartitionByRange",
               "rollup",
               "sample",
               "sample_frac",
               "sampleBy",
+              "saveAsParquetFile",
               "saveAsTable",
               "saveDF",
               "schema",
@@ -169,6 +174,7 @@ exportMethods("arrange",
               "toJSON",
               "transform",
               "union",
+              "unionAll",
               "unionByName",
               "unique",
               "unpersist",
@@ -268,7 +274,6 @@ exportMethods("%<=>%",
               "floor",
               "format_number",
               "format_string",
-              "from_csv",
               "from_json",
               "from_unixtime",
               "from_utc_timestamp",
@@ -408,14 +413,18 @@ export("as.DataFrame",
        "cacheTable",
        "clearCache",
        "createDataFrame",
+       "createExternalTable",
        "createTable",
        "currentDatabase",
+       "dropTempTable",
        "dropTempView",
+       "jsonFile",
        "listColumns",
        "listDatabases",
        "listFunctions",
        "listTables",
        "loadDF",
+       "parquetFile",
        "read.df",
        "read.jdbc",
        "read.json",

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
@@ -226,9 +226,7 @@ setMethod("showDF",
 
 #' show
 #'
-#' If eager evaluation is enabled and the Spark object is a SparkDataFrame, evaluate the
-#' SparkDataFrame and print top rows of the SparkDataFrame, otherwise, print the class
-#' and type information of the Spark object.
+#' Print class and type information of a Spark object.
 #'
 #' @param object a Spark object. Can be a SparkDataFrame, Column, GroupedData, WindowSpec.
 #'
@@ -246,33 +244,11 @@ setMethod("showDF",
 #' @note show(SparkDataFrame) since 1.4.0
 setMethod("show", "SparkDataFrame",
           function(object) {
-            allConf <- sparkR.conf()
-            prop <- allConf[["spark.sql.repl.eagerEval.enabled"]]
-            if (!is.null(prop) && identical(prop, "true")) {
-              argsList <- list()
-              argsList$x <- object
-              prop <- allConf[["spark.sql.repl.eagerEval.maxNumRows"]]
-              if (!is.null(prop)) {
-                numRows <- as.integer(prop)
-                if (numRows > 0) {
-                  argsList$numRows <- numRows
-                }
-              }
-              prop <- allConf[["spark.sql.repl.eagerEval.truncate"]]
-              if (!is.null(prop)) {
-                truncate <- as.integer(prop)
-                if (truncate > 0) {
-                  argsList$truncate <- truncate
-                }
-              }
-              do.call(showDF, argsList)
-            } else {
-              cols <- lapply(dtypes(object), function(l) {
-                paste(l, collapse = ":")
-              })
-              s <- paste(cols, collapse = ", ")
-              cat(paste(class(object), "[", s, "]\n", sep = ""))
-            }
+            cols <- lapply(dtypes(object), function(l) {
+              paste(l, collapse = ":")
+            })
+            s <- paste(cols, collapse = ", ")
+            cat(paste(class(object), "[", s, "]\n", sep = ""))
           })
 
 #' DataTypes
@@ -521,6 +497,32 @@ setMethod("createOrReplaceTempView",
               invisible(callJMethod(x@sdf, "createOrReplaceTempView", viewName))
           })
 
+#' (Deprecated) Register Temporary Table
+#'
+#' Registers a SparkDataFrame as a Temporary Table in the SparkSession
+#' @param x A SparkDataFrame
+#' @param tableName A character vector containing the name of the table
+#'
+#' @seealso \link{createOrReplaceTempView}
+#' @rdname registerTempTable-deprecated
+#' @name registerTempTable
+#' @aliases registerTempTable,SparkDataFrame,character-method
+#' @examples
+#'\dontrun{
+#' sparkR.session()
+#' path <- "path/to/file.json"
+#' df <- read.json(path)
+#' registerTempTable(df, "json_df")
+#' new_df <- sql("SELECT * FROM json_df")
+#'}
+#' @note registerTempTable since 1.4.0
+setMethod("registerTempTable",
+          signature(x = "SparkDataFrame", tableName = "character"),
+          function(x, tableName) {
+              .Deprecated("createOrReplaceTempView")
+              invisible(callJMethod(x@sdf, "createOrReplaceTempView", tableName))
+          })
+
 #' insertInto
 #'
 #' Insert the contents of a SparkDataFrame into a table registered in the current SparkSession.
@@ -930,6 +932,7 @@ setMethod("write.orc",
 #' path <- "path/to/file.json"
 #' df <- read.json(path)
 #' write.parquet(df, "/tmp/sparkr-tmp1/")
+#' saveAsParquetFile(df, "/tmp/sparkr-tmp2/")
 #'}
 #' @note write.parquet since 1.6.0
 setMethod("write.parquet",
@@ -940,6 +943,17 @@ setMethod("write.parquet",
             invisible(handledCallJMethod(write, "parquet", path))
           })
 
+#' @rdname write.parquet
+#' @name saveAsParquetFile
+#' @aliases saveAsParquetFile,SparkDataFrame,character-method
+#' @note saveAsParquetFile since 1.4.0
+setMethod("saveAsParquetFile",
+          signature(x = "SparkDataFrame", path = "character"),
+          function(x, path) {
+            .Deprecated("write.parquet")
+            write.parquet(x, path)
+          })
+
 #' Save the content of SparkDataFrame in a text file at the specified path.
 #'
 #' Save the content of the SparkDataFrame in a text file at the specified path.
@@ -2724,6 +2738,18 @@ setMethod("union",
             dataFrame(unioned)
           })
 
+#' unionAll is deprecated - use union instead
+#' @rdname union
+#' @name unionAll
+#' @aliases unionAll,SparkDataFrame,SparkDataFrame-method
+#' @note unionAll since 1.4.0
+setMethod("unionAll",
+          signature(x = "SparkDataFrame", y = "SparkDataFrame"),
+          function(x, y) {
+            .Deprecated("union")
+            union(x, y)
+          })
+
 #' Return a new SparkDataFrame containing the union of rows, matched by column names
 #'
 #' Return a new SparkDataFrame containing the union of rows in this SparkDataFrame