From 836c4bf80cc20f32db403f6e5d82fd82c94bf055 Mon Sep 17 00:00:00 2001 From: cafreeman Date: Wed, 8 Apr 2015 14:11:59 -0500 Subject: [PATCH] Update `createDataFrame` and `toDF` Refactored to use the new `structType` and `structField` functions. --- R/pkg/R/SQLContext.R | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R index 30fb0f8b5a64a..4f05ba524a01a 100644 --- a/R/pkg/R/SQLContext.R +++ b/R/pkg/R/SQLContext.R @@ -54,9 +54,9 @@ infer_type <- function(x) { # StructType types <- lapply(x, infer_type) fields <- lapply(1:length(x), function(i) { - field(names[[i]], types[[i]], TRUE) + structField(names[[i]], types[[i]], TRUE) }) - do.call(buildSchema, fields) + do.call(structType, fields) } } else if (length(x) > 1) { list(type = "array", elementType = type, containsNull = TRUE) @@ -110,7 +110,7 @@ createDataFrame <- function(sqlCtx, data, schema = NULL, samplingRatio = 1.0) { stop(paste("unexpected type:", class(data))) } - if (is.null(schema) || (!inherits(schema, "struct") && is.null(names(schema)))) { + if (is.null(schema) || (!inherits(schema, "structType") && is.null(names(schema)))) { row <- first(rdd) names <- if (is.null(schema)) { names(row) @@ -135,18 +135,18 @@ createDataFrame <- function(sqlCtx, data, schema = NULL, samplingRatio = 1.0) { types <- lapply(row, infer_type) fields <- lapply(1:length(row), function(i) { - field(names[[i]], types[[i]], TRUE) + structField(names[[i]], types[[i]], TRUE) }) - schema <- do.call(buildSchema, fields) + schema <- do.call(structType, fields) } - stopifnot(class(schema) == "struct") - schemaString <- tojson(schema) + stopifnot(class(schema) == "structType") + # schemaString <- tojson(schema) jrdd <- getJRDD(lapply(rdd, function(x) x), "row") srdd <- callJMethod(jrdd, "rdd") sdf <- callJStatic("org.apache.spark.sql.api.r.SQLUtils", "createDF", - srdd, schemaString, sqlCtx) + srdd, schema$jobj, sqlCtx) dataFrame(sdf) }