From 4fad75ad0c2f906cf7c8477321b906ed5b82463a Mon Sep 17 00:00:00 2001 From: carlmartin Date: Wed, 26 Nov 2014 20:04:16 +0800 Subject: [PATCH 1/5] Add the Error infomation using spark-sql in yarn-cluster mode --- .../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 8a62519bd2315..55f6843090148 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -142,6 +142,8 @@ object SparkSubmit { printErrorAndExit("Cluster deploy mode is currently not supported for python applications.") case (_, CLUSTER) if isShell(args.primaryResource) => printErrorAndExit("Cluster deploy mode is not applicable to Spark shells.") + case (_, CLUSTER) if isSql(args.mainClass) => + printErrorAndExit("Cluster deploy mode is not applicable to Spark SQLs.") case _ => } @@ -388,6 +390,13 @@ object SparkSubmit { primaryResource == SPARK_SHELL || primaryResource == PYSPARK_SHELL } + /** + * Return whether the given main class represents a sql shell. + */ + private[spark] def isSql(mainClass: String): Boolean = { + mainClass == "org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" + } + /** * Return whether the given primary resource requires running python. */ From ec957bc3d065c2db70838568da59f1b2dd2bf211 Mon Sep 17 00:00:00 2001 From: carlmartin Date: Wed, 26 Nov 2014 20:30:11 +0800 Subject: [PATCH 2/5] Add the some error infomation if using spark-sql in yarn-cluster mode --- .../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 55f6843090148..b5b6fe4e7b0ec 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -142,8 +142,8 @@ object SparkSubmit { printErrorAndExit("Cluster deploy mode is currently not supported for python applications.") case (_, CLUSTER) if isShell(args.primaryResource) => printErrorAndExit("Cluster deploy mode is not applicable to Spark shells.") - case (_, CLUSTER) if isSql(args.mainClass) => - printErrorAndExit("Cluster deploy mode is not applicable to Spark SQLs.") + case (_, CLUSTER) if isSqlShell(args.mainClass) => + printErrorAndExit("Cluster deploy mode is not applicable to Spark Sql shells.") case _ => } @@ -393,7 +393,7 @@ object SparkSubmit { /** * Return whether the given main class represents a sql shell. */ - private[spark] def isSql(mainClass: String): Boolean = { + private[spark] def isSqlShell(mainClass: String): Boolean = { mainClass == "org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" } From 8e112c58b80f2a61358e068da05479b126580305 Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Thu, 27 Nov 2014 10:43:10 +0800 Subject: [PATCH 3/5] singular form --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index b5b6fe4e7b0ec..1b203c97c5385 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -143,7 +143,7 @@ object SparkSubmit { case (_, CLUSTER) if isShell(args.primaryResource) => printErrorAndExit("Cluster deploy mode is not applicable to Spark shells.") case (_, CLUSTER) if isSqlShell(args.mainClass) => - printErrorAndExit("Cluster deploy mode is not applicable to Spark Sql shells.") + printErrorAndExit("Cluster deploy mode is not applicable to Spark Sql shell.") case _ => } From e6c1eb7b0978da8d9faa394bc2ec35f51d05117c Mon Sep 17 00:00:00 2001 From: carlmartin Date: Sun, 30 Nov 2014 16:25:35 +0800 Subject: [PATCH 4/5] add a comment in bin/spark-sql to remind user who wants to change the class --- bin/spark-sql | 1 + core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/spark-sql b/bin/spark-sql index 63d00437d508d..5676233694e4b 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -23,6 +23,7 @@ # Enter posix mode for bash set -o posix +# SparkSQLCLIDriver class name was used in SparkSubmit. Be careful about renaming it. CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" # Figure out where Spark is installed diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 0da8c538848eb..0c7d247519447 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -143,7 +143,7 @@ object SparkSubmit { case (_, CLUSTER) if isShell(args.primaryResource) => printErrorAndExit("Cluster deploy mode is not applicable to Spark shells.") case (_, CLUSTER) if isSqlShell(args.mainClass) => - printErrorAndExit("Cluster deploy mode is not applicable to Spark Sql shell.") + printErrorAndExit("Cluster deploy mode is not applicable to Spark SQL shell.") case _ => } From 35829a9a6dfffc9fb365627a9ffd9da192030464 Mon Sep 17 00:00:00 2001 From: carlmartin Date: Sun, 30 Nov 2014 18:53:08 +0800 Subject: [PATCH 5/5] improve the description of comment --- bin/spark-sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/spark-sql b/bin/spark-sql index 5676233694e4b..3b6cc420fea81 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -23,7 +23,8 @@ # Enter posix mode for bash set -o posix -# SparkSQLCLIDriver class name was used in SparkSubmit. Be careful about renaming it. +# NOTE: This exact class name is matched downstream by SparkSubmit. +# Any changes need to be reflected there. CLASS="org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver" # Figure out where Spark is installed