From b735acc35c5450949b2d3fa01304f13ac31e1dd5 Mon Sep 17 00:00:00 2001 From: Junqing Cai Date: Tue, 15 Sep 2020 16:01:05 +0800 Subject: [PATCH] KE-17158 renew executor in standalone (#171) * dynamic request executors and renew mem/core in standalone --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- .../spark/ExecutorAllocationClient.scala | 8 +++ .../scala/org/apache/spark/SparkContext.scala | 17 +++++ .../spark/deploy/ApplicationDescription.scala | 4 +- .../apache/spark/deploy/DeployMessage.scala | 5 ++ .../deploy/client/StandaloneAppClient.scala | 23 +++++++ .../apache/spark/deploy/master/Master.scala | 65 +++++++++++++++++++ .../CoarseGrainedSchedulerBackend.scala | 31 +++++++++ .../cluster/StandaloneSchedulerBackend.scala | 15 +++++ examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kafka-0-8-assembly/pom.xml | 2 +- external/kafka-0-8/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- .../kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 48 files changed, 206 insertions(+), 42 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 2f0cad8cfb76e..659cd45733ce6 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index 3a366dd118334..5b61bbc50ad74 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index faab7505809f2..887b52d7d9832 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index d0c55e36c9955..04c23bd3a6f8c 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index ff8bdccfbcbde..2c2e8702788ef 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 5e951de41eba0..f99c8eb62f417 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 66bf236624721..a6408db21ea7b 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 06a761c8e84d5..c7855eaac622f 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 762ffda5e7b29..258b2232b471b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala index 63d87b4cd385c..a90fe55dc4d09 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationClient.scala @@ -46,6 +46,14 @@ private[spark] trait ExecutorAllocationClient { localityAwareTasks: Int, hostToLocalTaskCount: Map[String, Int]): Boolean + private[spark] def requestTotalExecutors( + numExecutors: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int]): Boolean = { + requestTotalExecutors(numExecutors, 0, Map.empty) + } + /** * Request an additional number of executors from the cluster manager. * @return whether the request is acknowledged by the cluster manager. diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 1b35bc0d63d6c..beb9a6d418881 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1643,6 +1643,23 @@ class SparkContext(config: SparkConf) extends Logging { } } + @DeveloperApi + def requestTotalExecutors( + numExecutors: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int] + ): Boolean = { + schedulerBackend match { + case b: ExecutorAllocationClient => + b.requestTotalExecutors(numExecutors, forceKillOldExecutors, + newMemoryPerExecutorMB, newCoresPerExecutor) + case _ => + logWarning("Requesting executors is not supported by current scheduler.") + false + } + } + /** * :: DeveloperApi :: * Request an additional number of executors from the cluster manager. diff --git a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala index c5c5c60923f4e..44d1b24787568 100644 --- a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala +++ b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala @@ -22,13 +22,13 @@ import java.net.URI private[spark] case class ApplicationDescription( name: String, maxCores: Option[Int], - memoryPerExecutorMB: Int, + var memoryPerExecutorMB: Int, command: Command, appUiUrl: String, eventLogDir: Option[URI] = None, // short name of compression codec used when writing event logs, if any (e.g. lzf) eventLogCodec: Option[String] = None, - coresPerExecutor: Option[Int] = None, + var coresPerExecutor: Option[Int] = None, // number of executors this application wants to start with, // only used if dynamic allocation is enabled initialExecutorLimit: Option[Int] = None, diff --git a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala index 88ad63d87a71d..a432342b0cc26 100644 --- a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala @@ -142,6 +142,11 @@ object DeployMessages { case class RequestExecutors(appId: String, requestedTotal: Int) + case class RequestExecutorsRenew(appId: String, requestedTotal: Int + , forceKillOldExecutors: Boolean + , newMemoryPerExecutorMB: Option[Int] + , newCoresPerExecutor: Option[Int]) + case class KillExecutors(appId: String, executorIds: Seq[String]) // Master to AppClient diff --git a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala index 34ade4ce6f39b..12ad2aa834a56 100644 --- a/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala +++ b/core/src/main/scala/org/apache/spark/deploy/client/StandaloneAppClient.scala @@ -207,6 +207,14 @@ private[spark] class StandaloneAppClient( logWarning("Attempted to request executors before registering with Master.") context.reply(false) } + case r: RequestExecutorsRenew => + logInfo("send RequestExecutorsRenew rpc") + master match { + case Some(m) => askAndReplyAsync(m, context, r) + case None => + logWarning("Attempted to request and renew executors.") + context.reply(false) + } case k: KillExecutors => master match { @@ -304,6 +312,21 @@ private[spark] class StandaloneAppClient( } } + def requestTotalExecutors(requestedTotal: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int]): Future[Boolean] = { + if (endpoint.get != null && appId.get != null) { + endpoint.get.ask[Boolean]( + RequestExecutorsRenew(appId.get, + requestedTotal, forceKillOldExecutors, + newMemoryPerExecutorMB, newCoresPerExecutor)) + } else { + logWarning("Attempted to request and refresh executors before driver fully initialized.") + Future.successful(false) + } + } + /** * Kill the given list of executors through the Master. * @return whether the kill request is acknowledged. diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index a471c2d0319fe..5a0b54d75d768 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -503,6 +503,11 @@ private[deploy] class Master( case RequestExecutors(appId, requestedTotal) => context.reply(handleRequestExecutors(appId, requestedTotal)) + case RequestExecutorsRenew(appId, requestedTotal, forceKillOldExecutors, + newMemoryPerExecutorMB, newCoresPerExecutor) => + context.reply(handleRefreshApplicationAndExecutors(appId, requestedTotal, + forceKillOldExecutors, newMemoryPerExecutorMB, newCoresPerExecutor)) + case KillExecutors(appId, executorIds) => val formattedExecutorIds = formatExecutorIds(executorIds) context.reply(handleKillExecutors(appId, formattedExecutorIds)) @@ -921,6 +926,66 @@ private[deploy] class Master( } } + private def printExecutorStatus(appInfo: ApplicationInfo): Unit = { + logInfo(s"${appInfo.id} current executor:" + + appInfo.executors.values.map(executor => s"(${executor.id},${executor.state})") + .toList.mkString(" ")) + } + + private def handleRefreshApplicationAndExecutors(appId: String, requestedTotal: Int + , forceKillOldExecutors: Boolean + , newMemoryPerExecutorMB: Option[Int] + , newCoresPerExecutor: Option[Int]): Boolean = { + idToApp.get(appId) match { + case Some(appInfo) => + val appDesc = appInfo.desc + logInfo(s"Application $appId before requested and " + + s"renew to set total executors to $requestedTotal, " + + s"newMemoryPerExecutorMB to ${appDesc.memoryPerExecutorMB}, " + + s"newCoresPerExecutor to ${appDesc.coresPerExecutor}, ") + printExecutorStatus(appInfo) + + newMemoryPerExecutorMB match { + case Some(i) => appDesc.memoryPerExecutorMB = i + } + + if (newCoresPerExecutor != null) { + appDesc.coresPerExecutor = newCoresPerExecutor + } + appInfo.executorLimit = requestedTotal + + handleKillOldExecutors(appInfo, forceKillOldExecutors) + schedule() + + logInfo(s"Application $appId after requested and renew to set " + + s"newMemoryPerExecutorMB to ${appDesc.memoryPerExecutorMB}, " + + s"newCoresPerExecutor to ${appDesc.coresPerExecutor}, ") + printExecutorStatus(appInfo) + true + case None => + logWarning(s"Unknown application $appId requested $requestedTotal total executors.") + false + } + } + + private def handleKillOldExecutors(appInfo: ApplicationInfo, + forceKillOldExecutors: Boolean): Boolean = { + for (execId <- appInfo.executors.keys) { + appInfo.executors.get(execId) match { + case Some(executorDesc) => + if (forceKillOldExecutors || ExecutorState.isFinished(executorDesc.state)) { + appInfo.removeExecutor(executorDesc) + killExecutor(executorDesc) + logInfo("remote kill executor " + executorDesc.fullId + + " on worker " + executorDesc.worker.id) + } + case None => + logWarning("executor not found when kill") + } + } + true + } + /** * Handle a kill request from the given application. * diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala index 45aed63aa55d0..eea0737fb5b51 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala @@ -588,6 +588,32 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp defaultAskTimeout.awaitResult(response) } + final override def requestTotalExecutors(numExecutors: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int]): Boolean = { + if (numExecutors < 0) { + throw new IllegalArgumentException( + "Attempted to request a negative number of executor(s) " + + s"$numExecutors from the cluster manager. Please specify a positive number!") + } + logInfo(s"request to total executor and renew $numExecutors , " + + s"forceKillOldExecutors:$forceKillOldExecutors , " + + s"newMemoryPerExecutorMB:$newMemoryPerExecutorMB ," + + s"newCoresPerExecutor:$newCoresPerExecutor") + + val response = synchronized { + this.requestedTotalExecutors = numExecutors + numPendingExecutors = + math.max(numExecutors - numExistingExecutors + executorsPendingToRemove.size, 0) + + doRequestTotalExecutors(numExecutors, + forceKillOldExecutors, newMemoryPerExecutorMB, newCoresPerExecutor) + } + + defaultAskTimeout.awaitResult(response) + } + /** * Request executors from the cluster manager by specifying the total number desired, * including existing pending and running executors. @@ -603,6 +629,11 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp protected def doRequestTotalExecutors(requestedTotal: Int): Future[Boolean] = Future.successful(false) + protected def doRequestTotalExecutors(requestedTotal: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int]): Future[Boolean] = + Future.successful(false) /** * Request that the cluster manager kill the specified executors. * diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala index f73a58ff5d48c..dc42243532ca4 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala @@ -198,6 +198,21 @@ private[spark] class StandaloneSchedulerBackend( } } + protected override def doRequestTotalExecutors + (requestedTotal: Int, + forceKillOldExecutors: Boolean, + newMemoryPerExecutorMB: Option[Int], + newCoresPerExecutor: Option[Int]): + Future[Boolean] = { + Option(client) match { + case Some(c) => c.requestTotalExecutors( + requestedTotal, forceKillOldExecutors, + newMemoryPerExecutorMB, newCoresPerExecutor) + case None => + logWarning("Attempted to request executors before driver fully initialized.") + Future.successful(false) + } + } /** * Kill the given list of executors through the Master. * @return whether the kill request is acknowledged. diff --git a/examples/pom.xml b/examples/pom.xml index 2ed0cac94bba5..cfbae9ff95c10 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 36f6971eddff9..2e1d1634e2915 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 729a3fb586507..f9f0b7523bcf9 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 0698949d2ec31..5dd51dfd42818 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index b97d8e1ea8ff5..e19c6ef4fd99e 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index a93290d844ff3..90900a12e93af 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index f083eaad79583..191d0f2e0e7a1 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 58c6f8ba5e08e..450b5016a90ca 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 41464b20bfd26..3b3b8509776a1 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml index 7c10eef453876..30fdabec562ec 100644 --- a/external/kafka-0-8-assembly/pom.xml +++ b/external/kafka-0-8-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index f6295af090ead..abaa07de057a6 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 0dffd74d59c4a..5c995f72ba711 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 3227fdd0197cf..166b3d5a8c502 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 48b7874a88a40..cf339bf45daa9 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index affc26ee13d56..6433f0232c99c 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index e5b4ac9e1db31..90c0aa54a3648 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 4aea3d1a08c15..613741989af6c 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 4604d49b379f7..9ede6e5e57709 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index efb34db2cb6f8..a403f202a553b 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/pom.xml b/pom.xml index 4d3fa0ff233f9..07ed7898e66c9 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 78d05e1f9f1d2..871fff4d293d5 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 82debf60ac72c..0edcbe7b05c3a 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 6d16b18c3a9c5..9e4a86c550ad9 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index ef37400a3dbe2..b0b591bf9b063 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index eb4ad88b912b1..2853b4c06ccf6 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 5021c275b18aa..15d6706ffa679 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 537802f964aff..c142d26d6d14b 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 3dc79370ceeb8..77f3a45e9bcaa 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 66f8d85531ca5..88092025827e0 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 119b59cfe7322..59da9e42fe062 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 58b140d909e6e..067ba2b6d62d9 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 2.4.1-kylin-r48 + 2.4.1-kylin-r51 ../pom.xml