From 23b392b36018664fb293c07cc0b3ebe08e0ae043 Mon Sep 17 00:00:00 2001 From: xushiwei 00425595 Date: Sun, 19 Jan 2020 14:14:45 -0800 Subject: [PATCH] [SPARK-30371][K8S] Add spark.kubernetes.driver.master conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What changes were proposed in this pull request? make KUBERNETES_MASTER_INTERNAL_URL configurable ### Why are the changes needed? we do not always use the default port number 443 to access our kube-apiserver, and even in some mulit-tenant cluster, people do not use the service `kubernetes.default.svc` to access the kube-apiserver, so make the internal master configurable is necessary。 ### Does this PR introduce any user-facing change? user can configure the internal master url by ``` --conf spark.kubernetes.internal.master=https://kubernetes.default.svc:6443 ``` ### How was this patch tested? run in multi-cluster that do not use the https://kubernetes.default.svc to access the kube-apiserver Closes #27029 from wackxu/internalmaster. Authored-by: xushiwei 00425595 Signed-off-by: Dongjoon Hyun --- docs/running-on-kubernetes.md | 7 +++++++ .../main/scala/org/apache/spark/deploy/k8s/Config.scala | 7 +++++++ .../scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index b289b898297c8..009eb27ec7061 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -433,6 +433,13 @@ See the [configuration page](configuration.html) for information on Spark config configuration properties e.g. spark.kubernetes.namespace. + + spark.kubernetes.driver.master + https://kubernetes.default.svc + + The internal Kubernetes master (API server) address to be used for driver to request executors. + + spark.kubernetes.namespace default diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 45385b4809588..f6354a821c0be 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -36,6 +36,13 @@ private[spark] object Config extends Logging { .stringConf .createOptional + val KUBERNETES_DRIVER_MASTER_URL = + ConfigBuilder("spark.kubernetes.driver.master") + .doc("The internal Kubernetes master (API server) address " + + "to be used for driver to request executors.") + .stringConf + .createWithDefault(KUBERNETES_MASTER_INTERNAL_URL) + val KUBERNETES_NAMESPACE = ConfigBuilder("spark.kubernetes.namespace") .doc("The namespace that will be used for running the driver and executor pods.") diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 31ca06b721c5d..7d8efacbad87d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -51,7 +51,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit "If the application is deployed using spark-submit in cluster mode, the driver pod name " + "must be provided.") (KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX, - KUBERNETES_MASTER_INTERNAL_URL, + sc.conf.get(KUBERNETES_DRIVER_MASTER_URL), Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)), Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH))) } else {