From 3f3253d31ecdf6c2d9e497244b86d8e92b87ee7b Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Wed, 19 Oct 2022 16:18:45 +0800
Subject: [PATCH 01/15] init

---
 .../org/apache/kyuubi/config/KyuubiConf.scala | 23 +++--
 .../org/apache/kyuubi/util/ThreadUtils.scala  |  5 +
 .../KubernetesApplicationOperation.scala      | 34 +++----
 .../apache/kyuubi/util/KubernetesUtils.scala  | 95 +++++++++++++++++++
 4 files changed, 129 insertions(+), 28 deletions(-)
 create mode 100644 kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index f83866c4ade..9bf8009e25f 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -828,6 +828,21 @@ object KyuubiConf {
       .version("1.4.0")
       .fallbackConf(FRONTEND_WORKER_KEEPALIVE_TIME)
 
+  val KUBERNETES_CONTEXT: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.context")
+      .doc("The desired context from your kubernetes config file used to configure the K8S " +
+        "client for interacting with the cluster.")
+      .version("1.6.0")
+      .stringConf
+      .createOptional
+
+  val SERVER_PREFER_BUILD_K8S_CLIENT_FROM_POD_ENV: ConfigEntry[Boolean] =
+    buildConf("kyuubi.kubernetes.client.build.preferFromPodEnv")
+      .doc("")
+      .version("1.6.1")
+      .booleanConf
+      .createWithDefault(false)
+
   // ///////////////////////////////////////////////////////////////////////////////////////////////
   //                                 SQL Engine Configuration                                    //
   // ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -1950,14 +1965,6 @@ object KyuubiConf {
       .booleanConf
       .createWithDefault(true)
 
-  val KUBERNETES_CONTEXT: OptionalConfigEntry[String] =
-    buildConf("kyuubi.kubernetes.context")
-      .doc("The desired context from your kubernetes config file used to configure the K8S " +
-        "client for interacting with the cluster.")
-      .version("1.6.0")
-      .stringConf
-      .createOptional
-
   private val serverOnlyConfEntries: Set[ConfigEntry[_]] = Set(
     FRONTEND_BIND_HOST,
     FRONTEND_BIND_PORT,
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala
index d95d7a6e99b..8ce4bb2e589 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala
@@ -62,6 +62,11 @@ object ThreadUtils extends Logging {
     Executors.newFixedThreadPool(nThreads, threadFactory).asInstanceOf[ThreadPoolExecutor]
   }
 
+  def newDaemonCachedThreadPool(prefix: String): ThreadPoolExecutor = {
+    val threadFactory = new NamedThreadFactory(prefix, daemon = true)
+    Executors.newCachedThreadPool(threadFactory).asInstanceOf[ThreadPoolExecutor]
+  }
+
   def awaitResult[T](awaitable: Awaitable[T], atMost: Duration): T = {
     try {
       // `awaitPermission` is not actually used anywhere so it's safe to pass in null here.
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index 85a9794cc15..08fcc2c2744 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -18,13 +18,14 @@
 package org.apache.kyuubi.engine
 
 import io.fabric8.kubernetes.api.model.{Pod, PodList}
-import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient, KubernetesClient, KubernetesClientException}
+import io.fabric8.kubernetes.client.KubernetesClient
 import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.KUBERNETES_CONTEXT
 import org.apache.kyuubi.engine.ApplicationState.{ApplicationState, FAILED, FINISHED, PENDING, RUNNING}
+import org.apache.kyuubi.engine.KubernetesApplicationOperation.SPARK_APP_SELECTOR
+import org.apache.kyuubi.util.KubernetesUtils
 
 class KubernetesApplicationOperation extends ApplicationOperation with Logging {
 
@@ -33,24 +34,16 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
   private var jpsOperation: JpsApplicationOperation = _
 
   override def initialize(conf: KyuubiConf): Unit = {
-    info("Start Initialize Kubernetes Client.")
-    val contextOpt = conf.get(KUBERNETES_CONTEXT)
-    if (contextOpt.isEmpty) {
-      warn("Skip Initialize Kubernetes Client, because of Context not set.")
-      return
-    }
     jpsOperation = new JpsApplicationOperation
     jpsOperation.initialize(conf)
-    kubernetesClient =
-      try {
-        val client = new DefaultKubernetesClient(Config.autoConfigure(contextOpt.get))
-        info(s"Initialized Kubernetes Client connect to: ${client.getMasterUrl}")
-        client
-      } catch {
-        case e: KubernetesClientException =>
-          error("Fail to init KubernetesClient for KubernetesApplicationOperation", e)
-          null
-      }
+
+    info("Start Initialize Kubernetes Client.")
+    kubernetesClient = KubernetesUtils.buildKubernetesClient(conf)
+    if (kubernetesClient == null) {
+      warn("Fail to init Kubernetes Client for Kubernetes Application Operation")
+    } else {
+      info(s"Initialized Kubernetes Client connect to: ${kubernetesClient.getMasterUrl}")
+    }
   }
 
   override def isSupported(clusterManager: Option[String]): Boolean = {
@@ -91,8 +84,8 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
         if (podList.size() != 0) {
           val pod = podList.get(0)
           val info = ApplicationInfo(
-            // Can't get appId, get Pod UID instead.
-            id = pod.getMetadata.getUid,
+            // spark pods always tag label `spark-app-selector:<spark-app-id>`
+            id = pod.getMetadata.getLabels.get(SPARK_APP_SELECTOR),
             name = pod.getMetadata.getName,
             state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase),
             error = Option(pod.getStatus.getReason))
@@ -135,6 +128,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
 
 object KubernetesApplicationOperation extends Logging {
   val LABEL_KYUUBI_UNIQUE_KEY = "kyuubi-unique-tag"
+  val SPARK_APP_SELECTOR = "spark-app-selector"
 
   def toApplicationState(state: String): ApplicationState = state match {
     // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/core/types.go#L2396
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
new file mode 100644
index 00000000000..c55d2666fcb
--- /dev/null
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.util
+
+import java.io.{File, FileNotFoundException}
+
+import com.google.common.base.Charsets
+import com.google.common.io.Files
+import io.fabric8.kubernetes.client.{Config, ConfigBuilder, DefaultKubernetesClient, KubernetesClient, KubernetesClientException}
+import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
+import okhttp3.{Dispatcher, OkHttpClient}
+
+import org.apache.kyuubi.{Logging, Utils}
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.KUBERNETES_CONTEXT
+
+object KubernetesUtils extends Logging {
+
+  def buildKubernetesClient(conf: KyuubiConf): KubernetesClient = {
+    if (conf.get(KyuubiConf.SERVER_PREFER_BUILD_K8S_CLIENT_FROM_POD_ENV) && Utils.isOnK8s) {
+      try {
+        buildKubernetesClientFromPodEnv
+      } catch {
+        case e: KubernetesClientException =>
+          error("Fail to build kubernetes client for kubernetes application operation", e)
+          null
+        case e: FileNotFoundException =>
+          error(
+            "Fail to build kubernetes client for kubernetes application operation, " +
+              "due to file not found",
+            e)
+          null
+      }
+    } else {
+      val contextOpt = conf.get(KUBERNETES_CONTEXT)
+      if (contextOpt.isEmpty) {
+        warn("Skip initialize kubernetes client, because of context not set.")
+        null
+      } else {
+        try {
+          val client = new DefaultKubernetesClient(Config.autoConfigure(contextOpt.get))
+          info(s"Initialized kubernetes client connect to: ${client.getMasterUrl}")
+          client
+        } catch {
+          case e: KubernetesClientException =>
+            error("Fail to init kubernetes client for kubernetes application operation", e)
+            null
+        }
+      }
+    }
+  }
+
+  private def buildKubernetesClientFromPodEnv: KubernetesClient = {
+    val nsFile = new File("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+
+    // use service account for client ca and token
+    val ca = new File("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
+    val token = new File("/var/run/secrets/kubernetes.io/serviceaccount/token")
+
+    // (tcp://XXX:XXX) => (https://XXX:XXX)
+    val master = sys.env("KUBERNETES_PORT").replace("tcp", "https")
+
+    val config = new ConfigBuilder()
+      .withMasterUrl(master)
+      .withNamespace(Files.asCharSource(nsFile, Charsets.UTF_8).readFirstLine())
+      .withOauthToken(Files.asCharSource(token, Charsets.UTF_8).readFirstLine())
+      .withCaCertFile(ca.getAbsolutePath)
+      .build()
+
+    val dispatcher = new Dispatcher(
+      ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
+    val factoryWithCustomDispatcher = new OkHttpClientFactory() {
+      override protected def additionalConfig(builder: OkHttpClient.Builder): Unit = {
+        builder.dispatcher(dispatcher)
+      }
+    }
+
+    new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config)
+  }
+}

From b77bbd9b014d6c7d100c3dc1328ece766b43a755 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Wed, 19 Oct 2022 17:14:19 +0800
Subject: [PATCH 02/15] add doc

---
 .../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 9bf8009e25f..7181ef266a4 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -838,8 +838,9 @@ object KyuubiConf {
 
   val SERVER_PREFER_BUILD_K8S_CLIENT_FROM_POD_ENV: ConfigEntry[Boolean] =
     buildConf("kyuubi.kubernetes.client.build.preferFromPodEnv")
-      .doc("")
-      .version("1.6.1")
+      .doc("If enabled, kyuubi will build kubernetes client from pod env " +
+        "and service account files. (only when kyuubi run in kubernetes pod)")
+      .version("1.7.0")
       .booleanConf
       .createWithDefault(false)
 

From e7f9ec05728221d08fe2d192bb92369a8007c4b9 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Wed, 19 Oct 2022 17:47:58 +0800
Subject: [PATCH 03/15] add setting

---
 docs/deployment/settings.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 5905f868923..0ac93b9ccc5 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -369,6 +369,7 @@ kyuubi.kinit.principal|&lt;undefined&gt;|Name of the Kerberos principal.|string|
 
 Key | Default | Meaning | Type | Since
 --- | --- | --- | --- | ---
+kyuubi.kubernetes.client.build.preferFromPodEnv|false|If enabled, kyuubi will build kubernetes client from pod env and service account files. (only when kyuubi run in kubernetes pod)|boolean|1.7.0
 kyuubi.kubernetes.context|&lt;undefined&gt;|The desired context from your kubernetes config file used to configure the K8S client for interacting with the cluster.|string|1.6.0
 
 

From a7f0b55167a3467dfa639621715c868a42065b1e Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Thu, 20 Oct 2022 17:44:11 +0800
Subject: [PATCH 04/15] ad

---
 .../kyuubi/engine/KubernetesApplicationOperation.scala    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index 08fcc2c2744..062535557e4 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -24,7 +24,7 @@ import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.engine.ApplicationState.{ApplicationState, FAILED, FINISHED, PENDING, RUNNING}
-import org.apache.kyuubi.engine.KubernetesApplicationOperation.SPARK_APP_SELECTOR
+import org.apache.kyuubi.engine.KubernetesApplicationOperation.SPARK_APP_ID_LABEL
 import org.apache.kyuubi.util.KubernetesUtils
 
 class KubernetesApplicationOperation extends ApplicationOperation with Logging {
@@ -37,7 +37,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
     jpsOperation = new JpsApplicationOperation
     jpsOperation.initialize(conf)
 
-    info("Start Initialize Kubernetes Client.")
+    info("Start initializing Kubernetes Client.")
     kubernetesClient = KubernetesUtils.buildKubernetesClient(conf)
     if (kubernetesClient == null) {
       warn("Fail to init Kubernetes Client for Kubernetes Application Operation")
@@ -85,7 +85,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
           val pod = podList.get(0)
           val info = ApplicationInfo(
             // spark pods always tag label `spark-app-selector:<spark-app-id>`
-            id = pod.getMetadata.getLabels.get(SPARK_APP_SELECTOR),
+            id = pod.getMetadata.getLabels.get(SPARK_APP_ID_LABEL),
             name = pod.getMetadata.getName,
             state = KubernetesApplicationOperation.toApplicationState(pod.getStatus.getPhase),
             error = Option(pod.getStatus.getReason))
@@ -128,7 +128,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
 
 object KubernetesApplicationOperation extends Logging {
   val LABEL_KYUUBI_UNIQUE_KEY = "kyuubi-unique-tag"
-  val SPARK_APP_SELECTOR = "spark-app-selector"
+  val SPARK_APP_ID_LABEL = "spark-app-selector"
 
   def toApplicationState(state: String): ApplicationState = state match {
     // https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/core/types.go#L2396

From 5884475d758486b458d054efc62fa76998e07403 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 12:40:09 +0800
Subject: [PATCH 05/15] rebuild

---
 docs/deployment/settings.md                   |   9 +-
 .../org/apache/kyuubi/config/KyuubiConf.scala |  55 +++++++-
 .../apache/kyuubi/util/KubernetesUtils.scala  | 129 +++++++++++-------
 3 files changed, 141 insertions(+), 52 deletions(-)

diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 0ac93b9ccc5..ceeb7a0b6f7 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -369,8 +369,15 @@ kyuubi.kinit.principal|&lt;undefined&gt;|Name of the Kerberos principal.|string|
 
 Key | Default | Meaning | Type | Since
 --- | --- | --- | --- | ---
-kyuubi.kubernetes.client.build.preferFromPodEnv|false|If enabled, kyuubi will build kubernetes client from pod env and service account files. (only when kyuubi run in kubernetes pod)|boolean|1.7.0
+kyuubi.kubernetes.authenticate.ca cert file|&lt;undefined&gt;|kubernetes client authenticate ca cert file|string|1.7.0
+kyuubi.kubernetes.authenticate.clientCertFile|&lt;undefined&gt;|kubernetes client authenticate client cert file|string|1.7.0
+kyuubi.kubernetes.authenticate.clientKeyFile|&lt;undefined&gt;|kubernetes client authenticate client key file|string|1.7.0
+kyuubi.kubernetes.authenticate.oauthToken|&lt;undefined&gt;|kubernetes client authenticate oauth token value|string|1.7.0
+kyuubi.kubernetes.authenticate.oauthTokenFile|&lt;undefined&gt;|kubernetes client authenticate oauth token file|string|1.7.0
 kyuubi.kubernetes.context|&lt;undefined&gt;|The desired context from your kubernetes config file used to configure the K8S client for interacting with the cluster.|string|1.6.0
+kyuubi.kubernetes.master.address|&lt;undefined&gt;|kubernetes master address for build kubernetes client|string|1.7.0
+kyuubi.kubernetes.namespace|default||string|1.7.0
+kyuubi.kubernetes.trust.certificates|false|If set to true then client can submit to kubernetes cluster only with token|boolean|1.7.0
 
 
 ### Metadata
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 7181ef266a4..9509c9d404b 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -836,10 +836,57 @@ object KyuubiConf {
       .stringConf
       .createOptional
 
-  val SERVER_PREFER_BUILD_K8S_CLIENT_FROM_POD_ENV: ConfigEntry[Boolean] =
-    buildConf("kyuubi.kubernetes.client.build.preferFromPodEnv")
-      .doc("If enabled, kyuubi will build kubernetes client from pod env " +
-        "and service account files. (only when kyuubi run in kubernetes pod)")
+  val KUBERNETES_NAMESPACE: ConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.namespace")
+      .version("1.7.0")
+      .stringConf
+      .createWithDefault("default")
+
+  val KUBERNETES_MASTER: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.master.address")
+      .doc("kubernetes master address for build kubernetes client")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.authenticate.oauthTokenFile")
+      .doc("kubernetes client authenticate oauth token file")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_AUTHENTICATE_OAUTH_TOKEN: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.authenticate.oauthToken")
+      .doc("kubernetes client authenticate oauth token value")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_AUTHENTICATE_CLIENT_KEY_FILE: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.authenticate.clientKeyFile")
+      .doc("kubernetes client authenticate client key file")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_AUTHENTICATE_CLIENT_CERT_FILE: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.authenticate.clientCertFile")
+      .doc("kubernetes client authenticate client cert file")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_AUTHENTICATE_CA_CERT_FILE: OptionalConfigEntry[String] =
+    buildConf("kyuubi.kubernetes.authenticate.ca cert file")
+      .doc("kubernetes client authenticate ca cert file")
+      .version("1.7.0")
+      .stringConf
+      .createOptional
+
+  val KUBERNETES_TRUST_CERTIFICATES: ConfigEntry[Boolean] =
+    buildConf("kyuubi.kubernetes.trust.certificates")
+      .doc("If set to true then client can submit to kubernetes cluster only with token")
       .version("1.7.0")
       .booleanConf
       .createWithDefault(false)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index c55d2666fcb..095c834f791 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -17,71 +17,77 @@
 
 package org.apache.kyuubi.util
 
-import java.io.{File, FileNotFoundException}
+import java.io.File
 
+import com.fasterxml.jackson.databind.ObjectMapper
 import com.google.common.base.Charsets
 import com.google.common.io.Files
-import io.fabric8.kubernetes.client.{Config, ConfigBuilder, DefaultKubernetesClient, KubernetesClient, KubernetesClientException}
+import io.fabric8.kubernetes.client.{Config, ConfigBuilder, DefaultKubernetesClient, KubernetesClient}
+import io.fabric8.kubernetes.client.Config.autoConfigure
 import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
 import okhttp3.{Dispatcher, OkHttpClient}
 
-import org.apache.kyuubi.{Logging, Utils}
+import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.KUBERNETES_CONTEXT
+import org.apache.kyuubi.config.KyuubiConf.{KUBERNETES_AUTHENTICATE_CA_CERT_FILE, KUBERNETES_AUTHENTICATE_CLIENT_CERT_FILE, KUBERNETES_AUTHENTICATE_CLIENT_KEY_FILE, KUBERNETES_AUTHENTICATE_OAUTH_TOKEN, KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE, KUBERNETES_CONTEXT, KUBERNETES_MASTER, KUBERNETES_NAMESPACE, KUBERNETES_TRUST_CERTIFICATES}
 
 object KubernetesUtils extends Logging {
 
   def buildKubernetesClient(conf: KyuubiConf): KubernetesClient = {
-    if (conf.get(KyuubiConf.SERVER_PREFER_BUILD_K8S_CLIENT_FROM_POD_ENV) && Utils.isOnK8s) {
-      try {
-        buildKubernetesClientFromPodEnv
-      } catch {
-        case e: KubernetesClientException =>
-          error("Fail to build kubernetes client for kubernetes application operation", e)
-          null
-        case e: FileNotFoundException =>
-          error(
-            "Fail to build kubernetes client for kubernetes application operation, " +
-              "due to file not found",
-            e)
-          null
-      }
-    } else {
-      val contextOpt = conf.get(KUBERNETES_CONTEXT)
-      if (contextOpt.isEmpty) {
-        warn("Skip initialize kubernetes client, because of context not set.")
-        null
-      } else {
-        try {
-          val client = new DefaultKubernetesClient(Config.autoConfigure(contextOpt.get))
-          info(s"Initialized kubernetes client connect to: ${client.getMasterUrl}")
-          client
-        } catch {
-          case e: KubernetesClientException =>
-            error("Fail to init kubernetes client for kubernetes application operation", e)
-            null
-        }
-      }
+    val master = masterAddress(conf)
+    if (master == null || master.isEmpty) {
+      warn("Need set kubernetes master url, if you want to set up kubernetes client")
+      return null
     }
-  }
 
-  private def buildKubernetesClientFromPodEnv: KubernetesClient = {
-    val nsFile = new File("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+    val namespace = conf.get(KUBERNETES_NAMESPACE)
+    val serviceAccountToken =
+      Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)).filter(_.exists)
+    val serviceAccountCaCrt =
+      Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH)).filter(_.exists)
+
+    val oauthTokenFile = conf.get(KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE)
+      .map(new File(_))
+      .orElse(serviceAccountToken)
+    val oauthTokenValue = conf.get(KUBERNETES_AUTHENTICATE_OAUTH_TOKEN)
 
-    // use service account for client ca and token
-    val ca = new File("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")
-    val token = new File("/var/run/secrets/kubernetes.io/serviceaccount/token")
+    KubernetesUtils.requireNandDefined(
+      oauthTokenFile,
+      oauthTokenValue,
+      s"Cannot specify OAuth token through both a oauth token file and a " +
+        s"oauth token value.")
 
-    // (tcp://XXX:XXX) => (https://XXX:XXX)
-    val master = sys.env("KUBERNETES_PORT").replace("tcp", "https")
+    val caCertFile = conf
+      .get(KUBERNETES_AUTHENTICATE_CA_CERT_FILE)
+      .orElse(serviceAccountCaCrt.map(_.getAbsolutePath))
+    val clientKeyFile = conf.get(KUBERNETES_AUTHENTICATE_CLIENT_KEY_FILE)
+    val clientCertFile = conf.get(KUBERNETES_AUTHENTICATE_CLIENT_CERT_FILE)
 
-    val config = new ConfigBuilder()
+    // Allow for specifying a context used to auto-configure from the users K8S config file
+    val kubeContext = conf.get(KUBERNETES_CONTEXT).filter(_.nonEmpty)
+    info("Auto-configuring K8S client using " +
+      kubeContext.map("context " + _).getOrElse("current context") +
+      " from users K8S config file")
+
+    val config = new ConfigBuilder(autoConfigure(kubeContext.orNull))
+      .withApiVersion("v1")
       .withMasterUrl(master)
-      .withNamespace(Files.asCharSource(nsFile, Charsets.UTF_8).readFirstLine())
-      .withOauthToken(Files.asCharSource(token, Charsets.UTF_8).readFirstLine())
-      .withCaCertFile(ca.getAbsolutePath)
-      .build()
+      .withNamespace(namespace)
+      .withTrustCerts(conf.get(KUBERNETES_TRUST_CERTIFICATES))
+      .withOption(oauthTokenValue) {
+        (token, configBuilder) => configBuilder.withOauthToken(token)
+      }.withOption(oauthTokenFile) {
+        (file, configBuilder) =>
+          configBuilder.withOauthToken(Files.asCharSource(file, Charsets.UTF_8).read())
+      }.withOption(caCertFile) {
+        (file, configBuilder) => configBuilder.withCaCertFile(file)
+      }.withOption(clientKeyFile) {
+        (file, configBuilder) => configBuilder.withClientKeyFile(file)
+      }.withOption(clientCertFile) {
+        (file, configBuilder) => configBuilder.withClientCertFile(file)
+      }.build()
 
+    // https://github.com/fabric8io/kubernetes-client/issues/3547
     val dispatcher = new Dispatcher(
       ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
     val factoryWithCustomDispatcher = new OkHttpClientFactory() {
@@ -90,6 +96,35 @@ object KubernetesUtils extends Logging {
       }
     }
 
+    debug("Kubernetes client config: " +
+      new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
     new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config)
   }
+
+  implicit private class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder)
+    extends AnyVal {
+
+    def withOption[T](option: Option[T])(configurator: ((T, ConfigBuilder) => ConfigBuilder))
+        : ConfigBuilder = {
+      option.map { opt =>
+        configurator(opt, configBuilder)
+      }.getOrElse(configBuilder)
+    }
+  }
+
+  def requireNandDefined(opt1: Option[_], opt2: Option[_], errMessage: String): Unit = {
+    opt1.foreach { _ => require(opt2.isEmpty, errMessage) }
+    opt2.foreach { _ => require(opt1.isEmpty, errMessage) }
+  }
+
+  def masterAddress(conf: KyuubiConf): String = {
+    conf.get(KUBERNETES_MASTER).getOrElse {
+      // if user not set kubernetes master
+      // find kubernetes master which run this kyuubi pod
+      // set null when kyuubi not in pod
+      // (tcp://XXX:XXX) => (https://XXX:XXX)
+      debug("Try find kubernetes master address from env")
+      sys.env.get("KUBERNETES_PORT").map(_.replace("tcp", "https")).orNull
+    }
+  }
 }

From b56829c5b4f92ce9da74a96689289147b86ae729 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 13:45:07 +0800
Subject: [PATCH 06/15] fix helm

---
 docker/helm/templates/kyuubi-configmap.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/helm/templates/kyuubi-configmap.yaml b/docker/helm/templates/kyuubi-configmap.yaml
index 68b3a44f812..1cfb37e3ee9 100644
--- a/docker/helm/templates/kyuubi-configmap.yaml
+++ b/docker/helm/templates/kyuubi-configmap.yaml
@@ -47,5 +47,6 @@ data:
     #
     kyuubi.frontend.bind.host={{ .Values.server.bind.host }}
     kyuubi.frontend.bind.port={{ .Values.server.bind.port }}
+    kyuubi.kubernetes.namespace= {{ .Release.Namespace }}
 
     # Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html
\ No newline at end of file

From 7b42133ce687aac7e768c1a0fe2da54528815670 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 15:40:29 +0800
Subject: [PATCH 07/15] add doc

---
 .../org/apache/kyuubi/config/KyuubiConf.scala | 25 +++++++++++++------
 .../KubernetesApplicationOperation.scala      | 12 +++++----
 .../apache/kyuubi/util/KubernetesUtils.scala  |  8 +++---
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 9509c9d404b..d016a7150af 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -838,48 +838,59 @@ object KyuubiConf {
 
   val KUBERNETES_NAMESPACE: ConfigEntry[String] =
     buildConf("kyuubi.kubernetes.namespace")
+      .doc("The namespace that will be used for running the kyuubi pods and find engines.")
       .version("1.7.0")
       .stringConf
       .createWithDefault("default")
 
   val KUBERNETES_MASTER: OptionalConfigEntry[String] =
     buildConf("kyuubi.kubernetes.master.address")
-      .doc("kubernetes master address for build kubernetes client")
+      .doc("The internal Kubernetes master (API server) address to be used for kyuubi.")
       .version("1.7.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE: OptionalConfigEntry[String] =
     buildConf("kyuubi.kubernetes.authenticate.oauthTokenFile")
-      .doc("kubernetes client authenticate oauth token file")
+      .doc("Path to the file containing the OAuth token to use when authenticating against " +
+        "the Kubernetes API server. Specify this as a path as opposed to a URI " +
+        "(i.e. do not provide a scheme)")
       .version("1.7.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_AUTHENTICATE_OAUTH_TOKEN: OptionalConfigEntry[String] =
     buildConf("kyuubi.kubernetes.authenticate.oauthToken")
-      .doc("kubernetes client authenticate oauth token value")
+      .doc("The OAuth token to use when authenticating against the Kubernetes API server. " +
+        "Note that unlike the other authentication options, this must be the exact string value " +
+        "of the token to use for the authentication.")
       .version("1.7.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_AUTHENTICATE_CLIENT_KEY_FILE: OptionalConfigEntry[String] =
     buildConf("kyuubi.kubernetes.authenticate.clientKeyFile")
-      .doc("kubernetes client authenticate client key file")
+      .doc("Path to the client key file for connecting to the Kubernetes API server " +
+        "over TLS from the kyuubi. Specify this as a path as opposed to a URI " +
+        "(i.e. do not provide a scheme)")
       .version("1.7.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_AUTHENTICATE_CLIENT_CERT_FILE: OptionalConfigEntry[String] =
     buildConf("kyuubi.kubernetes.authenticate.clientCertFile")
-      .doc("kubernetes client authenticate client cert file")
+      .doc("Path to the client cert file for connecting to the Kubernetes API server " +
+        "over TLS from the kyuubi. Specify this as a path as opposed to a URI " +
+        "(i.e. do not provide a scheme)")
       .version("1.7.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_AUTHENTICATE_CA_CERT_FILE: OptionalConfigEntry[String] =
-    buildConf("kyuubi.kubernetes.authenticate.ca cert file")
-      .doc("kubernetes client authenticate ca cert file")
+    buildConf("kyuubi.kubernetes.authenticate.caCertFile")
+      .doc("Path to the CA cert file for connecting to the Kubernetes API server " +
+        "over TLS from the kyuubi. Specify this as a path as opposed to a URI " +
+        "(i.e. do not provide a scheme)")
       .version("1.7.0")
       .stringConf
       .createOptional
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index 062535557e4..a2d680ab216 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -38,11 +38,13 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
     jpsOperation.initialize(conf)
 
     info("Start initializing Kubernetes Client.")
-    kubernetesClient = KubernetesUtils.buildKubernetesClient(conf)
-    if (kubernetesClient == null) {
-      warn("Fail to init Kubernetes Client for Kubernetes Application Operation")
-    } else {
-      info(s"Initialized Kubernetes Client connect to: ${kubernetesClient.getMasterUrl}")
+    kubernetesClient = KubernetesUtils.buildKubernetesClient(conf) match {
+      case Some(client) =>
+        info(s"Initialized Kubernetes Client connect to: ${client.getMasterUrl}")
+        client
+      case None =>
+        warn("Fail to init Kubernetes Client for Kubernetes Application Operation")
+        null
     }
   }
 
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index 095c834f791..59c51089afb 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -29,15 +29,15 @@ import okhttp3.{Dispatcher, OkHttpClient}
 
 import org.apache.kyuubi.Logging
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.{KUBERNETES_AUTHENTICATE_CA_CERT_FILE, KUBERNETES_AUTHENTICATE_CLIENT_CERT_FILE, KUBERNETES_AUTHENTICATE_CLIENT_KEY_FILE, KUBERNETES_AUTHENTICATE_OAUTH_TOKEN, KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE, KUBERNETES_CONTEXT, KUBERNETES_MASTER, KUBERNETES_NAMESPACE, KUBERNETES_TRUST_CERTIFICATES}
+import org.apache.kyuubi.config.KyuubiConf._
 
 object KubernetesUtils extends Logging {
 
-  def buildKubernetesClient(conf: KyuubiConf): KubernetesClient = {
+  def buildKubernetesClient(conf: KyuubiConf): Option[KubernetesClient] = {
     val master = masterAddress(conf)
     if (master == null || master.isEmpty) {
       warn("Need set kubernetes master url, if you want to set up kubernetes client")
-      return null
+      return None
     }
 
     val namespace = conf.get(KUBERNETES_NAMESPACE)
@@ -98,7 +98,7 @@ object KubernetesUtils extends Logging {
 
     debug("Kubernetes client config: " +
       new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
-    new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config)
+    Some(new DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config), config))
   }
 
   implicit private class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder)

From 683cc260fcb1aa28708f2b80db933c05bc8199c1 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 17:16:12 +0800
Subject: [PATCH 08/15] context provided master

---
 .../org/apache/kyuubi/util/KubernetesUtils.scala | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index 59c51089afb..3a93794cc17 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -34,12 +34,6 @@ import org.apache.kyuubi.config.KyuubiConf._
 object KubernetesUtils extends Logging {
 
   def buildKubernetesClient(conf: KyuubiConf): Option[KubernetesClient] = {
-    val master = masterAddress(conf)
-    if (master == null || master.isEmpty) {
-      warn("Need set kubernetes master url, if you want to set up kubernetes client")
-      return None
-    }
-
     val namespace = conf.get(KUBERNETES_NAMESPACE)
     val serviceAccountToken =
       Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)).filter(_.exists)
@@ -71,7 +65,9 @@ object KubernetesUtils extends Logging {
 
     val config = new ConfigBuilder(autoConfigure(kubeContext.orNull))
       .withApiVersion("v1")
-      .withMasterUrl(master)
+      .withOption(Some(masterAddress(conf))) {
+        (master, configBuilder) => configBuilder.withMasterUrl(master)
+      }
       .withNamespace(namespace)
       .withTrustCerts(conf.get(KUBERNETES_TRUST_CERTIFICATES))
       .withOption(oauthTokenValue) {
@@ -87,6 +83,12 @@ object KubernetesUtils extends Logging {
         (file, configBuilder) => configBuilder.withClientCertFile(file)
       }.build()
 
+    // kyuubi need context or config set kubernetes master address
+    if (config.getMasterUrl == null || config.getMasterUrl.isEmpty) {
+      warn("Need set kubernetes master url, if you want to set up kubernetes client")
+      return None
+    }
+
     // https://github.com/fabric8io/kubernetes-client/issues/3547
     val dispatcher = new Dispatcher(
       ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))

From e6826609f21e78559d2c3105d2d00ee68ecc0464 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 17:31:08 +0800
Subject: [PATCH 09/15] master

---
 .../apache/kyuubi/util/KubernetesUtils.scala  | 31 +++++++++----------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index 3a93794cc17..1d17fcc5e3b 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -34,6 +34,7 @@ import org.apache.kyuubi.config.KyuubiConf._
 object KubernetesUtils extends Logging {
 
   def buildKubernetesClient(conf: KyuubiConf): Option[KubernetesClient] = {
+    val master = conf.get(KUBERNETES_MASTER)
     val namespace = conf.get(KUBERNETES_NAMESPACE)
     val serviceAccountToken =
       Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)).filter(_.exists)
@@ -65,10 +66,9 @@ object KubernetesUtils extends Logging {
 
     val config = new ConfigBuilder(autoConfigure(kubeContext.orNull))
       .withApiVersion("v1")
-      .withOption(Some(masterAddress(conf))) {
+      .withOption(master) {
         (master, configBuilder) => configBuilder.withMasterUrl(master)
-      }
-      .withNamespace(namespace)
+      }.withNamespace(namespace)
       .withTrustCerts(conf.get(KUBERNETES_TRUST_CERTIFICATES))
       .withOption(oauthTokenValue) {
         (token, configBuilder) => configBuilder.withOauthToken(token)
@@ -83,10 +83,10 @@ object KubernetesUtils extends Logging {
         (file, configBuilder) => configBuilder.withClientCertFile(file)
       }.build()
 
-    // kyuubi need context or config set kubernetes master address
-    if (config.getMasterUrl == null || config.getMasterUrl.isEmpty) {
-      warn("Need set kubernetes master url, if you want to set up kubernetes client")
-      return None
+    (master, kubeContext, loadMasterAddressFromEnv) match {
+      case (None, None, Some(url)) =>
+        debug(s"Set kubernetes master address $url from env KUBERNETES_PORT")
+        config.setMasterUrl(url)
     }
 
     // https://github.com/fabric8io/kubernetes-client/issues/3547
@@ -119,14 +119,13 @@ object KubernetesUtils extends Logging {
     opt2.foreach { _ => require(opt1.isEmpty, errMessage) }
   }
 
-  def masterAddress(conf: KyuubiConf): String = {
-    conf.get(KUBERNETES_MASTER).getOrElse {
-      // if user not set kubernetes master
-      // find kubernetes master which run this kyuubi pod
-      // set null when kyuubi not in pod
-      // (tcp://XXX:XXX) => (https://XXX:XXX)
-      debug("Try find kubernetes master address from env")
-      sys.env.get("KUBERNETES_PORT").map(_.replace("tcp", "https")).orNull
-    }
+  /**
+   * if user not set kubernetes master
+   * find kubernetes master which run this kyuubi pod
+   * set null when kyuubi not in pod
+   * (tcp://XXX:XXX) => (https://XXX:XXX)
+   */
+  def loadMasterAddressFromEnv: Option[String] = {
+    sys.env.get("KUBERNETES_PORT").map(_.replace("tcp", "https"))
   }
 }

From 75380d134221812b6157583246373d12badd6c14 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 17:32:19 +0800
Subject: [PATCH 10/15] comment

---
 .../main/scala/org/apache/kyuubi/util/KubernetesUtils.scala    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index 1d17fcc5e3b..ef8c58a10f5 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -120,9 +120,8 @@ object KubernetesUtils extends Logging {
   }
 
   /**
-   * if user not set kubernetes master
+   * if can't load master address from config or context
    * find kubernetes master which run this kyuubi pod
-   * set null when kyuubi not in pod
    * (tcp://XXX:XXX) => (https://XXX:XXX)
    */
   def loadMasterAddressFromEnv: Option[String] = {

From ddc9a2b2cc6b3278e709206d1d25212e5cd1113e Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Fri, 21 Oct 2022 17:36:53 +0800
Subject: [PATCH 11/15] fix scala

---
 .../src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index ef8c58a10f5..c17ead6cd3a 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -87,6 +87,8 @@ object KubernetesUtils extends Logging {
       case (None, None, Some(url)) =>
         debug(s"Set kubernetes master address $url from env KUBERNETES_PORT")
         config.setMasterUrl(url)
+      case _ =>
+        // do noting
     }
 
     // https://github.com/fabric8io/kubernetes-client/issues/3547

From 943c68dd2a85c60da8362b1f9a907b19c99e05d6 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Mon, 24 Oct 2022 10:35:32 +0800
Subject: [PATCH 12/15] add unit test

---
 docker/helm/templates/kyuubi-configmap.yaml   |  2 +-
 .../kubernetes/test/KubernetesUtilsTest.scala | 49 +++++++++++++++++++
 .../apache/kyuubi/util/KubernetesUtils.scala  | 17 -------
 3 files changed, 50 insertions(+), 18 deletions(-)
 create mode 100644 integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala

diff --git a/docker/helm/templates/kyuubi-configmap.yaml b/docker/helm/templates/kyuubi-configmap.yaml
index 1cfb37e3ee9..4b8e920d8fd 100644
--- a/docker/helm/templates/kyuubi-configmap.yaml
+++ b/docker/helm/templates/kyuubi-configmap.yaml
@@ -49,4 +49,4 @@ data:
     kyuubi.frontend.bind.port={{ .Values.server.bind.port }}
     kyuubi.kubernetes.namespace= {{ .Release.Namespace }}
 
-    # Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html
\ No newline at end of file
+    # Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala
new file mode 100644
index 00000000000..04dd745873e
--- /dev/null
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.kubernetes.test
+
+import io.fabric8.kubernetes.client.Config
+
+import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.{KUBERNETES_CONTEXT, KUBERNETES_MASTER}
+import org.apache.kyuubi.util.KubernetesUtils
+
+class KubernetesUtilsTest extends KyuubiFunSuite {
+
+  test("Test kubernetesUtils build Kubernetes client") {
+    val testMaster = "https://localhost:12345/"
+    System.setProperty(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY, testMaster)
+    val conf = KyuubiConf()
+    val client1 = KubernetesUtils.buildKubernetesClient(conf)
+    assert(client1.nonEmpty && client1.get.getMasterUrl.toString.equals(testMaster))
+
+    // start up minikube
+    MiniKube.getIp
+    conf.set(KUBERNETES_CONTEXT.key, "minikube")
+    val client2 = KubernetesUtils.buildKubernetesClient(conf)
+    assert(client2.nonEmpty && client2.get.getMasterUrl.equals(
+      MiniKube.getKubernetesClient.getMasterUrl))
+
+    // user set master uri should replace uri in context
+    val master = "https://kyuubi-test:8443/"
+    conf.set(KUBERNETES_MASTER.key, master)
+    val client3 = KubernetesUtils.buildKubernetesClient(conf)
+    assert(client3.nonEmpty && client3.get.getMasterUrl.toString.equals(master))
+  }
+}
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index c17ead6cd3a..170292fc8b5 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -83,14 +83,6 @@ object KubernetesUtils extends Logging {
         (file, configBuilder) => configBuilder.withClientCertFile(file)
       }.build()
 
-    (master, kubeContext, loadMasterAddressFromEnv) match {
-      case (None, None, Some(url)) =>
-        debug(s"Set kubernetes master address $url from env KUBERNETES_PORT")
-        config.setMasterUrl(url)
-      case _ =>
-        // do noting
-    }
-
     // https://github.com/fabric8io/kubernetes-client/issues/3547
     val dispatcher = new Dispatcher(
       ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
@@ -120,13 +112,4 @@ object KubernetesUtils extends Logging {
     opt1.foreach { _ => require(opt2.isEmpty, errMessage) }
     opt2.foreach { _ => require(opt1.isEmpty, errMessage) }
   }
-
-  /**
-   * if can't load master address from config or context
-   * find kubernetes master which run this kyuubi pod
-   * (tcp://XXX:XXX) => (https://XXX:XXX)
-   */
-  def loadMasterAddressFromEnv: Option[String] = {
-    sys.env.get("KUBERNETES_PORT").map(_.replace("tcp", "https"))
-  }
 }

From fb28ea7ab71c74481f185178a9b0a136871495f1 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Mon, 24 Oct 2022 11:13:26 +0800
Subject: [PATCH 13/15] add ci test

---
 .github/workflows/master.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index bccfec36c69..46504e085a2 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -356,7 +356,7 @@ jobs:
           ./build/mvn ${MVN_OPT} clean install
           -pl integration-tests/kyuubi-kubernetes-it -am
           -Pkubernetes-it
-          -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment
+          -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.deployment,org.apache.kyuubi.kubernetes.test.KubernetesUtilsTest
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v2

From ce408db080341d317c13f4f25a416e493a5357b9 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Mon, 24 Oct 2022 14:06:29 +0800
Subject: [PATCH 14/15] fix test

---
 docker/helm/templates/kyuubi-configmap.yaml   |  2 +-
 docs/deployment/settings.md                   | 14 +++++------
 .../apache/kyuubi/util/KubernetesUtils.scala  | 25 +++++++++----------
 3 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/docker/helm/templates/kyuubi-configmap.yaml b/docker/helm/templates/kyuubi-configmap.yaml
index 4b8e920d8fd..b4fd82d4d27 100644
--- a/docker/helm/templates/kyuubi-configmap.yaml
+++ b/docker/helm/templates/kyuubi-configmap.yaml
@@ -47,6 +47,6 @@ data:
     #
     kyuubi.frontend.bind.host={{ .Values.server.bind.host }}
     kyuubi.frontend.bind.port={{ .Values.server.bind.port }}
-    kyuubi.kubernetes.namespace= {{ .Release.Namespace }}
+    kyuubi.kubernetes.namespace={{ .Release.Namespace }}
 
     # Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index ceeb7a0b6f7..30f2c4673f2 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -369,14 +369,14 @@ kyuubi.kinit.principal|&lt;undefined&gt;|Name of the Kerberos principal.|string|
 
 Key | Default | Meaning | Type | Since
 --- | --- | --- | --- | ---
-kyuubi.kubernetes.authenticate.ca cert file|&lt;undefined&gt;|kubernetes client authenticate ca cert file|string|1.7.0
-kyuubi.kubernetes.authenticate.clientCertFile|&lt;undefined&gt;|kubernetes client authenticate client cert file|string|1.7.0
-kyuubi.kubernetes.authenticate.clientKeyFile|&lt;undefined&gt;|kubernetes client authenticate client key file|string|1.7.0
-kyuubi.kubernetes.authenticate.oauthToken|&lt;undefined&gt;|kubernetes client authenticate oauth token value|string|1.7.0
-kyuubi.kubernetes.authenticate.oauthTokenFile|&lt;undefined&gt;|kubernetes client authenticate oauth token file|string|1.7.0
+kyuubi.kubernetes.authenticate.caCertFile|&lt;undefined&gt;|Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0
+kyuubi.kubernetes.authenticate.clientCertFile|&lt;undefined&gt;|Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0
+kyuubi.kubernetes.authenticate.clientKeyFile|&lt;undefined&gt;|Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0
+kyuubi.kubernetes.authenticate.oauthToken|&lt;undefined&gt;|The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike the other authentication options, this must be the exact string value of the token to use for the authentication.|string|1.7.0
+kyuubi.kubernetes.authenticate.oauthTokenFile|&lt;undefined&gt;|Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme)|string|1.7.0
 kyuubi.kubernetes.context|&lt;undefined&gt;|The desired context from your kubernetes config file used to configure the K8S client for interacting with the cluster.|string|1.6.0
-kyuubi.kubernetes.master.address|&lt;undefined&gt;|kubernetes master address for build kubernetes client|string|1.7.0
-kyuubi.kubernetes.namespace|default||string|1.7.0
+kyuubi.kubernetes.master.address|&lt;undefined&gt;|The internal Kubernetes master (API server) address to be used for kyuubi.|string|1.7.0
+kyuubi.kubernetes.namespace|default|The namespace that will be used for running the kyuubi pods and find engines.|string|1.7.0
 kyuubi.kubernetes.trust.certificates|false|If set to true then client can submit to kubernetes cluster only with token|boolean|1.7.0
 
 
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
index 170292fc8b5..921aa04ae3c 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/util/KubernetesUtils.scala
@@ -66,21 +66,20 @@ object KubernetesUtils extends Logging {
 
     val config = new ConfigBuilder(autoConfigure(kubeContext.orNull))
       .withApiVersion("v1")
-      .withOption(master) {
-        (master, configBuilder) => configBuilder.withMasterUrl(master)
+      .withOption(master) { (master, configBuilder) =>
+        configBuilder.withMasterUrl(master)
       }.withNamespace(namespace)
       .withTrustCerts(conf.get(KUBERNETES_TRUST_CERTIFICATES))
-      .withOption(oauthTokenValue) {
-        (token, configBuilder) => configBuilder.withOauthToken(token)
-      }.withOption(oauthTokenFile) {
-        (file, configBuilder) =>
-          configBuilder.withOauthToken(Files.asCharSource(file, Charsets.UTF_8).read())
-      }.withOption(caCertFile) {
-        (file, configBuilder) => configBuilder.withCaCertFile(file)
-      }.withOption(clientKeyFile) {
-        (file, configBuilder) => configBuilder.withClientKeyFile(file)
-      }.withOption(clientCertFile) {
-        (file, configBuilder) => configBuilder.withClientCertFile(file)
+      .withOption(oauthTokenValue) { (token, configBuilder) =>
+        configBuilder.withOauthToken(token)
+      }.withOption(oauthTokenFile) { (file, configBuilder) =>
+        configBuilder.withOauthToken(Files.asCharSource(file, Charsets.UTF_8).read())
+      }.withOption(caCertFile) { (file, configBuilder) =>
+        configBuilder.withCaCertFile(file)
+      }.withOption(clientKeyFile) { (file, configBuilder) =>
+        configBuilder.withClientKeyFile(file)
+      }.withOption(clientCertFile) { (file, configBuilder) =>
+        configBuilder.withClientCertFile(file)
       }.build()
 
     // https://github.com/fabric8io/kubernetes-client/issues/3547

From 0a4c8c7955b6ec06070df753ab7f733776bff958 Mon Sep 17 00:00:00 2001
From: zwangsheng <2213335496@qq.com>
Date: Mon, 24 Oct 2022 18:48:54 +0800
Subject: [PATCH 15/15] clear properties

---
 .../kubernetes/test/KubernetesUtilsTest.scala | 31 ++++++++++---------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala
index 04dd745873e..e6b06f6b710 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesUtilsTest.scala
@@ -28,22 +28,23 @@ class KubernetesUtilsTest extends KyuubiFunSuite {
 
   test("Test kubernetesUtils build Kubernetes client") {
     val testMaster = "https://localhost:12345/"
-    System.setProperty(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY, testMaster)
-    val conf = KyuubiConf()
-    val client1 = KubernetesUtils.buildKubernetesClient(conf)
-    assert(client1.nonEmpty && client1.get.getMasterUrl.toString.equals(testMaster))
+    withSystemProperty(Map(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY -> testMaster)) {
+      val conf = KyuubiConf()
+      val client1 = KubernetesUtils.buildKubernetesClient(conf)
+      assert(client1.nonEmpty && client1.get.getMasterUrl.toString.equals(testMaster))
 
-    // start up minikube
-    MiniKube.getIp
-    conf.set(KUBERNETES_CONTEXT.key, "minikube")
-    val client2 = KubernetesUtils.buildKubernetesClient(conf)
-    assert(client2.nonEmpty && client2.get.getMasterUrl.equals(
-      MiniKube.getKubernetesClient.getMasterUrl))
+      // start up minikube
+      MiniKube.getIp
+      conf.set(KUBERNETES_CONTEXT.key, "minikube")
+      val client2 = KubernetesUtils.buildKubernetesClient(conf)
+      assert(client2.nonEmpty && client2.get.getMasterUrl.equals(
+        MiniKube.getKubernetesClient.getMasterUrl))
 
-    // user set master uri should replace uri in context
-    val master = "https://kyuubi-test:8443/"
-    conf.set(KUBERNETES_MASTER.key, master)
-    val client3 = KubernetesUtils.buildKubernetesClient(conf)
-    assert(client3.nonEmpty && client3.get.getMasterUrl.toString.equals(master))
+      // user set master uri should replace uri in context
+      val master = "https://kyuubi-test:8443/"
+      conf.set(KUBERNETES_MASTER.key, master)
+      val client3 = KubernetesUtils.buildKubernetesClient(conf)
+      assert(client3.nonEmpty && client3.get.getMasterUrl.toString.equals(master))
+    }
   }
 }