diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 748664cf41b74..af0b1ec3dc76c 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -268,6 +268,22 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro + + spark.kubernetes.test.driverRequestCores + + Set cpu resource for each driver pod in test, this is currently only for test on cpu resource limited cluster, + it's not recommended for other scenarios. + + + + + spark.kubernetes.test.executorRequestCores + + Set cpu resource for each executor pod in test, this is currently only for test on cpu resource limited cluster, + it's not recommended for other scenarios. + + + # Running the Kubernetes Integration Tests with SBT diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 3d7a9313031b0..041b2886c4174 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -192,6 +192,12 @@ class KubernetesSuite extends SparkFunSuite .set("spark.kubernetes.driver.label.spark-app-locator", appLocator) .set("spark.kubernetes.executor.label.spark-app-locator", appLocator) .set(NETWORK_AUTH_ENABLED.key, "true") + sys.props.get(CONFIG_DRIVER_REQUEST_CORES).map { cpu => + sparkAppConf.set("spark.kubernetes.driver.request.cores", cpu) + } + sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).map { cpu => + sparkAppConf.set("spark.kubernetes.executor.request.cores", cpu) + } if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { kubernetesTestComponents.createNamespace() } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index c46839f1dffcc..2175d23d44977 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -34,4 +34,6 @@ object TestConstants { val CONFIG_KEY_IMAGE_TAG_FILE = "spark.kubernetes.test.imageTagFile" val CONFIG_KEY_IMAGE_REPO = "spark.kubernetes.test.imageRepo" val CONFIG_KEY_UNPACK_DIR = "spark.kubernetes.test.unpackSparkDir" + val CONFIG_DRIVER_REQUEST_CORES = "spark.kubernetes.test.driverRequestCores" + val CONFIG_EXECUTOR_REQUEST_CORES = "spark.kubernetes.test.executorRequestCores" }