From 72e5f44c7ea61912835abe8f45c6d7922545f456 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Sun, 13 Mar 2022 17:56:43 +0800 Subject: [PATCH] K8S IT in GA --- .github/workflows/build_and_test.yml | 59 ++++++++++++++++++- .../k8s/integrationtest/KubernetesSuite.scala | 6 ++ .../k8s/integrationtest/TestConstants.scala | 2 + 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ebe17b5963f20..d7a2146881012 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -97,7 +97,7 @@ jobs: else echo '::set-output name=java::8' echo '::set-output name=branch::master' # Default branch to run on. CHANGE here when a branch is cut out. - echo '::set-output name=type::regular' + echo '::set-output name=type::k8s' echo '::set-output name=envs::{"SPARK_ANSI_SQL_MODE": "${{ inputs.ansi_enabled }}"}' echo '::set-output name=hadoop::hadoop3' fi @@ -768,6 +768,63 @@ jobs: name: unit-tests-log-tpcds--8-${{ needs.configure-jobs.outputs.hadoop }}-hive2.3 path: "**/target/unit-tests.log" + k8s-integration-tests: + needs: [configure-jobs, precondition] + if: needs.configure-jobs.outputs.type == 'k8s' && fromJson(needs.precondition.outputs.required).build == 'true' + name: Run K8S integration tests + runs-on: ubuntu-20.04 + env: + SPARK_LOCAL_IP: localhost + steps: + - name: Checkout Spark repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: apache/spark + ref: master + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + run: | + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" + - name: Cache Scala, SBT and Maven + uses: actions/cache@v2 + with: + path: | + build/apache-maven-* + build/scala-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Coursier local repository + uses: actions/cache@v2 + with: + path: ~/.cache/coursier + key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + k8s-integration-coursier- + - name: Install Java 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: start minikube + run: | + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube start --cpus 2 --memory 6144 + - name: Show all K8S pods and nodes + run: | + kubectl get pods -A + kubectl get nodes -oyaml + - name: Run K8S integration test + run: | + kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true + eval $(minikube docker-env) + build/sbt -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.driver.cpu=0.5 -Dspark.kubernetes.test.executor.cpu=0.2 -Dtest.exclude.tags=minikube,r,local "kubernetes-integration-tests/test" + docker-integration-tests: needs: [configure-jobs, precondition] if: needs.configure-jobs.outputs.type == 'regular' && fromJson(needs.precondition.outputs.required).docker == 'true' diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 685149f09d72f..4ae04f882b95e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -192,6 +192,12 @@ class KubernetesSuite extends SparkFunSuite .set("spark.kubernetes.driver.label.spark-app-locator", appLocator) .set("spark.kubernetes.executor.label.spark-app-locator", appLocator) .set(NETWORK_AUTH_ENABLED.key, "true") + sys.props.get(CONFIG_DRIVER_REQUEST_CPU).map { cpu => + sparkAppConf.set("spark.kubernetes.driver.request.cores", cpu) + } + sys.props.get(CONFIG_EXECUTOR_REQUEST_CPU).map { cpu => + sparkAppConf.set("spark.kubernetes.executor.request.cores", cpu) + } if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { kubernetesTestComponents.createNamespace() } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index c46839f1dffcc..4950e471cd6c4 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -34,4 +34,6 @@ object TestConstants { val CONFIG_KEY_IMAGE_TAG_FILE = "spark.kubernetes.test.imageTagFile" val CONFIG_KEY_IMAGE_REPO = "spark.kubernetes.test.imageRepo" val CONFIG_KEY_UNPACK_DIR = "spark.kubernetes.test.unpackSparkDir" + val CONFIG_DRIVER_REQUEST_CPU = "spark.kubernetes.test.driver.cpu" + val CONFIG_EXECUTOR_REQUEST_CPU = "spark.kubernetes.test.executor.cpu" }