From 50b6792e8bd45101cb6b66db0147a97083f471e6 Mon Sep 17 00:00:00 2001 From: huangtianhua Date: Mon, 23 Sep 2019 16:25:16 +0800 Subject: [PATCH] Run python tests --- .zuul.yaml | 14 +++++ .../spark-build/run_python_tests.yaml | 53 +++++++++++++++++++ common/kvstore/pom.xml | 3 +- common/network-common/pom.xml | 2 +- pom.xml | 9 +++- .../mllib/tests/test_streaming_algorithms.py | 8 +-- 6 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 .zuul.yaml create mode 100644 .zuul/playbooks/spark-build/run_python_tests.yaml diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000000000..74308e11f433f --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,14 @@ +- project: + name: theopenlab/spark + check: + jobs: + - spark-build-and-python-test-arm64 + +- job: + name: spark-build-and-python-test-arm64 + parent: init-test + description: | + The spark build and test other modules in openlab cluster. + run: .zuul/playbooks/spark-build/run_python_tests.yaml + nodeset: ubuntu-xenial-arm64 + timeout: 86400 diff --git a/.zuul/playbooks/spark-build/run_python_tests.yaml b/.zuul/playbooks/spark-build/run_python_tests.yaml new file mode 100644 index 0000000000000..b446294ff698c --- /dev/null +++ b/.zuul/playbooks/spark-build/run_python_tests.yaml @@ -0,0 +1,53 @@ +- hosts: all + tasks: + - name: Build spark master using mvn with hadoop 2.7 + shell: + cmd: | + set -exo pipefail + sudo apt-get update -y + + # Install java + sudo apt-get install default-jre -y + sudo apt-get install default-jdk -y + java_home=$(dirname $(dirname $(update-alternatives --list javac))) + echo "export JAVA_HOME=${java_home}" >> ~/.profile + echo "export PATH=${java_home}/bin:$PATH" >> ~/.profile + source ~/.profile + + # Install maven + wget http://www.us.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz + tar -xvf apache-maven-3.6.2-bin.tar.gz + export PATH=$PWD/apache-maven-3.6.2/bin:$PATH + + # fix kafka authfail tests + sudo sed -i "s|127.0.0.1 $(hostname) localhost|127.0.0.1 localhost $(hostname)|" /etc/hosts + + cd {{ ansible_user_dir }}/{{ zuul.project.src_dir }} + + ./build/mvn install -DskipTests -Phadoop-2.7 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos + + # use leveldbjni arm supporting jar + wget https://repo1.maven.org/maven2/org/openlabtesting/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar + mvn install:install-file -DgroupId=org.fusesource.leveldbjni -DartifactId=leveldbjni-all -Dversion=1.8 -Dpackaging=jar -Dfile=leveldbjni-all-1.8.jar + + # install python3.6 + sudo add-apt-repository ppa:jonathonf/python-3.6 -y + sudo apt-get update -y + sudo apt-get install python3.6 -y + sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 + sudo apt-get install python3.6-dev -y + + # install pip(pip3) + curl https://bootstrap.pypa.io/get-pip.py | sudo python3.6 + + # install packages needed + sudo pip2 install coverage numpy + sudo pip install coverage numpy + + sleep 36000 + # run python tests + python/run-tests --python-executables=python2.7,python3.6 + + chdir: '/home/zuul/src' + executable: /bin/bash + environment: '{{ global_env }}' \ No newline at end of file diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index f042a12fda3d2..705061620f107 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -45,8 +45,9 @@ guava - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all + 1.8 com.fasterxml.jackson.core diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index 2ee17800c10e4..163c250054e4d 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -52,7 +52,7 @@ - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all 1.8 diff --git a/pom.xml b/pom.xml index bc8595a1f4ef7..6d7262bc9c22f 100644 --- a/pom.xml +++ b/pom.xml @@ -241,6 +241,7 @@ ${session.executionRootDirectory} 1g + org.fusesource.leveldbjni @@ -527,7 +528,7 @@ ${commons.httpcore.version} - org.fusesource.leveldbjni + ${leveldbjni.group} leveldbjni-all 1.8 @@ -3073,5 +3074,11 @@ sparkr + diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py index 6f098f4582930..97197522c8a9d 100644 --- a/python/pyspark/mllib/tests/test_streaming_algorithms.py +++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py @@ -33,14 +33,14 @@ class MLLibStreamingTestCase(unittest.TestCase): def setUp(self): self.sc = SparkContext('local[4]', "MLlib tests") - self.ssc = StreamingContext(self.sc, 1.0) + self.ssc = StreamingContext(self.sc, 3.0) def tearDown(self): self.ssc.stop(False) self.sc.stop() @staticmethod - def _eventually(condition, timeout=30.0, catch_assertions=False): + def _eventually(condition, timeout=120.0, catch_assertions=False): """ Wait a given amount of time for a condition to pass, else fail with an error. This is a helper utility for streaming ML tests. @@ -289,7 +289,7 @@ def condition(): return True # We want all batches to finish for this test. - self._eventually(condition, 60.0, catch_assertions=True) + self._eventually(condition, catch_assertions=True) t_models = array(models) diff = t_models[1:] - t_models[:-1] @@ -364,7 +364,7 @@ def condition(): return True return "Latest errors: " + ", ".join(map(lambda x: str(x), errors)) - self._eventually(condition, timeout=60.0) + self._eventually(condition) class StreamingLinearRegressionWithTests(MLLibStreamingTestCase):