Merge branch 'master' into master

mars-project · Sep 9, 2021 · 7e0c9d9 · 7e0c9d9
2 parents ff2b4f7 + 4db786c
commit 7e0c9d9
Show file tree

Hide file tree

Showing 463 changed files with 14,546 additions and 2,303 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,6 +3,8 @@
 .git
 .github
 .idea
+.empty
+.readthedocs.yml
 .vscode
 .pytest*
 .dist-coverage
@@ -16,10 +18,13 @@ misc
 .DS_Store
 .flake8
 .gitignore
-.travis.yml
-appveyor.yml
+.codacy.yml
+.codecov.yml
+azure-pipelines.yml
 *.iml
+*.md
 conda-spec.txt
+CODEOWNERS
 CONTRIBUTING.rst
 LICENSE
 MANIFEST.in

diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,18 @@
+* text=auto
+# enforce text on certain files
+*.py text
+*.pyx text
+*.pyi text
+*.pxd text
+*.c text
+*.cpp text
+*.h text
+*.hpp text
+*.html text
+*.csv text
+*.js text
+*.jsx text
+*.ts text
+*.json text
+*.yaml text
+*.yml text
diff --git a/.github/workflows/cancel-prev.yml b/.github/workflows/cancel-prev.yml
@@ -9,4 +9,4 @@ jobs:
       - uses: styfle/[email protected]
         with:
           access_token: ${{ github.token }}
-          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml,checks.yml
+          workflow_id: core-ci.yml,os-compat-ci.yml,platform-ci.yml
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml
@@ -51,7 +51,7 @@ jobs:
         source ./ci/rewrite-cov-config.sh
 
         pip install git+https://github.com/mars-project/pytest-asyncio.git
-        pip install numpy scipy cython
+        pip install numpy scipy cython oss2
         pip install -e ".[dev,extra]"
         pip install virtualenv flaky
 
@@ -63,20 +63,7 @@ jobs:
           fi
 
           conda install -n test --quiet --yes -c pkgs/main python=$PYTHON certifi
-
-          if [[ ! "$PYTHON" =~ "3.9" ]]; then
-            conda install --quiet --yes -n test -c conda-forge --no-deps python=$PYTHON \
-              libxgboost py-xgboost xgboost lightgbm tensorflow
-          fi
-          if [[ "$PYTHON" =~ "3.6" ]]; then
-            pip install torch==1.4.0 torchvision==0.5.0 faiss-cpu fastparquet
-          fi
-          if [[ ! "$PYTHON" =~ "3.6" ]] && [[ ! "$PYTHON" =~ "3.9" ]]; then
-            pip install torch torchvision
-            pip install statsmodels tsfresh
-          fi
         fi
-        retry ./.github/workflows/download-etcd.sh
         conda list -n test
 
     - name: Test with pytest

diff --git a/.github/workflows/docker-cd.yml b/.github/workflows/docker-cd.yml
@@ -0,0 +1,31 @@
+name: Mars CD for DockerHub
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push Docker image
+        shell: bash
+        env:
+          DOCKER_ORG: ${{ secrets.DOCKERHUB_USERNAME }}
+        run: |
+          source ./ci/reload-env.sh
+          if [[ "$DOCKER_ORG" == "marsuploader" ]]; then
+            export DOCKER_ORG="marsproject"
+          fi
+          bash bin/kube-image-tool.sh -o "$DOCKER_ORG" -t "$GIT_TAG" build
+          docker push "$DOCKER_ORG/mars:$GIT_TAG"
diff --git a/.github/workflows/os-compat-ci.yml b/.github/workflows/os-compat-ci.yml
@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macOS-latest, windows-latest]
-        python-version: [3.6, 3.8]
+        python-version: [3.7, 3.8]
 
     steps:
     - name: Check out code
@@ -38,7 +38,7 @@ jobs:
         source ./ci/rewrite-cov-config.sh
 
         pip install git+https://github.com/mars-project/pytest-asyncio.git
-        pip install numpy scipy cython
+        pip install numpy scipy cython oss2
         pip install -e ".[dev,extra]"
         pip install virtualenv flaky
         conda list -n test
@@ -61,16 +61,16 @@ jobs:
             mars/oscar mars/services mars/storage mars/lib mars/tests
           mv .coverage build/.coverage.dist.file
         elif [[ $UNAME == "windows" ]]; then
-          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=5 --timeout=1500 \
+          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=10 --timeout=1500 \
             mars/oscar
           mv .coverage build/.coverage.oscar.file
 
-          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=5 --timeout=1500 \
+          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=10 --timeout=1500 \
             mars/services
           mv .coverage build/.coverage.services.file
 
           # skip mars/tests temporarily, as it may cause test stuck and needs more inspection
-          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=5 --timeout=1500 \
+          python -m pytest $PYTEST_CONFIG --force-flaky --max-runs=10 --timeout=1500 \
             mars/storage mars/lib  # mars/tests
           mv .coverage build/.coverage.misc.file
         fi

diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml
@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8-kubernetes, 3.8-hadoop, 3.8-ray, 3.8-vineyard]
+        python-version: [3.8-kubernetes, 3.8-hadoop, 3.8-ray, 3.8-vineyard, 3.8-dask]
         include:
           - { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
               no-deploy: 1, with-kubernetes: "with Kubernetes" }
@@ -19,6 +19,8 @@ jobs:
              no-deploy: 1, with-vineyard: "with vineyard" }
           - { os: ubuntu-latest, python-version: 3.8-ray, no-common-tests: 1,
               no-deploy: 1, with-ray: "with ray" }
+          - { os: ubuntu-latest, python-version: 3.8-dask, no-common-tests: 1,
+              no-deploy: 1, run-dask: "run dask" }
 
     steps:
       - name: Check out code
@@ -40,6 +42,7 @@ jobs:
           WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
           WITH_VINEYARD: ${{ matrix.with-vineyard }}
           WITH_RAY: ${{ matrix.with-ray }}
+          RUN_DASK: ${{ matrix.run-dask }}
           NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
         shell: bash
         run: |
@@ -51,11 +54,6 @@ jobs:
           pip install git+https://github.com/mars-project/pytest-asyncio.git
           pip install numpy scipy cython
 
-          # remove if vineyard supports pandas 1.3.
-          if [ -n "$WITH_VINEYARD" ]; then
-            pip install pandas\<1.3
-          fi
-
           pip install -e ".[dev,extra]"
 
           if [[ $UNAME == "windows" ]]; then
@@ -73,7 +71,7 @@ jobs:
               conda install -n test --quiet --yes -c conda-forge python=$PYTHON skein conda-pack
             fi
             if [ -n "$WITH_VINEYARD" ]; then
-              pip install vineyard==0.2.4 -i https://pypi.org/simple
+              pip install vineyard==0.2.7 -i https://pypi.org/simple
 
               mkdir -p /tmp/etcd-download-test
               export ETCD_VER=v3.4.13
@@ -85,8 +83,11 @@ jobs:
               rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
             fi
             if [ -n "$WITH_RAY" ]; then
-              # remove ray version after https://github.com/ray-project/ray/issues/16802 got fixed
-              pip install ray[default]==1.3.0
+              # Change back to `pip install ray[default]` when ray-1.7.0 is released.
+              pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/aee7ba2510dd0eeed8f84dba3e9c5d58cb97d49a/ray-2.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+            fi
+            if [ -n "$RUN_DASK" ]; then
+              pip install dask[complete] mimesis sklearn
             fi
           fi
           conda list -n test
@@ -98,6 +99,7 @@ jobs:
           WITH_CYTHON: ${{ matrix.with-cython }}
           WITH_VINEYARD: ${{ matrix.with-vineyard }}
           WITH_RAY: ${{ matrix.with-ray }}
+          RUN_DASK: ${{ matrix.run-dask }}
           NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
           NUMPY_EXPERIMENTAL_ARRAY_FUNCTION: 1
           CHANGE_MINIKUBE_NONE_USER: true
@@ -121,12 +123,21 @@ jobs:
             pytest $PYTEST_CONFIG mars/deploy/oscar/tests/test_local.py
             mv .coverage build/.coverage.test_local.file
 
+            pytest $PYTEST_CONFIG -k "vineyard" \
+                mars/tensor/datastore/tests/test_datastore_execution.py \
+                mars/dataframe/datastore/tests/test_datastore_execution.py
+            mv .coverage build/.coverage.test_vineyard_op.file
+
             coverage combine build/ && coverage report
           fi
           if [ -n "$WITH_RAY" ]; then
             pytest $PYTEST_CONFIG --timeout=120 -m ray
             coverage report
           fi
+          if [ -n "$RUN_DASK" ]; then
+            pytest $PYTEST_CONFIG mars/contrib/dask/tests/test_dask.py
+            coverage report
+          fi
           coverage xml
 
       - name: Stop vineyard runtime

diff --git a/.github/workflows/cd.yml → .github/workflows/pypi-cd.yml b/.github/workflows/cd.yml → .github/workflows/pypi-cd.yml
@@ -1,4 +1,4 @@
-name: Mars CD
+name: Mars CD for PyPI
 
 on:
   push:

diff --git a/LICENSE b/LICENSE
@@ -222,10 +222,10 @@ Apache Software Foundation License 2.0
 - kubernetes-client:11.0.0
 - async_generator:1.10
 - dataclasses:0.8
-- vineyard:0.1.2
+- vineyard:0.2.7
 - ray:1.4.0
-- aiohttp:3.7.0
 - uvloop:0.15.2
+- oss2:2.12.0
 
 
 BSD 3-Clause
@@ -239,7 +239,6 @@ BSD 3-Clause
 - dask:0.17.1
 - python-lz4:2.1.0
 - sparse:0.5.0
-- bokeh:1.0.1
 - psutil:5.4.7
 - uhashring:1.1
 - skein:0.8.0
@@ -265,6 +264,13 @@ MIT License
 - react-router-dom:5.2.0
 - material-ui:4.0.0
 - lodash:4.17.0
+- dagre-d3:0.6.4
+
+
+ISC License
+-----------
+
+- d3:7.0.0
 
 
 CC0 1.0 Universal (CC0 1.0)

diff --git a/README.rst b/README.rst
@@ -138,6 +138,14 @@ Mars learn provides a familiar interface like scikit-learn.
 |                                             |                                                    |
 +---------------------------------------------+----------------------------------------------------+
 
+Mars learn also integrates with many libraries:
+
+- `TensorFlow <https://docs.pymars.org/en/latest/user_guide/learn/tensorflow.html>`_
+- `PyTorch <https://docs.pymars.org/en/latest/user_guide/learn/pytorch.html>`_
+- `XGBoost <https://docs.pymars.org/en/latest/user_guide/learn/xgboost.html>`_
+- `LightGBM <https://docs.pymars.org/en/latest/user_guide/learn/lightgbm.html>`_
+- `Joblib <https://docs.pymars.org/en/latest/user_guide/learn/joblib.html>`_
+- `Statsmodels <https://docs.pymars.org/en/latest/user_guide/learn/statsmodels.html>`_
 
 Mars remote
 -----------
@@ -179,6 +187,10 @@ Mars remote allows users to execute functions in parallel.
 |                                           |                                            |
 +-------------------------------------------+--------------------------------------------+
 
+DASK on Mars
+------------
+
+Refer to `DASK on Mars`_ for more information.
 
 Eager Mode
 ```````````
@@ -300,3 +312,4 @@ Thank you in advance for your contributions!
 .. _`中文文档`: https://docs.pymars.org/zh_CN/latest/
 .. _`Run on Kubernetes`: https://docs.pymars.org/en/latest/installation/kubernetes.html
 .. _`Run on Yarn`: https://docs.pymars.org/en/latest/installation/yarn.html
+.. _`DASK on Mars`: https://docs.pymars.org/en/latest/user_guide/contrib/dask.html