diff --git a/.github/workflows/centos7.yml b/.github/workflows/centos7.yml index 6b97281cc93..dc3dae95067 100644 --- a/.github/workflows/centos7.yml +++ b/.github/workflows/centos7.yml @@ -19,7 +19,6 @@ jobs: # ImportError: /lib64/libstdc++.so.6: version `CXXABI_1.3.8' not found # (required by /__w/espnet/espnet/tools/venv/envs/espnet/lib/python3.6/site-packages/pyworld/pyworld.cpython-36m-x86_64-linux-gnu.so) TH_VERSION: 1.8.1 - K2_VERSION: 1.3.dev20210806 CHAINER_VERSION: 6.0.0 USE_CONDA: true CC: /opt/rh/devtoolset-7/root/usr/bin/gcc diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 57d08c7031c..bf93991ed11 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,20 +18,17 @@ jobs: python-version: [3.7] pytorch-version: [1.3.1, 1.4.0, 1.5.1, 1.6.0, 1.7.1, 1.8.1, 1.9.0] chainer-version: [6.0.0] - k2-version: [1.2.dev20210726+cpu] # NOTE(kamo): Conda is tested by Circle-CI use-conda: [false] include: - os: ubuntu-20.04 python-version: 3.8 pytorch-version: 1.8.1 - k2-version: 1.2.dev20210726+cpu chainer-verssion: 6.0.0 use-conda: false - os: ubuntu-20.04 python-version: 3.9 pytorch-version: 1.8.1 - k2-version: 1.2.dev20210726+cpu chainer-verssion: 6.0.0 use-conda: false steps: @@ -52,7 +49,6 @@ jobs: env: ESPNET_PYTHON_VERSION: ${{ matrix.python-version }} TH_VERSION: ${{ matrix.pytorch-version }} - K2_VERSION: ${{ matrix.k2-version }} CHAINER_VERSION: ${{ matrix.chainer-version }} USE_CONDA: ${{ matrix.use-conda }} CC: gcc-7 diff --git a/.github/workflows/debian9.yml b/.github/workflows/debian9.yml index c9d014ba1d3..73834ba8593 100644 --- a/.github/workflows/debian9.yml +++ b/.github/workflows/debian9.yml @@ -16,7 +16,6 @@ jobs: env: ESPNET_PYTHON_VERSION: 3.7 TH_VERSION: 1.9.0 - K2_VERSION: 1.3.dev20210806 CHAINER_VERSION: 6.0.0 USE_CONDA: true CC: gcc-6 diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index be67a3523a3..73f101e45a5 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -31,7 +31,6 @@ jobs: env: ESPNET_PYTHON_VERSION: 3.8 TH_VERSION: 1.9.0 - K2_VERSION: 1.2.dev20210726+cpu CHAINER_VERSION: 6.0.0 USE_CONDA: false CC: gcc-7 diff --git a/.mergify.yml b/.mergify.yml index 394947c7c5f..09999340fe5 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,19 +2,21 @@ pull_request_rules: - name: automatic merge if label=auto-merge conditions: - "label=auto-merge" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.3.1, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.4.0, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.5.1, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.6.0, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.7.1, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.8.1, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-18.04, 3.7, 1.9.0, 6.0.0, 1.2.dev20210726+cpu, false)" - - "check-success=linter_and_test (ubuntu-20.04, 3.8, 1.8.1, 1.2.dev20210726+cpu, false, 6.0.0)" - - "check-success=linter_and_test (ubuntu-20.04, 3.9, 1.8.1, 1.2.dev20210726+cpu, false, 6.0.0)" + - "check-success=centos7 / linter_and_test" + - "check-success=debian9 / linter_and_test" + - "check-success=doc / linter_and_test" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.3.1, 5.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.4.0, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.5.1, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.6.0, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.7.1, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.8.1, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-18.04, 3.7, 1.9.0, 6.0.0, false)" + - "check-success=CI / linter_and_test (ubuntu-20.04, 3.8, 1.8.1, false, 6.0.0)" + - "check-success=CI / linter_and_test (ubuntu-20.04, 3.9, 1.8.1, false, 6.0.0)" actions: merge: method: merge - strict: false - name: delete head branch after merged conditions: - merged diff --git a/ci/install.sh b/ci/install.sh index cc1187d31e6..43873140c3f 100755 --- a/ci/install.sh +++ b/ci/install.sh @@ -21,7 +21,7 @@ ${CXX:-g++} -v . ./activate_python.sh make TH_VERSION="${TH_VERSION}" - make warp-ctc.done warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done + make warp-ctc.done warp-transducer.done chainer_ctc.done nkf.done moses.done mwerSegmenter.done pesq pyopenjtalk.done py3mmseg.done s3prl.done transformers.done phonemizer.done fairseq.done k2.done rm -rf kaldi ) . tools/activate_python.sh @@ -31,12 +31,6 @@ python3 -m pip install https://github.com/kpu/kenlm/archive/master.zip # NOTE(kamo): tensorboardx is used for chainer mode only python3 -m pip install tensorboardx -if ${USE_CONDA}; then - conda install -c k2-fsa -c pytorch k2=${K2_VERSION} cpuonly pytorch=${TH_VERSION} -else - python3 -m pip install k2==${K2_VERSION}.torch${TH_VERSION} -f https://k2-fsa.org/nightly/ -fi - # NOTE(kan-bayashi): Fix the error in black installation. # See: https://github.com/psf/black/issues/1707 @@ -58,5 +52,5 @@ version = '$TH_VERSION'.split(".") next_version = f"{version[0]}.{version[1]}.{int(version[2]) + 1}" if L(torch.__version__) < L('$TH_VERSION') or L(torch.__version__) >= L(next_version): - raise RuntimeError(f"Pytorch=$TH_VERSION is expected, but got pytorch={torch.__version__}. This is a bug installation scripts") + raise RuntimeError(f"Pytorch=$TH_VERSION is expected, but got pytorch={torch.__version__}. This is a bug in installation scripts") EOF diff --git a/test/espnet2/bin/test_k2_asr_inference.py b/test/espnet2/bin/test_k2_asr_inference.py index e36c912d2f0..8cd3b5b5a05 100644 --- a/test/espnet2/bin/test_k2_asr_inference.py +++ b/test/espnet2/bin/test_k2_asr_inference.py @@ -12,6 +12,9 @@ from espnet2.tasks.lm import LMTask +pytest.importorskip("k2") + + def test_get_parser(): assert isinstance(get_parser(), ArgumentParser) diff --git a/tools/check_install.py b/tools/check_install.py index 4930666af64..67c0972160c 100644 --- a/tools/check_install.py +++ b/tools/check_install.py @@ -27,6 +27,7 @@ ("gtn", None, "installers/install_gtn.sh"), ("s3prl", None, "installers/install_s3prl.sh"), ("transformers", None, "installers/install_transformers.sh"), + ("k2", None, "installers/install_k2.sh"), ] executable_list = [ diff --git a/tools/installers/install_k2.sh b/tools/installers/install_k2.sh old mode 100644 new mode 100755 index cf3c4271675..9635e72c273 --- a/tools/installers/install_k2.sh +++ b/tools/installers/install_k2.sh @@ -1,24 +1,145 @@ #!/usr/bin/env bash set -euo pipefail +# Please update if too old. See https://k2-fsa.org/nightly/, https://anaconda.org/k2-fsa/k2/files +pip_k2_version="1.6.dev20210907" +conda_k2_version="1.6.dev20210824" # Empty indicates latest version -if [ $# != 0 ]; then - echo "Usage: $0" +if [ $# -gt 2 ]; then + echo "Usage: $0 [use-conda|true or false] []" exit 1; +elif [ $# -gt 0 ]; then + use_conda="$1" + if [ "${use_conda}" != false ] && [ "${use_conda}" != true ]; then + echo "[ERROR] must be true or false, but ${use_conda} is given." + echo "Usage: $0 [use-conda|true or false] []" + exit 1 + fi + + if [ $# -eq 2 ]; then + k2_version="$2" + pip_k2_version="${k2_version}" + conda_k2_version="${k2_version}" + fi +else + use_conda=$([[ $(conda list -e -c -f --no-pip pytorch 2>/dev/null) =~ pytorch ]] && echo true || echo false) +fi + + +python_36_plus=$(python3 <= V("3.6"): + print("true") +else: + print("false") +EOF +) + +cuda_version=$(python3 < 10.1 + print(version[0] + "." + version[1]) +else: + print("") +EOF +) +torch_version=$(python3 < 1.10.0 +torch_version=torch.__version__.split("+")[0] +print(torch_version) +EOF +) + +pytorch_plus(){ + python3 <= L('$1'): + print("true") +else: + print("false") +EOF +} + +echo "[INFO] torch_version=${torch_version}" +echo "[INFO] cuda_version=${cuda_version}" + +if ! "${python_36_plus}"; then + echo "[ERROR] k2 requires python>=3.6" + exit 1 +fi + +if ! "$(pytorch_plus 1.6.0)"; then + echo "[WARNING] Using pip to install k2" + use_conda=false fi -# Install k2 -if [ ! -e k2.done ]; then - ( - set -euo pipefail - # Will install pytorch==1.7.1 automatically. - # Refer to - # https://k2.readthedocs.io/en/latest/installation/index.html - # for more alternatives to install k2 - python3 -m pip install k2 - ) - touch k2.done + +# Check pytorch version. +# Please exit without error code for CI. +if "${use_conda}"; then + if [ "${conda_k2_version}" = "1.6.dev20210824" ]; then + if "$(pytorch_plus 1.9.1)"; then + echo "[WARNING] k2=${conda_k2_version} doesn't provide conda package for pytorch=${torch_version}. Skip k2-installation" + exit + elif ! "$(pytorch_plus 1.8.1)"; then + echo "[WARNING] k2=${conda_k2_version} doesn't provide conda package for pytorch=${torch_version}. Skip k2-installation" + exit + fi + if [ -n "${cuda_version}" ] && [ "${cuda_version}" != "10.2" ] && [ "${cuda_version}" != "11.0" ] && [ "${cuda_version}" != "11.1" ]; then + echo "[WARNING] k2=${conda_k2_version} provides conda package for CUDA10.2, 11.0, 11.1 only. Skip k2-installation" + exit + fi + fi else - echo "k2 is already installed" + if [ "${pip_k2_version}" = "1.6.dev20210907" ]; then + if "$(pytorch_plus 1.9.1)"; then + echo "[WARNING] k2=${pip_k2_version} for pip doesn't provide pytorch=${torch_version} binary. Skip k2-installation" + exit + elif ! "$(pytorch_plus 1.3.1)"; then + echo "[WARNING] k2=${pip_k2_version} for pip doesn't provide pytorch=${torch_version} binary. Skip k2-installation" + exit + fi + if [ -n "${cuda_version}" ] && [ "${torch_version}" != "1.7.1" ]; then + echo "[WARNING] k2=${pip_k2_version}+cuda for pip provides pytorch=1.7.1 binary only. Skip k2-installation" + exit + fi + if [ -n "${cuda_version}" ] && [ "${cuda_version}" != "10.1" ] && [ "${cuda_version}" != "10.2" ] && [ "${cuda_version}" != "11.0" ]; then + echo "[WARNING] k2=${pip_k2_version} for pip provides CUDA10.1, 10.2, 11.0 binary only. Skip k2-installation" + exit + fi + fi fi + + +if "${use_conda}"; then + [ -z "${conda_k2_version}" ] && k2="k2" || k2="k2=${conda_k2_version}" + + if [ -z "${cuda_version}" ]; then + echo conda install -y -c k2-fsa -c pytorch cpuonly "${k2}" "pytorch=${torch_version}" + conda install -y -c k2-fsa -c pytorch cpuonly "${k2}" "pytorch=${torch_version}" + else + # NOTE(kamo): K2 requires cudatoolkit from conda-forge channel and k2-cpu is installed if the other channel is used, e.g. anaconda, nvidia + echo conda install -y -c k2-fsa -c pytorch -c conda-forge "${k2}" "cudatoolkit=${cuda_version}" "pytorch=${torch_version}" + conda install -y -c k2-fsa -c pytorch -c conda-forge "${k2}" "cudatoolkit=${cuda_version}" "pytorch=${torch_version}" + fi + +else + if [ -z "${cuda_version}" ]; then + echo pip install "k2==${pip_k2_version}+cpu.torch${torch_version}" -f https://k2-fsa.org/nightly/ + pip install "k2==${pip_k2_version}+cpu.torch${torch_version}" -f https://k2-fsa.org/nightly/ + else + echo pip install "k2==${pip_k2_version}+cuda${cuda_version}.torch${torch_version}" -f https://k2-fsa.org/nightly/ + pip install "k2==${pip_k2_version}+cuda${cuda_version}.torch${torch_version}" -f https://k2-fsa.org/nightly/ + fi +fi