Pipeline changes for python 3.8 (#2753)

1. Pipeline changes for python 3.8 2. Fix a regression in setup.py which was just introduced in the previous commit. Please notice, we still haven't made python 3.8 + Windows + CUDA work.
microsoft · Jan 2, 2020 · 382fa86 · 382fa86
1 parent fd334af
commit 382fa86
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 8 deletions.
diff --git a/setup.py b/setup.py
@@ -214,7 +214,7 @@ def run(self):
     py_modules=python_modules_list,
     install_requires=[
         'onnx>=1.2.3',
-        'numpy>=1.18.0,<2'
+        'numpy>=1.18.0'
     ],
     entry_points= {
         'console_scripts': [

diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@@ -8,12 +8,19 @@ jobs:
       Python35:
         python.version: '3.5'
         python.dir: '/opt/python/cp35-cp35m'
+        python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
       Python36:
         python.version: '3.6'
         python.dir: '/opt/python/cp36-cp36m'
+        python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
       Python37:
         python.version: '3.7'
         python.dir: '/opt/python/cp37-cp37m'
+        python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
+      Python38:
+        python.version: '3.8'
+        python.dir: '/opt/python/cp38-cp38'
+        python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
   steps:
     - template: templates/set-test-data-variables-step.yml
 
@@ -40,7 +47,7 @@ jobs:
     - task: CmdLine@2
       inputs:
         script: |
-          docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync  --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so
+          docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync  --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
         workingDirectory: $(Build.SourcesDirectory)
 
     - task: CopyFiles@2
@@ -69,12 +76,19 @@ jobs:
       Python35:
         python.version: '3.5'
         python.dir: '/opt/python/cp35-cp35m'
+        python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m'
       Python36:
         python.version: '3.6'
         python.dir: '/opt/python/cp36-cp36m'
+        python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m'
       Python37:
         python.version: '3.7'
         python.dir: '/opt/python/cp37-cp37m'
+        python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m'
+      Python38:
+        python.version: '3.8'
+        python.dir: '/opt/python/cp38-cp38'
+        python.include.dir: '/opt/python/cp38-cp38/include/python3.8'
   steps:
     - template: templates/set-test-data-variables-step.yml
 
@@ -101,7 +115,7 @@ jobs:
     - task: CmdLine@2
       inputs:
         script: |
-          docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync  --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0  --cudnn_home=/usr/local/cuda-10.0
+          docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync  --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0  --cudnn_home=/usr/local/cuda-10.0
         workingDirectory: $(Build.SourcesDirectory)
 
     - task: CopyFiles@2
@@ -131,6 +145,8 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
   variables:
     OrtPackageId: 'Microsoft.ML.OnnxRuntime'
     MsbuildArguments: '-maxcpucount'
@@ -199,7 +215,7 @@ jobs:
   workspace:
     clean: all
   pool: 'Win-GPU-2019'
-  timeoutInMinutes:  120  
+  timeoutInMinutes:  60  
   variables:
     CUDA_VERSION: '10.0'
     EnvSetupScript: setup_env.bat
@@ -276,6 +292,8 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
   steps:
     - task: UsePythonVersion@0
       displayName: 'Use Python'

diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh
@@ -54,6 +54,8 @@ elif [[ "$PYTHON_VER" = "3.6" && -d "/opt/python/cp36-cp36m"  ]]; then
    PYTHON_EXE="/opt/python/cp36-cp36m/bin/python3.6"
 elif [[ "$PYTHON_VER" = "3.7" && -d "/opt/python/cp37-cp37m"  ]]; then
    PYTHON_EXE="/opt/python/cp37-cp37m/bin/python3.7"
+elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38"  ]]; then
+   PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8"
 else
    PYTHON_EXE="/usr/bin/python${PYTHON_VER}"
 fi

diff --git a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh
@@ -14,13 +14,12 @@ PYTHON_VER=$1
 
 if [[ "$PYTHON_VER" = "3.5" && -d "/opt/python/cp35-cp35m"  ]]; then
    PYTHON_EXE="/opt/python/cp35-cp35m/bin/python3.5"
-   export PATH=/opt/python/cp35-cp35m/bin:$PATH
 elif [[ "$PYTHON_VER" = "3.6" && -d "/opt/python/cp36-cp36m"  ]]; then
    PYTHON_EXE="/opt/python/cp36-cp36m/bin/python3.6"
-   export PATH=/opt/python/cp36-cp36m/bin:$PATH
 elif [[ "$PYTHON_VER" = "3.7" && -d "/opt/python/cp37-cp37m"  ]]; then
    PYTHON_EXE="/opt/python/cp37-cp37m/bin/python3.7"
-   export PATH=/opt/python/cp37-cp37m/bin:$PATH
+elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38"  ]]; then
+   PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8"
 else
    PYTHON_EXE="/usr/bin/python${PYTHON_VER}"
 fi
@@ -57,5 +56,6 @@ for v2t in ${version2tag[*]}; do
   mv /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py
   mkdir -p /data/onnx/${onnx_tag}
   ${PYTHON_EXE} -m pip install .
-  backend-test-tools generate-data -o /data/onnx/$onnx_tag
+  cd /tmp  
+  ${PYTHON_EXE} -m onnx.backend.test.cmd_tools generate-data -o /data/onnx/$onnx_tag
 done