Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Disable test coverage in MKL builds #18443

Merged
merged 7 commits into from
Jul 15, 2020
Merged

Disable test coverage in MKL builds #18443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
16bada7
Disable test coverage in MKL builds
leezu May 29, 2020
49fda1b
Enable test parallelization
leezu May 29, 2020
3f33bdc
Set OMP_NUM_THREADS
leezu Jun 1, 2020
e836b6a
Merge remote-tracking branch 'origin/master' into mkltestcoverage
leezu Jun 18, 2020
f6cf72a
Fix
leezu Jun 19, 2020
f260a0b
Merge remote-tracking branch 'origin/master' into mkltestcoverage
leezu Jun 25, 2020
a6cc571
Fix unpack_and_init
leezu Jul 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 30 additions & 40 deletions ci/docker/runtime_functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ build_ubuntu_cpu_mkl() {
cd /work/build
CC=gcc-7 CXX=g++-7 cmake \
-DCMAKE_BUILD_TYPE="RelWithDebInfo" \
-DENABLE_TESTCOVERAGE=ON \
-DENABLE_TESTCOVERAGE=OFF \
-DUSE_MKLDNN=OFF \
-DUSE_CUDA=OFF \
-DUSE_TVM_OP=ON \
Expand Down Expand Up @@ -625,7 +625,7 @@ build_ubuntu_cpu_mkldnn_mkl() {
cd /work/build
CC=gcc-7 CXX=g++-7 cmake \
-DCMAKE_BUILD_TYPE="RelWithDebInfo" \
-DENABLE_TESTCOVERAGE=ON \
-DENABLE_TESTCOVERAGE=OFF \
-DUSE_MKLDNN=ON \
-DUSE_CUDA=OFF \
-DUSE_TVM_OP=ON \
Expand Down Expand Up @@ -880,7 +880,7 @@ sanity_check() {
tools/license_header.py check
make cpplint
make pylint
pytest -n 4 tests/tutorials/test_sanity_tutorials.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py
}

# Tests libmxnet
Expand All @@ -899,7 +899,7 @@ cd_unittest_ubuntu() {

local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"}

pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --verbose tests/python/unittest

# https://github.com/apache/incubator-mxnet/issues/11801
Expand All @@ -910,9 +910,9 @@ cd_unittest_ubuntu() {
if [[ ${mxnet_variant} = cu* ]]; then
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --verbose tests/python/gpu

# TODO(szha): fix and reenable the hanging issue. tracked in #18098
Expand All @@ -922,7 +922,7 @@ cd_unittest_ubuntu() {
fi

if [[ ${mxnet_variant} = *mkl ]]; then
pytest -n 4 --durations=50 --verbose tests/python/mkl
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 --durations=50 --verbose tests/python/mkl
fi
}

Expand All @@ -934,24 +934,12 @@ unittest_ubuntu_python3_cpu() {
export MXNET_SUBGRAPH_VERBOSE=0
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=10
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
}

unittest_ubuntu_python3_cpu_serial() {
# TODO(szha): delete this and switch to unittest_ubuntu_python3_cpu once #18244 is fixed
set -ex
export PYTHONPATH=./python/
export MXNET_MKLDNN_DEBUG=0 # Ignored if not present
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
export MXNET_SUBGRAPH_VERBOSE=0
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=10
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
}

unittest_ubuntu_python3_cpu_mkldnn() {
set -ex
export PYTHONPATH=./python/
Expand All @@ -960,8 +948,10 @@ unittest_ubuntu_python3_cpu_mkldnn() {
export MXNET_SUBGRAPH_VERBOSE=0
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=10
# TODO(szha): enable parallel testing and naive engine for ops once #18244 is fixed
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose tests/python/mkl
}

Expand All @@ -975,10 +965,10 @@ unittest_ubuntu_python3_gpu() {
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=10
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
}

Expand All @@ -994,10 +984,10 @@ unittest_ubuntu_python3_gpu_cython() {
export DMLC_LOG_STACK_TRACE_DEPTH=10
check_cython
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
}

Expand All @@ -1010,10 +1000,10 @@ unittest_ubuntu_python3_gpu_nocudnn() {
export MXNET_ENABLE_CYTHON=0
export DMLC_LOG_STACK_TRACE_DEPTH=10
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
}

Expand All @@ -1026,11 +1016,11 @@ unittest_centos7_cpu() {
set -ex
source /opt/rh/rh-python36/enable
cd /work/mxnet
python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
}

unittest_centos7_gpu() {
Expand All @@ -1040,10 +1030,10 @@ unittest_centos7_gpu() {
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
export DMLC_LOG_STACK_TRACE_DEPTH=10
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
MXNET_ENGINE_TYPE=NaiveEngine \
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
}

Expand All @@ -1052,9 +1042,9 @@ integrationtest_ubuntu_cpu_onnx() {
export PYTHONPATH=./python/
export DMLC_LOG_STACK_TRACE_DEPTH=10
python3 tests/python/unittest/onnx/backend_test.py
pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py
pytest -n 4 tests/python/unittest/onnx/test_models.py
pytest -n 4 tests/python/unittest/onnx/test_node.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py
}

integrationtest_ubuntu_cpu_dist_kvstore() {
Expand Down Expand Up @@ -1130,9 +1120,9 @@ test_ubuntu_cpu_python3() {
cd /work/mxnet/python
pip3 install -e .
cd /work/mxnet
python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest
MXNET_ENGINE_TYPE=NaiveEngine \
python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest
python3 -m pytest -m 'serial' --durations=50 --verbose tests/python/unittest

popd
Expand Down Expand Up @@ -1502,7 +1492,7 @@ test_artifact_repository() {
set -ex
pushd .
cd cd/utils/
pytest -n 4 test_artifact_repository.py
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 test_artifact_repository.py
popd
}

Expand Down
16 changes: 5 additions & 11 deletions ci/jenkins/Jenkins_steps.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,6 @@ def python3_ut(docker_container_name) {
}
}

def python3_ut_serial(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_serial', false)
}
}

def python3_ut_mkldnn(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
utils.docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false)
Expand Down Expand Up @@ -177,7 +171,7 @@ def compile_unix_mkl_cpu(lib_name) {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkl', false)
utils.pack_lib(lib_name, mx_lib, true)
utils.pack_lib(lib_name, mx_lib, false)
}
}
}
Expand Down Expand Up @@ -219,7 +213,7 @@ def compile_unix_mkldnn_mkl_cpu(lib_name) {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn_mkl', false)
utils.pack_lib(lib_name, mx_mkldnn_lib, true)
utils.pack_lib(lib_name, mx_mkldnn_lib, false)
}
}
}
Expand Down Expand Up @@ -772,8 +766,8 @@ def test_unix_python3_mkl_cpu(lib_name) {
node(NODE_LINUX_CPU) {
ws('workspace/ut-python3-cpu') {
try {
utils.unpack_and_init(lib_name, mx_lib, true)
python3_ut_serial('ubuntu_cpu')
utils.unpack_and_init(lib_name, mx_lib)
python3_ut('ubuntu_cpu')
utils.publish_test_coverage()
} finally {
utils.collect_test_results_unix('tests_unittest.xml', 'tests_python3_cpu_unittest.xml')
Expand Down Expand Up @@ -854,7 +848,7 @@ def test_unix_python3_mkldnn_mkl_cpu(lib_name) {
node(NODE_LINUX_CPU) {
ws('workspace/ut-python3-mkldnn-mkl-cpu') {
try {
utils.unpack_and_init(lib_name, mx_lib, true)
utils.unpack_and_init(lib_name, mx_lib)
python3_ut_mkldnn('ubuntu_cpu')
utils.publish_test_coverage()
} finally {
Expand Down