Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
Test CD mxnet_lib/static and python/pypi stages on CI (#18559)
Browse files Browse the repository at this point in the history 
* add cd mxnet_lib/static stages to ci

* add cd pypi packaging stage to ci

* removing existing cmake static compile stages in favor of other added stages

* pass mxnet_variant correctly
  • Loading branch information
@mseth10
mseth10 authored Jun 16, 2020
1 parent 8039377 commit 103d839
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 36 deletions.
35 changes: 12 additions & 23 deletions ci/docker/runtime_functions.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1871,6 +1871,18 @@ build_static_libmxnet() {
popd
}

# Tests CD PyPI packaging in CI
ci_package_pypi() {
set -ex
# copies mkldnn header files to 3rdparty/mkldnn/include/ as in CD
mkdir -p 3rdparty/mkldnn/include
cp include/mkldnn/dnnl_version.h 3rdparty/mkldnn/include/.
cp include/mkldnn/dnnl_config.h 3rdparty/mkldnn/include/.
local mxnet_variant=${1:?"This function requires a python command as the first argument"}
cd_package_pypi ${mxnet_variant}
cd_integration_test_pypi
}

# Packages libmxnet into wheel file
cd_package_pypi() {
set -ex
Expand Down Expand Up @@ -1946,29 +1958,6 @@ build_static_python_cu92() {
popd
}

build_static_python_cpu_cmake() {
set -ex
pushd .
export mxnet_variant=cpu
export CMAKE_STATICBUILD=1
source /opt/rh/devtoolset-7/enable
source /opt/rh/rh-python36/enable
./ci/publish/python/build.sh
popd
}

build_static_python_cu92_cmake() {
set -ex
pushd .
export mxnet_variant=cu92
export CMAKE_STATICBUILD=1
export USE_SYSTEM_CUDA=1
source /opt/rh/devtoolset-7/enable
source /opt/rh/rh-python36/enable
./ci/publish/python/build.sh
popd
}

publish_scala_build() {
set -ex
pushd .
Expand Down
86 changes: 77 additions & 9 deletions ci/jenkins/Jenkins_steps.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ mx_lib_cpp_examples_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.
mx_lib_cpp_capi_make = 'lib/libmxnet.so, lib/libmxnet.a, lib/libtvm_runtime.so, lib/libtvmop.so, lib/tvmop.conf, libsample_lib.so, lib/libmkldnn.so.1, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/**, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so, build/tests/cpp/mxnet_unit_tests'
mx_lib_cpp_examples_no_tvm_op = 'build/libmxnet.so, build/libcustomop_lib.so, build/libcustomop_gpu_lib.so, build/libsubgraph_lib.so, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/**, python/mxnet/_cy3/*.so, python/mxnet/_ffi/_cy3/*.so'
mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/3rdparty/tvm/libtvm_runtime.so, build/libtvmop.so, build/tvmop.conf, build/3rdparty/openmp/runtime/src/libomp.so, build/cpp-package/example/**'
mx_cd_lib = 'lib/libmxnet.so, licenses/*, lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'

// Python unittest for CPU
// Python 3
Expand Down Expand Up @@ -747,13 +748,14 @@ def compile_static_python_cpu() {
}]
}

def compile_static_python_cpu_cmake() {
return ['Static build CPU CentOS7 Python with CMake' : {
def compile_static_cd_cpu(lib_name) {
return ['CPU: CD Static Build' : {
node(NODE_LINUX_CPU) {
ws('workspace/ut-publish-python-cpu') {
ws('workspace/build-cd-static/cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('centos7_cpu', 'build_static_python_cpu_cmake', false)
utils.docker_run('centos7_cpu', 'build_static_libmxnet cpu', false)
utils.pack_lib(lib_name, mx_cd_lib)
}
}
}
Expand All @@ -773,13 +775,14 @@ def compile_static_python_gpu() {
}]
}

def compile_static_python_gpu_cmake() {
return ['Static build GPU CentOS7 Python with CMake' : {
node(NODE_LINUX_GPU) {
ws('workspace/ut-publish-python-gpu') {
def compile_static_cd_gpu(lib_name) {
return ['GPU: CD Static Build' : {
node(NODE_LINUX_CPU) {
ws('workspace/build-cd-static/gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git()
utils.docker_run('centos7_gpu_cu92', 'build_static_python_cu92_cmake')
utils.docker_run('centos7_gpu_cu102', 'build_static_libmxnet cu102', false)
utils.pack_lib(lib_name, mx_cd_lib)
}
}
}
Expand Down Expand Up @@ -1252,6 +1255,32 @@ def test_centos7_python3_cpu(lib_name) {
}]
}

def test_centos7_python3_cd_cpu(lib_name) {
return ['Python3: CentOS 7 CPU CD': {
node(NODE_LINUX_CPU) {
ws('workspace/test-cd-static/cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cd_lib)
utils.docker_run('centos7_cpu', 'cd_unittest_ubuntu cpu', false)
}
}
}
}]
}

def test_centos7_pypi_package_cd_cpu(lib_name) {
return ['PyPI package: CentOS 7 CPU CD': {
node(NODE_LINUX_CPU) {
ws('workspace/test-cd-pypi/cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cd_lib)
utils.docker_run('centos7_cpu', 'ci_package_pypi cpu', false)
}
}
}
}]
}

def test_centos7_python3_gpu(lib_name) {
return ['Python3: CentOS 7 GPU': {
node(NODE_LINUX_GPU) {
Expand All @@ -1270,6 +1299,45 @@ def test_centos7_python3_gpu(lib_name) {
}]
}

def test_centos7_python3_cd_gpu(lib_name) {
return ['Python3: CentOS 7 GPU CD': {
node(NODE_LINUX_GPU) {
ws('workspace/test-cd-static/gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cd_lib)
utils.docker_run('centos7_gpu_cu102', 'cd_unittest_ubuntu cu102', true)
}
}
}
}]
}

def test_centos7_quantization_cd_gpu(lib_name) {
return ['Quantization Python3: CentOS 7 GPU CD': {
node(NODE_LINUX_GPU_P3) {
ws('workspace/test-cd-static/gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cd_lib)
utils.docker_run('centos7_gpu_cu102', 'unittest_ubuntu_python3_quantization_gpu', true)
}
}
}
}]
}

def test_centos7_pypi_package_cd_gpu(lib_name) {
return ['PyPI package: CentOS 7 GPU CD': {
node(NODE_LINUX_GPU) {
ws('workspace/test-cd-pypi/gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.unpack_and_init(lib_name, mx_cd_lib)
utils.docker_run('centos7_gpu_cu102', 'ci_package_pypi cu102', true)
}
}
}
}]
}

def test_centos7_scala_cpu(lib_name) {
return ['Scala: CentOS CPU Makefile': {
node(NODE_LINUX_CPU) {
Expand Down
6 changes: 4 additions & 2 deletions ci/jenkins/Jenkinsfile_centos_cpu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@ core_logic: {
custom_steps.compile_centos7_cpu_mkldnn(),
custom_steps.compile_static_scala_cpu(),
custom_steps.compile_static_python_cpu(),
custom_steps.compile_static_python_cpu_cmake()
custom_steps.compile_static_cd_cpu('centos7_cpu_cd')
])

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_cpu('centos7_cpu'),
custom_steps.test_centos7_scala_cpu('centos7_cpu_make')
custom_steps.test_centos7_scala_cpu('centos7_cpu_make'),
custom_steps.test_centos7_python3_cd_cpu('centos7_cpu_cd'),
custom_steps.test_centos7_pypi_package_cd_cpu('centos7_cpu_cd')
])
}
,
Expand Down
7 changes: 5 additions & 2 deletions ci/jenkins/Jenkinsfile_centos_gpu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,14 @@ core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_centos7_gpu('centos7_gpu'),
custom_steps.compile_static_python_gpu(),
custom_steps.compile_static_python_gpu_cmake()
custom_steps.compile_static_cd_gpu('centos7_gpu_cd')
])

utils.parallel_stage('Tests', [
custom_steps.test_centos7_python3_gpu('centos7_gpu')
custom_steps.test_centos7_python3_gpu('centos7_gpu'),
custom_steps.test_centos7_python3_cd_gpu('centos7_gpu_cd'),
custom_steps.test_centos7_quantization_cd_gpu('centos7_gpu_cd'),
custom_steps.test_centos7_pypi_package_cd_gpu('centos7_gpu_cd')
])
}
,
Expand Down

0 comments on commit 103d839

Please sign in to comment.