diff --git a/.clang-tidy b/.clang-tidy index af99026e5d70..25fbff9f1a4d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -54,7 +54,8 @@ Checks: > # In order to trigger an error, you must have a rule defined both in checks and in this section. WarningsAsErrors: > - cppcoreguidelines-no-malloc, modernize-use-nullptr, performance-unnecessary-copy-initialization + cppcoreguidelines-no-malloc, modernize-use-nullptr, performance-unnecessary-copy-initialization, + modernize-use-emplace, performance-move-const-arg # Todo: define a better regex match that includes most project headers, but excludes third party # code. diff --git a/.travis.yml b/.travis.yml index 817d1ace7f87..079293059842 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,12 @@ sudo: true language: cpp -cache: ccache +cache: + directories: + - $HOME/.ccache + - $HOME/.cache/pip + - $HOME/.mxnet + - $HOME/Library/Caches/Homebrew os: - osx @@ -17,7 +22,7 @@ before_install: - export PYTHONPATH=${PYTHONPATH}:${PWD}/python install: - - brew install ccache + - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache - export PATH="/usr/local/opt/ccache/libexec:$PATH" - source ci/travis/install.sh @@ -29,4 +34,10 @@ script: - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - mv make/osx.mk config.mk - make -j 2 - - python -m nose --verbose tests/python/unittest/ + + # Temporarily disabled due to https://github.com/apache/incubator-mxnet/issues/13136 + # We ignore several tests to avoid possible timeouts on large PRs. + # This lowers our test coverage, but is required for consistent Travis runs. + # These tests will be tested in a variety of environments in Jenkins based tests. +# - python -m nose --with-timer --exclude-test=test_sparse_operator.test_elemwise_binary_ops --exclude-test=test_gluon_model_zoo.test_models --exclude-test=test_random.test_shuffle --exclude-test=test_operator.test_broadcast_binary_op --exclude-test=test_operator.test_pick --exclude-test=test_profiler.test_continuous_profile_and_instant_marker --exclude-test=test_metric_perf.test_metric_performance --exclude-test=test_operator.test_order --verbose tests/python/unittest/ +# - python2 -m nose --verbose tools/coreml/test --exclude-test=test_mxnet_image diff --git a/src/operator/contrib/ctc_include/LICENSE b/3rdparty/ctc_include/LICENSE similarity index 100% rename from src/operator/contrib/ctc_include/LICENSE rename to 3rdparty/ctc_include/LICENSE diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/LICENSE b/3rdparty/ctc_include/contrib/moderngpu/LICENSE similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/LICENSE rename to 3rdparty/ctc_include/contrib/moderngpu/LICENSE diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctaloadbalance.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctamerge.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctascan.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctascan.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctascan.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctascan.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasearch.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegreduce.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegscan.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasegsort.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/ctasortedsearch.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/devicetypes.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/deviceutil.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/intrinsics.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/loadstore.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/loadstore.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/loadstore.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/loadstore.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/serialsets.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/serialsets.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/serialsets.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/serialsets.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/device/sortnetwork.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpudevice.cuh b/3rdparty/ctc_include/contrib/moderngpu/include/mgpudevice.cuh similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpudevice.cuh rename to 3rdparty/ctc_include/contrib/moderngpu/include/mgpudevice.cuh diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpuenums.h b/3rdparty/ctc_include/contrib/moderngpu/include/mgpuenums.h similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/mgpuenums.h rename to 3rdparty/ctc_include/contrib/moderngpu/include/mgpuenums.h diff --git a/src/operator/contrib/ctc_include/contrib/moderngpu/include/util/static.h b/3rdparty/ctc_include/contrib/moderngpu/include/util/static.h similarity index 100% rename from src/operator/contrib/ctc_include/contrib/moderngpu/include/util/static.h rename to 3rdparty/ctc_include/contrib/moderngpu/include/util/static.h diff --git a/src/operator/contrib/ctc_include/detail/cpu_ctc.h b/3rdparty/ctc_include/detail/cpu_ctc.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/cpu_ctc.h rename to 3rdparty/ctc_include/detail/cpu_ctc.h diff --git a/src/operator/contrib/ctc_include/detail/ctc_helper.h b/3rdparty/ctc_include/detail/ctc_helper.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/ctc_helper.h rename to 3rdparty/ctc_include/detail/ctc_helper.h diff --git a/src/operator/contrib/ctc_include/detail/gpu_ctc.h b/3rdparty/ctc_include/detail/gpu_ctc.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/gpu_ctc.h rename to 3rdparty/ctc_include/detail/gpu_ctc.h diff --git a/src/operator/contrib/ctc_include/detail/gpu_ctc_kernels.h b/3rdparty/ctc_include/detail/gpu_ctc_kernels.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/gpu_ctc_kernels.h rename to 3rdparty/ctc_include/detail/gpu_ctc_kernels.h diff --git a/src/operator/contrib/ctc_include/detail/hostdevice.h b/3rdparty/ctc_include/detail/hostdevice.h similarity index 100% rename from src/operator/contrib/ctc_include/detail/hostdevice.h rename to 3rdparty/ctc_include/detail/hostdevice.h diff --git a/3rdparty/mkldnn b/3rdparty/mkldnn index 0e7ca738866d..9910b480296a 160000 --- a/3rdparty/mkldnn +++ b/3rdparty/mkldnn @@ -1 +1 @@ -Subproject commit 0e7ca738866d22cc700aa33b8de120b938f910d0 +Subproject commit 9910b480296a0d1496db466531e56729b3922bbf diff --git a/3rdparty/mshadow b/3rdparty/mshadow index 463c0dffe3ea..696803bd7723 160000 --- a/3rdparty/mshadow +++ b/3rdparty/mshadow @@ -1 +1 @@ -Subproject commit 463c0dffe3eae8c39caf7989c85b7244823df27e +Subproject commit 696803bd7723ade8230af878460d96c68a550fbc diff --git a/3rdparty/tvm b/3rdparty/tvm index 5fec9adbcaf8..0f053c82a747 160000 --- a/3rdparty/tvm +++ b/3rdparty/tvm @@ -1 +1 @@ -Subproject commit 5fec9adbcaf8debb720b56beffd45bd6941eff63 +Subproject commit 0f053c82a747b4dcdf49570ec87c17e0067b7439 diff --git a/CMakeLists.txt b/CMakeLists.txt index ddf99e714130..42f6bffb9207 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,27 +112,17 @@ else(MSVC) endif() # For cross complication, turn off flag if target device does not support it if(USE_F16C) - check_cxx_compiler_flag("-mf16c" COMPILER_SUPPORT_MF16C) - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - execute_process(COMMAND cat /proc/cpuinfo - COMMAND grep flags - COMMAND grep f16c - OUTPUT_VARIABLE CPU_SUPPORT_F16C) - elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - execute_process(COMMAND sysctl -a - COMMAND grep machdep.cpu.features - COMMAND grep F16C - OUTPUT_VARIABLE CPU_SUPPORT_F16C) - endif() - if(NOT CPU_SUPPORT_F16C) - message("CPU does not support F16C instructions") - endif() - if(CPU_SUPPORT_F16C AND COMPILER_SUPPORT_MF16C) - set(SUPPORT_F16C TRUE) - endif() + # Determine if hardware supports F16C instruction set + message(STATUS "Determining F16C support") + include(cmake/AutoDetectF16C.cmake) else() set(SUPPORT_F16C FALSE) endif() + if(SUPPORT_F16C) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mf16c") + else() + add_definitions(-DMSHADOW_USE_F16C=0) + endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -Wno-sign-compare") if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$") @@ -234,13 +224,15 @@ if(ENABLE_TESTCOVERAGE) endif() if(USE_MKLDNN) - include(cmake/MklDnn.cmake) + include(cmake/DownloadMKLML.cmake) # CPU architecture (e.g., C5) can't run on another architecture (e.g., g3). if(NOT MSVC) set(ARCH_OPT_FLAGS "-mtune=generic") endif() - set(WITH_TEST OFF) - set(WITH_EXAMPLE OFF) + set(WITH_TEST OFF CACHE INTERNAL "" FORCE) + set(WITH_EXAMPLE OFF CACHE INTERNAL "" FORCE) + set(ARCH_OPT_FLAGS "" CACHE INTERNAL "" FORCE) + add_subdirectory(3rdparty/mkldnn) include_directories(3rdparty/mkldnn/include) @@ -302,7 +294,7 @@ else() endif() if(USE_ASAN) - set(CMAKE_CFLAGS "${CMAKE_CFLAGS} -fno-omit-frame-pointer -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -fsanitize=address") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address") set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fsanitize=address") set(GTEST_LIBRARIES "${GTEST_LIBRARIES} -fsanitize=address") diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3ae61298de8e..d1dd9b90708a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -44,6 +44,11 @@ The committers are the granted write access to the project. * [Sergey Kolychev](https://github.com/sergeykolychev) - Sergey is original author and current maintainer of Perl5 interface. * [Naveen Swamy](https://github.com/nswamy) +* [Marco de Abreu](https://github.com/marcoabreu) + - Marco is the creator of the current MXNet CI. +* [Carin Meier](https://github.com/gigasquid) + - Carin created and is the current maintainer for the Clojure interface. + ### Become a Committer MXNet is a opensource project and we are actively looking for new committers @@ -53,6 +58,7 @@ who are willing to help maintaining and leading the project. Committers come fro New committers will be proposed by current committers, with support from more than two of current committers. + List of Contributors -------------------- * [Full List of Contributors](https://github.com/apache/incubator-mxnet/graphs/contributors) @@ -153,8 +159,6 @@ List of Contributors * [Manu Seth](https://github.com/mseth10/) * [Calum Leslie](https://github.com/calumleslie) * [Andre Tamm](https://github.com/andretamm) -* [Marco de Abreu](https://github.com/marcoabreu) - - Marco is the creator of the current MXNet CI. * [Julian Salazar](https://github.com/JulianSlzr) * [Meghna Baijal](https://github.com/mbaijal) * [Tao Hu](https://github.com/dongzhuoyao) @@ -180,3 +184,24 @@ List of Contributors * [Per Goncalves da Silva](https://github.com/perdasilva) * [Zhijingcheng Yu](https://github.com/jasonyu1996) * [Cheng-Che Lee](https://github.com/stu1130) +* [Chaitanya Bapat](https://github.com/ChaiBapchya) +* [LuckyPigeon](https://github.com/LuckyPigeon) +* [Anton Chernov](https://github.com/lebeg) +* [Denisa Roberts](https://github.com/D-Roberts) +* [Dick Carter](https://github.com/DickJC123) +* [Rahul Padmanabhan](https://github.com/rahul3) +* [Yuxi Hu](https://github.com/yuxihu) +* [Harsh Patel](https://github.com/harshp8l) + +Label Bot +--------- +* [mxnet-label-bot](https://github.com/mxnet-label-bot) + - mxnet-label-bot provides users with the functionality to manage labels for Issues/Pull Requests on the repository + - To use me, comment: + - @mxnet-label-bot add [specify comma separated labels here] + - @mxnet-label-bot remove [specify comma separated labels here] + - @mxnet-label-bot update [specify comma separated labels here] + (i.e. @mxnet-label-bot update [Bug, Python]) + + - Available label names which are supported: [Labels](https://github.com/apache/incubator-mxnet/labels) + - For further details: [My Wiki Page](https://cwiki.apache.org/confluence/display/MXNET/Machine+Learning+Based+GitHub+Bot) diff --git a/Jenkinsfile b/Jenkinsfile index 81a25deca27b..fca85395194a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,6 +22,10 @@ // mxnet libraries mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a' + +// Python wheels +mx_pip = 'build/*.whl' + // for scala build, need to pass extra libs when run with dist_kvstore mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a' // mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default. @@ -89,6 +93,30 @@ def python3_gpu_ut_nocudnn(docker_container_name) { } } +def deploy_docs() { + parallel 'Docs': { + node(NODE_LINUX_CPU) { + ws('workspace/docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run('ubuntu_cpu', 'deploy_docs', false) + sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" + } + } + } + }, + 'Julia docs': { + node(NODE_LINUX_CPU) { + ws('workspace/julia-docs') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'deploy_jl_docs', false) + } + } + } + } +} + node('mxnetlinux-cpu') { // Loading the utilities requires a node context unfortunately checkout scm @@ -124,7 +152,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('centos7_cpu', 'build_centos7_cpu', false) - utils.pack_lib('centos7_cpu', mx_lib, true) + utils.pack_lib('centos7_cpu', mx_dist_lib, true) } } } @@ -390,6 +418,7 @@ core_logic: { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() utils.docker_run('armv7', 'build_armv7', false) + utils.pack_lib('armv7', mx_pip) } } } @@ -669,6 +698,17 @@ core_logic: { } } }, + 'Scala: CentOS CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-scala-centos7-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('centos7_cpu', mx_dist_lib, true) + utils.docker_run('centos7_cpu', 'unittest_centos7_cpu_scala', false) + utils.publish_test_coverage() + } + } + } + }, 'Clojure: CPU': { node(NODE_LINUX_CPU) { ws('workspace/ut-clojure-cpu') { @@ -746,6 +786,16 @@ core_logic: { } } }, + 'Julia 0.6: CPU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-julia06-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('cpu', mx_lib) + utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06', false) + } + } + } + }, 'Python 2: CPU Win':{ node(NODE_WINDOWS_CPU) { @@ -907,19 +957,21 @@ core_logic: { } } } + }, + 'ARMv7 QEMU': { + node(NODE_LINUX_CPU) { + ws('workspace/ut-armv7-qemu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.unpack_and_init('armv7', mx_pip) + sh "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu" + } + } + } } } stage('Deploy') { - node(NODE_LINUX_CPU) { - ws('workspace/docs') { - timeout(time: max_time, unit: 'MINUTES') { - utils.init_git() - utils.docker_run('ubuntu_cpu', 'deploy_docs', false) - sh "ci/other/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}" - } - } - } + deploy_docs() } } , diff --git a/MKLDNN_README.md b/MKLDNN_README.md index 43cced49ed0d..2618d23388e7 100644 --- a/MKLDNN_README.md +++ b/MKLDNN_README.md @@ -1,6 +1,10 @@ # Build/Install MXNet with MKL-DNN -Building MXNet with [Intel MKL-DNN](https://github.com/intel/mkl-dnn) will gain better performance when using Intel Xeon CPUs for training and inference. The improvement of performance can be seen in this [page](https://mxnet.incubator.apache.org/faq/perf.html#intel-cpu). Below are instructions for linux, MacOS and Windows platform. +A better training and inference perforamce are expected to achieved on Intel-Architecture CPUs with MXNET built with [Intel MKL-DNN](https://github.com/intel/mkl-dnn) on multiple operating system, including Linux, Windows and MacOS. +In the following sections, you will find building instructions for MXNET with Intel MKL-DNN on Linux, MacOS and Windows. + +The detailed performance data collected on Intel Xeon CPU with MXNET built with Intel MKL-DNN can be found at [here](https://mxnet.incubator.apache.org/faq/perf.html#intel-cpu). +

Contents

@@ -9,7 +13,9 @@ Building MXNet with [Intel MKL-DNN](https://github.com/intel/mkl-dnn) will gain * [3. Windows](#3) * [4. Verify MXNet with python](#4) * [5. Enable MKL BLAS](#5) -* [6. Support](#6) +* [6. Enable graph optimization](#6) +* [7. Quantization](#7) +* [8. Support](#8)

Linux

@@ -36,7 +42,7 @@ cd incubator-mxnet make -j $(nproc) USE_OPENCV=1 USE_MKLDNN=1 USE_BLAS=mkl USE_INTEL_PATH=/opt/intel ``` -If you don't have full [MKL](https://software.intel.com/en-us/intel-mkl) library installed, you can use OpenBLAS by setting `USE_BLAS=openblas`. +If you don't have the full [MKL](https://software.intel.com/en-us/intel-mkl) library installation, you might use OpenBLAS as the blas library, by setting USE_BLAS=openblas.

MacOS

@@ -45,7 +51,7 @@ If you don't have full [MKL](https://software.intel.com/en-us/intel-mkl) library Install the dependencies, required for MXNet, with the following commands: - [Homebrew](https://brew.sh/) -- gcc (clang in macOS does not support OpenMP) +- llvm (clang in macOS does not support OpenMP) - OpenCV (for computer vision operations) ``` @@ -59,8 +65,7 @@ brew install graphviz brew tap homebrew/core brew install opencv brew tap homebrew/versions -brew install gcc49 -brew link gcc49 #gcc-5 and gcc-7 also work +brew install llvm ``` ### Clone MXNet sources @@ -70,31 +75,16 @@ git clone --recursive https://github.com/apache/incubator-mxnet.git cd incubator-mxnet ``` -### Enable OpenMP for MacOS - -If you want to enable OpenMP for better performance, you should modify the Makefile in MXNet root dictionary: - -Add CFLAGS '-fopenmp' for Darwin. - -``` -ifeq ($(USE_OPENMP), 1) -# ifneq ($(UNAME_S), Darwin) - CFLAGS += -fopenmp -# endif -endif -``` - ### Build MXNet with MKL-DNN ``` -make -j $(sysctl -n hw.ncpu) CC=gcc-4.9 CXX=g++-4.9 USE_OPENCV=0 USE_OPENMP=1 USE_MKLDNN=1 USE_BLAS=apple USE_PROFILER=1 +LIBRARY_PATH=$(brew --prefix llvm)/lib/ make -j $(sysctl -n hw.ncpu) CC=$(brew --prefix llvm)/bin/clang++ CXX=$(brew --prefix llvm)/bin/clang++ USE_OPENCV=1 USE_OPENMP=1 USE_MKLDNN=1 USE_BLAS=apple USE_PROFILER=1 ``` -*Note: Temporarily disable OPENCV.* -

Windows

-We recommend to build and install MXNet yourself using [Microsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/), or you can also try experimentally the latest [Microsoft Visual Studio 2017](https://www.visualstudio.com/downloads/). +On Windows, you can use [Micrsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/) and [Microsoft Visual Studio 2017](https://www.visualstudio.com/downloads/) to compile MXNET with Intel MKL-DNN. +[Micrsoft Visual Studio 2015](https://www.visualstudio.com/vs/older-downloads/) is recommended. **Visual Studio 2015** @@ -228,11 +218,11 @@ o = exe.outputs[0] t = o.asnumpy() ``` -You can open the `MKLDNN_VERBOSE` flag by setting environment variable: +More detailed debugging and profiling information can be logged by setting the environment variable 'MKLDNN_VERBOSE': ``` export MKLDNN_VERBOSE=1 ``` -Then by running above code snippet, you probably will get the following output message which means `convolution` and `reorder` primitive from MKL-DNN are called. Layout information and primitive execution performance are also demonstrated in the log message. +For example, by running above code snippet, the following debugging logs providing more insights on MKL-DNN primitives `convolution` and `reorder`. That includes: Memory layout, infer shape and the time cost of primitive execution. ``` mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_nchw out:f32_nChw16c,num:1,32x32x256x256,6.47681 mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_oihw out:f32_OIhw16i16o,num:1,32x32x3x3,0.0429688 @@ -243,9 +233,9 @@ mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_nChw16c out:f32_nchw,num:1,32x3

Enable MKL BLAS

-To make it convenient for customers, Intel introduced a new license called [Intel® Simplified license](https://software.intel.com/en-us/license/intel-simplified-software-license) that allows to redistribute not only dynamic libraries but also headers, examples and static libraries. - -Installing and enabling the full MKL installation enables MKL support for all operators under the linalg namespace. +With MKL BLAS, the performace is expected to furtherly improved with variable range depending on the computation load of the models. +You can redistribute not only dynamic libraries but also headers, examples and static libraries on accepting the license [Intel® Simplified license](https://software.intel.com/en-us/license/intel-simplified-software-license). +Installing the full MKL installation enables MKL support for all operators under the linalg namespace. 1. Download and install the latest full MKL version following instructions on the [intel website.](https://software.intel.com/en-us/mkl) @@ -292,10 +282,32 @@ MKL_VERBOSE Intel(R) MKL 2018.0 Update 1 Product build 20171007 for Intel(R) 64 MKL_VERBOSE SGEMM(T,N,12,10,8,0x7f7f927b1378,0x1bc2140,8,0x1ba8040,8,0x7f7f927b1380,0x7f7f7400a280,12) 8.93ms CNR:OFF Dyn:1 FastMM:1 TID:0 NThr:40 WDiv:HOST:+0.000 ``` -

Next Steps and Support

+

Enable graph optimization

+ +Graph optimization by subgraph feature are available in master branch. You can build from source and then use below command to enable this *experimental* feature for better performance: + +``` +export MXNET_SUBGRAPH_BACKEND=MKLDNN +``` + +This limitations of this experimental feature are: + +- Use this feature only for inference. When training, be sure to turn the feature off by unsetting the `MXNET_SUBGRAPH_BACKEND` environment variable. + +- This feature will only run on the CPU, even if you're using a GPU-enabled build of MXNet. + +- [MXNet Graph Optimization and Quantization Technical Information and Performance Details](https://cwiki.apache.org/confluence/display/MXNET/MXNet+Graph+Optimization+and+Quantization+based+on+subgraph+and+MKL-DNN). + +

Quantization and Inference with INT8

+ +Benefiting from Intel® MKL-DNN, MXNet built with Intel® MKL-DNN brings outstanding performance improvement on quantization and inference with INT8 Intel® CPU Platform on Intel® Xeon® Scalable Platform. + +- [CNN Quantization Examples](https://github.com/apache/incubator-mxnet/tree/master/example/quantization). + +

Next Steps and Support

-- For questions or support specific to MKL, visit the [Intel MKL](https://software.intel.com/en-us/mkl) +- For questions or support specific to MKL, visit the [Intel MKL](https://software.intel.com/en-us/mkl) website. -- For questions or support specific to MKL, visit the [Intel MKLDNN](https://github.com/intel/mkl-dnn) +- For questions or support specific to MKL, visit the [Intel MKLDNN](https://github.com/intel/mkl-dnn) website. -- If you find bugs, please open an issue on GitHub for [MXNet with MKL](https://github.com/apache/incubator-mxnet/labels/MKL) or [MXNet with MKLDNN](https://github.com/apache/incubator-mxnet/labels/MKLDNN) +- If you find bugs, please open an issue on GitHub for [MXNet with MKL](https://github.com/apache/incubator-mxnet/labels/MKL) or [MXNet with MKLDNN](https://github.com/apache/incubator-mxnet/labels/MKLDNN). diff --git a/Makefile b/Makefile index 69b25daa9d1c..ad7f0ff3485f 100644 --- a/Makefile +++ b/Makefile @@ -66,8 +66,8 @@ $(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.") endif ifeq ($(USE_MKLDNN), 1) - MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/install - MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/install + MKLDNNROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install + MKLROOT = $(ROOTDIR)/3rdparty/mkldnn/build/install export USE_MKLML = 1 endif @@ -145,9 +145,7 @@ else endif ifeq ($(USE_OPENMP), 1) - ifneq ($(UNAME_S), Darwin) - CFLAGS += -fopenmp - endif + CFLAGS += -fopenmp endif ifeq ($(USE_NNPACK), 1) @@ -231,25 +229,19 @@ endif # gperftools malloc library (tcmalloc) ifeq ($(USE_GPERFTOOLS), 1) -FIND_LIBFILE=$(wildcard $(USE_GPERFTOOLS_PATH)/libtcmalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard $(USE_GPERFTOOLS_PATH)/libtcmalloc.so) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /lib/libtcmalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /lib/libtcmalloc.so) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/libtcmalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/libtcmalloc.so) +FIND_LIBFILEEXT=so +ifeq ($(USE_GPERFTOOLS_STATIC), 1) +FIND_LIBFILEEXT=a +endif +FIND_LIBFILE=$(wildcard $(USE_GPERFTOOLS_PATH)/libtcmalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/local/lib/libtcmalloc.a) +FIND_LIBFILE=$(wildcard /lib/libtcmalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/local/lib/libtcmalloc.so) +FIND_LIBFILE=$(wildcard /usr/lib/libtcmalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib64/libtcmalloc.a) +FIND_LIBFILE=$(wildcard /usr/local/lib/libtcmalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib64/libtcmalloc.so) +FIND_LIBFILE=$(wildcard /usr/lib64/libtcmalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) USE_GPERFTOOLS=0 endif @@ -257,11 +249,6 @@ endif endif endif endif -endif -endif -endif -endif -endif ifeq ($(USE_GPERFTOOLS), 1) CFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free LDFLAGS += $(FIND_LIBFILE) @@ -270,29 +257,21 @@ endif # jemalloc malloc library (if not using gperftools) else ifeq ($(USE_JEMALLOC), 1) -FIND_LIBFILE=$(wildcard $(USE_JEMALLOC_PATH)/libjemalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard $(USE_JEMALLOC_PATH)/libjemalloc.so) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /lib/libjemalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /lib/libjemalloc.so) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/libjemalloc.a) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/libjemalloc.so) -ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/local/lib/libjemalloc.a) +FIND_LIBFILEEXT=so +ifeq ($(USE_JEMALLOC_STATIC), 1) +FIND_LIBFILEEXT=a +endif +FIND_LIBFILE=$(wildcard $(USE_JEMALLOC_PATH)/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/local/lib/libjemalloc.so) +FIND_LIBFILE=$(wildcard /lib/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/x86_64-linux-gnu/libjemalloc.a) +FIND_LIBFILE=$(wildcard /usr/lib/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib/x86_64-linux-gnu/libjemalloc.so) +FIND_LIBFILE=$(wildcard /usr/local/lib/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib64/libjemalloc.a) +FIND_LIBFILE=$(wildcard /usr/lib/x86_64-linux-gnu/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) -FIND_LIBFILE=$(wildcard /usr/lib64/libjemalloc.so) +FIND_LIBFILE=$(wildcard /usr/lib64/libjemalloc.$(FIND_LIBFILEEXT)) ifeq (,$(FIND_LIBFILE)) USE_JEMALLOC=0 endif @@ -301,12 +280,6 @@ endif endif endif endif -endif -endif -endif -endif -endif -endif ifeq ($(USE_JEMALLOC), 1) CFLAGS += -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc \ -fno-builtin-free -DUSE_JEMALLOC @@ -611,19 +584,20 @@ rpkg: cp -rf 3rdparty/tvm/nnvm/include/* R-package/inst/include Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}" Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)" - echo "import(Rcpp)" > R-package/NAMESPACE - echo "import(methods)" >> R-package/NAMESPACE + cp R-package/dummy.NAMESPACE R-package/NAMESPACE + echo "import(Rcpp)" >> R-package/NAMESPACE R CMD INSTALL R-package - Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package')" - Rscript -e "if (!require('roxygen2')||packageVersion('roxygen2')!= '5.0.1'){\ - devtools::install_version('roxygen2',version='5.0.1',\ - repo='https://cloud.r-project.org/',quiet=TRUE)}" - Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package')" + Rscript -e "if (!require('roxygen2')||packageVersion('roxygen2') < '5.0.1'){\ + devtools::install_version('roxygen2',version='5.0.1',\ + repos='https://cloud.r-project.org/',quiet=TRUE)}" + Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package'); warnings()" + rm R-package/NAMESPACE + Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package'); warnings()" R CMD INSTALL R-package rpkgtest: Rscript -e 'require(testthat);res<-test_dir("R-package/tests/testthat");if(!testthat:::all_passed(res)){stop("Test failures", call. = FALSE)}' - Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file("r-package_coverage.json");writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' + Rscript -e 'res<-covr:::package_coverage("R-package");fileConn<-file(paste("r-package_coverage_",toString(runif(1)),".json"));writeLines(covr:::to_codecov(res), fileConn);close(fileConn)' scalaclean: (cd $(ROOTDIR)/scala-package; \ @@ -684,10 +658,13 @@ scaladeploy: jnilint: 3rdparty/dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src -ifneq ($(EXTRA_OPERATORS),) -clean: cyclean $(EXTRA_PACKAGES_CLEAN) - $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ +rclean: + $(RM) -r R-package/src/image_recordio.h R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz + +ifneq ($(EXTRA_OPERATORS),) +clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN) + $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - @@ -695,9 +672,8 @@ clean: cyclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r $(patsubst %, %/*.d, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.d, $(EXTRA_OPERATORS)) $(RM) -r $(patsubst %, %/*.o, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.o, $(EXTRA_OPERATORS)) else -clean: mkldnn_clean cyclean testclean $(EXTRA_PACKAGES_CLEAN) - $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ - R-package/inst R-package/src/image_recordio.h R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz +clean: rclean mkldnn_clean cyclean testclean $(EXTRA_PACKAGES_CLEAN) + $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - diff --git a/R-package/.Rbuildignore b/R-package/.Rbuildignore index 423105a7932e..d0da835e2c00 100644 --- a/R-package/.Rbuildignore +++ b/R-package/.Rbuildignore @@ -3,5 +3,6 @@ src/*.so$ \.dll$ ^.*\.Rproj$ ^\.Rproj\.user$ +dummy.NAMESPACE README.md diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 0990c63e0272..244219b9ee86 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -32,4 +32,4 @@ Depends: R (>= 3.3.0) LinkingTo: Rcpp VignetteBuilder: knitr -RoxygenNote: 5.0.1 +RoxygenNote: 6.1.0 diff --git a/R-package/R/metric.R b/R-package/R/metric.R index b29d0ea487f0..8715ccfb3a1a 100644 --- a/R-package/R/metric.R +++ b/R-package/R/metric.R @@ -59,7 +59,8 @@ mx.metric.top_k_accuracy <- mx.metric.custom("top_k_accuracy", function(label, p #' #' @export mx.metric.mse <- mx.metric.custom("mse", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) res <- mx.nd.mean(mx.nd.square(label-pred)) return(as.array(res)) }) @@ -68,7 +69,8 @@ mx.metric.mse <- mx.metric.custom("mse", function(label, pred) { #' #' @export mx.metric.rmse <- mx.metric.custom("rmse", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) res <- mx.nd.sqrt(mx.nd.mean(mx.nd.square(label-pred))) return(as.array(res)) }) @@ -77,7 +79,8 @@ mx.metric.rmse <- mx.metric.custom("rmse", function(label, pred) { #' #' @export mx.metric.mae <- mx.metric.custom("mae", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) res <- mx.nd.mean(mx.nd.abs(label-pred)) return(as.array(res)) }) @@ -86,7 +89,8 @@ mx.metric.mae <- mx.metric.custom("mae", function(label, pred) { #' #' @export mx.metric.rmsle <- mx.metric.custom("rmsle", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) res <- mx.nd.sqrt(mx.nd.mean(mx.nd.square(mx.nd.log1p(pred) - mx.nd.log1p(label)))) return(as.array(res)) }) @@ -95,13 +99,13 @@ mx.metric.rmsle <- mx.metric.custom("rmsle", function(label, pred) { #' #' @export mx.metric.Perplexity <- mx.metric.custom("Perplexity", function(label, pred, mask_element = -1) { - + label <- mx.nd.reshape(label, shape = -1) pred_probs <- mx.nd.pick(data = pred, index = label, axis = 1) - + mask <- label != mask_element mask_length <- mx.nd.sum(mask) - + NLL <- -mx.nd.sum(mx.nd.log(pred_probs) * mask) / mask_length res <- mx.nd.exp(NLL) return(as.array(res)) @@ -111,7 +115,8 @@ mx.metric.Perplexity <- mx.metric.custom("Perplexity", function(label, pred, mas #' #' @export mx.metric.logloss <- mx.metric.custom("logloss", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) pred <- mx.nd.clip(pred, a_min = 1e-15, a_max = 1-1e-15) res <- -mx.nd.mean(label * mx.nd.log(pred) + (1-label) * mx.nd.log(1-pred)) return(as.array(res)) @@ -121,7 +126,8 @@ mx.metric.logloss <- mx.metric.custom("logloss", function(label, pred) { #' #' @export mx.metric.logistic_acc <- mx.metric.custom("accuracy", function(label, pred) { - pred <- mx.nd.reshape(pred, shape = 0) > 0.5 + label <- mx.nd.reshape(label, shape = -1) + pred <- mx.nd.reshape(pred, shape = -1) > 0.5 res <- mx.nd.mean(label == pred) return(as.array(res)) }) diff --git a/R-package/R/zzz.R b/R-package/R/zzz.R index 6c63081af597..db7573ab2553 100644 --- a/R-package/R/zzz.R +++ b/R-package/R/zzz.R @@ -34,8 +34,8 @@ NULL .onLoad <- function(libname, pkgname) { # Require methods for older versions of R require(methods) - library.dynam("libmxnet", pkgname, libname, local=FALSE) - library.dynam("mxnet", pkgname, libname) + tryCatch(library.dynam("libmxnet", pkgname, libname, local=FALSE), error = function(e) { print('Loading local: inst/libs/libmxnet.so'); dyn.load("R-package/inst/libs/libmxnet.so", local=FALSE) }) + tryCatch(library.dynam("mxnet", pkgname, libname), error = function(e) { print('Loading local: src/mxnet.so'); dyn.load("R-package/src/mxnet.so") }) loadModule("mxnet", TRUE) init.symbol.methods() init.context.default() diff --git a/R-package/dummy.NAMESPACE b/R-package/dummy.NAMESPACE new file mode 100644 index 000000000000..6225fbf702e2 --- /dev/null +++ b/R-package/dummy.NAMESPACE @@ -0,0 +1,16 @@ +# Generated by roxygen2: do not edit by hand + +import(Rcpp) +import(methods) +importFrom(DiagrammeR,add_global_graph_attrs) +importFrom(DiagrammeR,create_edge_df) +importFrom(DiagrammeR,create_graph) +importFrom(DiagrammeR,create_node_df) +importFrom(DiagrammeR,render_graph) +importFrom(jsonlite,fromJSON) +importFrom(magrittr,"%>%") +importFrom(stringr,str_extract_all) +importFrom(stringr,str_replace_all) +importFrom(stringr,str_replace_na) +importFrom(stringr,str_trim) +importFrom(visNetwork,visHierarchicalLayout) diff --git a/R-package/vignettes/CharRnnModel.Rmd b/R-package/vignettes/CharRnnModel.Rmd index 9d3fd5c14786..3c302bb5bf10 100644 --- a/R-package/vignettes/CharRnnModel.Rmd +++ b/R-package/vignettes/CharRnnModel.Rmd @@ -1,32 +1,28 @@ -# Char RNN Example +# Character-level Language Model using RNN -## Load Data +This tutorial will demonstrate creating a language model using a character level RNN model using MXNet-R package. You will need the following R packages to run this tutorial - + - readr + - stringr + - stringi + - mxnet -First of all, load in the data and preprocess it. +We will use the [tinyshakespeare](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare) dataset to build this model. -```{r} -require(mxnet) -``` -Set basic network parameters. - -```{r} -batch.size = 32 -seq.len = 32 -num.hidden = 16 -num.embed = 16 -num.lstm.layer = 1 -num.round = 1 -learning.rate= 0.1 -wd=0.00001 -clip_gradient=1 -update.period = 1 +```R +library("readr") +library("stringr") +library("stringi") +library("mxnet") ``` -Download the data. +## Preprocess and prepare the data + +Download the data: + -```{r} +```R download.data <- function(data_dir) { dir.create(data_dir, showWarnings = FALSE) if (!file.exists(paste0(data_dir,'input.txt'))) { @@ -36,237 +32,262 @@ download.data <- function(data_dir) { } ``` -Make dictionary from text. - -```{r} -make.dict <- function(text, max.vocab=10000) { - text <- strsplit(text, '') - dic <- list() - idx <- 1 - for (c in text[[1]]) { - if (!(c %in% names(dic))) { - dic[[c]] <- idx - idx <- idx + 1 - } - } - if (length(dic) == max.vocab - 1) - dic[["UNKNOWN"]] <- idx - cat(paste0("Total unique char: ", length(dic), "\n")) - return (dic) -} -``` - -Transfer text into data feature. - -```{r} -make.data <- function(file.path, seq.len=32, max.vocab=10000, dic=NULL) { - fi <- file(file.path, "r") - text <- paste(readLines(fi), collapse="\n") - close(fi) - - if (is.null(dic)) - dic <- make.dict(text, max.vocab) - lookup.table <- list() - for (c in names(dic)) { - idx <- dic[[c]] - lookup.table[[idx]] <- c - } - - char.lst <- strsplit(text, '')[[1]] - num.seq <- as.integer(length(char.lst) / seq.len) - char.lst <- char.lst[1:(num.seq * seq.len)] - data <- array(0, dim=c(seq.len, num.seq)) - idx <- 1 - for (i in 1:num.seq) { - for (j in 1:seq.len) { - if (char.lst[idx] %in% names(dic)) - data[j, i] <- dic[[ char.lst[idx] ]]-1 - else { - data[j, i] <- dic[["UNKNOWN"]]-1 - } - idx <- idx + 1 - } - } - return (list(data=data, dic=dic, lookup.table=lookup.table)) +Next we transform the test into feature vectors that is fed into the RNN model. The `make_data` function reads the dataset, cleans it of any non-alphanumeric characters, splits it into individual characters and groups it into sequences of length `seq.len`. + + +```R +make_data <- function(path, seq.len = 32, dic=NULL) { + + text_vec <- read_file(file = path) + text_vec <- stri_enc_toascii(str = text_vec) + text_vec <- str_replace_all(string = text_vec, pattern = "[^[:print:]]", replacement = "") + text_vec <- strsplit(text_vec, '') %>% unlist + + if (is.null(dic)) { + char_keep <- sort(unique(text_vec)) + } else char_keep <- names(dic)[!dic == 0] + + # Remove terms not part of dictionary + text_vec <- text_vec[text_vec %in% char_keep] + + # Build dictionary + dic <- 1:length(char_keep) + names(dic) <- char_keep + + # reverse dictionary + rev_dic <- names(dic) + names(rev_dic) <- dic + + # Adjust by -1 to have a 1-lag for labels + num.seq <- (length(text_vec) - 1) %/% seq.len + + features <- dic[text_vec[1:(seq.len * num.seq)]] + labels <- dic[text_vec[1:(seq.len * num.seq)+1]] + + features_array <- array(features, dim = c(seq.len, num.seq)) + labels_array <- array(labels, dim = c(seq.len, num.seq)) + + return (list(features_array = features_array, labels_array = labels_array, dic = dic, rev_dic = rev_dic)) } -``` -Move tail text. - -```{r} -drop.tail <- function(X, batch.size) { - shape <- dim(X) - nstep <- as.integer(shape[2] / batch.size) - return (X[, 1:(nstep * batch.size)]) -} -``` -Get the label of X - -```{r} -get.label <- function(X) { - label <- array(0, dim=dim(X)) - d <- dim(X)[1] - w <- dim(X)[2] - for (i in 0:(w-1)) { - for (j in 1:d) { - label[i*d+j] <- X[(i*d+j)%%(w*d)+1] - } - } - return (label) -} +seq.len <- 100 +data_prep <- make_data(path = "input.txt", seq.len = seq.len, dic = NULL) ``` -Get training data and eval data +Fetch the features and labels for training the model, and split the data into training and evaluation in 9:1 ratio. -```{r} -download.data("./data/") -ret <- make.data("./data/input.txt", seq.len=seq.len) -X <- ret$data -dic <- ret$dic -lookup.table <- ret$lookup.table +```R +X <- data_prep$features_array +Y <- data_prep$labels_array +dic <- data_prep$dic +rev_dic <- data_prep$rev_dic vocab <- length(dic) -shape <- dim(X) +samples <- tail(dim(X), 1) train.val.fraction <- 0.9 -size <- shape[2] -X.train.data <- X[, 1:as.integer(size * train.val.fraction)] -X.val.data <- X[, -(1:as.integer(size * train.val.fraction))] -X.train.data <- drop.tail(X.train.data, batch.size) -X.val.data <- drop.tail(X.val.data, batch.size) +X.train.data <- X[, 1:as.integer(samples * train.val.fraction)] +X.val.data <- X[, -(1:as.integer(samples * train.val.fraction))] -X.train.label <- get.label(X.train.data) -X.val.label <- get.label(X.val.data) - -X.train <- list(data=X.train.data, label=X.train.label) -X.val <- list(data=X.val.data, label=X.val.label) -``` +X.train.label <- Y[, 1:as.integer(samples * train.val.fraction)] +X.val.label <- Y[, -(1:as.integer(samples * train.val.fraction))] -## Training Model - - -In `mxnet`, we have a function called `mx.lstm` so that users can build a general lstm model. - -```{r} -model <- mx.lstm(X.train, X.val, - ctx=mx.cpu(), - num.round=num.round, - update.period=update.period, - num.lstm.layer=num.lstm.layer, - seq.len=seq.len, - num.hidden=num.hidden, - num.embed=num.embed, - num.label=vocab, - batch.size=batch.size, - input.size=vocab, - initializer=mx.init.uniform(0.1), - learning.rate=learning.rate, - wd=wd, - clip_gradient=clip_gradient) +train_buckets <- list("100" = list(data = X.train.data, label = X.train.label)) +eval_buckets <- list("100" = list(data = X.val.data, label = X.val.label)) +train_buckets <- list(buckets = train_buckets, dic = dic, rev_dic = rev_dic) +eval_buckets <- list(buckets = eval_buckets, dic = dic, rev_dic = rev_dic) ``` -## Inference from model +Create iterators for training and evaluation datasets. -Some helper functions for random sample. +```R +vocab <- length(eval_buckets$dic) -```{r} -cdf <- function(weights) { - total <- sum(weights) - result <- c() - cumsum <- 0 - for (w in weights) { - cumsum <- cumsum+w - result <- c(result, cumsum / total) - } - return (result) -} +batch.size <- 32 -search.val <- function(cdf, x) { - l <- 1 - r <- length(cdf) - while (l <= r) { - m <- as.integer((l+r)/2) - if (cdf[m] < x) { - l <- m+1 - } else { - r <- m-1 - } - } - return (l) -} +train.data <- mx.io.bucket.iter(buckets = train_buckets$buckets, batch.size = batch.size, + data.mask.element = 0, shuffle = TRUE) -choice <- function(weights) { - cdf.vals <- cdf(as.array(weights)) - x <- runif(1) - idx <- search.val(cdf.vals, x) - return (idx) -} +eval.data <- mx.io.bucket.iter(buckets = eval_buckets$buckets, batch.size = batch.size, + data.mask.element = 0, shuffle = FALSE) ``` -We can use random output or fixed output by choosing largest probability. - -```{r} -make.output <- function(prob, sample=FALSE) { - if (!sample) { - idx <- which.max(as.array(prob)) - } - else { - idx <- choice(prob) - } - return (idx) - +## Train the Model + + +This model is a multi-layer RNN for sampling from character-level language models. It has a one-to-one model configuration since for each character, we want to predict the next one. For a sequence of length 100, there are also 100 labels, corresponding the same sequence of characters but offset by a position of +1. The parameters output_last_state is set to TRUE in order to access the state of the RNN cells when performing inference. + + +```R +rnn_graph_one_one <- rnn.graph(num_rnn_layer = 3, + num_hidden = 96, + input_size = vocab, + num_embed = 64, + num_decode =vocab, + dropout = 0.2, + ignore_label = 0, + cell_type = "lstm", + masking = F, + output_last_state = T, + loss_output = "softmax", + config = "one-to-one") + +graph.viz(rnn_graph_one_one, type = "graph", direction = "LR", + graph.height.px = 180, shape=c(100, 64)) + +devices <- mx.cpu() + +initializer <- mx.init.Xavier(rnd_type = "gaussian", factor_type = "avg", magnitude = 3) + +optimizer <- mx.opt.create("adadelta", rho = 0.9, eps = 1e-5, wd = 1e-8, + clip_gradient = 5, rescale.grad = 1/batch.size) + +logger <- mx.metric.logger() +epoch.end.callback <- mx.callback.log.train.metric(period = 1, logger = logger) +batch.end.callback <- mx.callback.log.train.metric(period = 50) + +mx.metric.custom_nd <- function(name, feval) { + init <- function() { + c(0, 0) + } + update <- function(label, pred, state) { + m <- feval(label, pred) + state <- c(state[[1]] + 1, state[[2]] + m) + return(state) + } + get <- function(state) { + list(name = name, value = (state[[2]]/state[[1]])) + } + ret <- (list(init = init, update = update, get = get)) + class(ret) <- "mx.metric" + return(ret) } -``` -In `mxnet`, we have a function called `mx.lstm.inference` so that users can build a inference from lstm model and then use function `mx.lstm.forward` to get forward output from the inference. - -```{r} -infer.model <- mx.lstm.inference(num.lstm.layer=num.lstm.layer, - input.size=vocab, - num.hidden=num.hidden, - num.embed=num.embed, - num.label=vocab, - arg.params=model$arg.params, - ctx=mx.cpu()) +mx.metric.Perplexity <- mx.metric.custom_nd("Perplexity", function(label, pred) { + label <- mx.nd.reshape(label, shape = -1) + label_probs <- as.array(mx.nd.choose.element.0index(pred, label)) + batch <- length(label_probs) + NLL <- -sum(log(pmax(1e-15, as.array(label_probs)))) / batch + Perplexity <- exp(NLL) + return(Perplexity) +}) + +model <- mx.model.buckets(symbol = rnn_graph_one_one, + train.data = train.data, eval.data = eval.data, + num.round = 20, ctx = devices, verbose = TRUE, + metric = mx.metric.Perplexity, + initializer = initializer, optimizer = optimizer, + batch.end.callback = NULL, + epoch.end.callback = epoch.end.callback) + +mx.model.save(model, prefix = "one_to_one_seq_model", iteration = 20) ``` -Generate a sequence of 75 chars using function `mx.lstm.forward`. - -```{r} -start <- 'a' -seq.len <- 75 -random.sample <- TRUE - -last.id <- dic[[start]] -out <- "a" -for (i in (1:(seq.len-1))) { - input <- c(last.id-1) - ret <- mx.lstm.forward(infer.model, input, FALSE) - infer.model <- ret$model - prob <- ret$prob - last.id <- make.output(prob, random.sample) - out <- paste0(out, lookup.table[[last.id]]) + Start training with 1 devices + [1] Train-Perplexity=13.7040474322178 + [1] Validation-Perplexity=7.94617194460922 + [2] Train-Perplexity=6.57039815554525 + [2] Validation-Perplexity=6.60806110658011 + [3] Train-Perplexity=5.65360504501481 + [3] Validation-Perplexity=6.18932770630876 + [4] Train-Perplexity=5.32547285727298 + [4] Validation-Perplexity=6.02198756798859 + [5] Train-Perplexity=5.14373631472579 + [5] Validation-Perplexity=5.8095658243407 + [6] Train-Perplexity=5.03077673487379 + [6] Validation-Perplexity=5.72582993567431 + [7] Train-Perplexity=4.94453383291536 + [7] Validation-Perplexity=5.6445258528126 + [8] Train-Perplexity=4.88635290100261 + [8] Validation-Perplexity=5.6730024536433 + [9] Train-Perplexity=4.84205646230548 + [9] Validation-Perplexity=5.50960780230982 + [10] Train-Perplexity=4.80441673535513 + [10] Validation-Perplexity=5.57002263750006 + [11] Train-Perplexity=4.77763413242626 + [11] Validation-Perplexity=5.55152143269169 + [12] Train-Perplexity=4.74937775290777 + [12] Validation-Perplexity=5.44968305351486 + [13] Train-Perplexity=4.72824849541467 + [13] Validation-Perplexity=5.50889348298234 + [14] Train-Perplexity=4.70980846981694 + [14] Validation-Perplexity=5.51473225859859 + [15] Train-Perplexity=4.69685776886122 + [15] Validation-Perplexity=5.45391985233811 + [16] Train-Perplexity=4.67837107034824 + [16] Validation-Perplexity=5.46636764997829 + [17] Train-Perplexity=4.66866961934873 + [17] Validation-Perplexity=5.44267086113492 + [18] Train-Perplexity=4.65611469144194 + [18] Validation-Perplexity=5.4290169469462 + [19] Train-Perplexity=4.64614689879405 + [19] Validation-Perplexity=5.44221549833917 + [20] Train-Perplexity=4.63764001963654 + [20] Validation-Perplexity=5.42114250842862 + + +## Inference on the Model + +We now use the saved model to do inference and sample text character by character that will look like the original training data. + + +```R +set.seed(0) +model <- mx.model.load(prefix = "one_to_one_seq_model", iteration = 20) + +internals <- model$symbol$get.internals() +sym_state <- internals$get.output(which(internals$outputs %in% "RNN_state")) +sym_state_cell <- internals$get.output(which(internals$outputs %in% "RNN_state_cell")) +sym_output <- internals$get.output(which(internals$outputs %in% "loss_output")) +symbol <- mx.symbol.Group(sym_output, sym_state, sym_state_cell) + +infer_raw <- c("Thou ") +infer_split <- dic[strsplit(infer_raw, '') %>% unlist] +infer_length <- length(infer_split) + +infer.data <- mx.io.arrayiter(data = matrix(infer_split), label = matrix(infer_split), + batch.size = 1, shuffle = FALSE) + +infer <- mx.infer.rnn.one(infer.data = infer.data, + symbol = symbol, + arg.params = model$arg.params, + aux.params = model$aux.params, + input.params = NULL, + ctx = devices) + +pred_prob <- as.numeric(as.array(mx.nd.slice.axis( + infer$loss_output, axis=0, begin = infer_length-1, end = infer_length))) +pred <- sample(length(pred_prob), prob = pred_prob, size = 1) - 1 +predict <- c(predict, pred) + +for (i in 1:200) { + + infer.data <- mx.io.arrayiter(data = as.matrix(pred), label = as.matrix(pred), + batch.size = 1, shuffle = FALSE) + + infer <- mx.infer.rnn.one(infer.data = infer.data, + symbol = symbol, + arg.params = model$arg.params, + aux.params = model$aux.params, + input.params = list(rnn.state = infer[[2]], + rnn.state.cell = infer[[3]]), + ctx = devices) + + pred_prob <- as.numeric(as.array(infer$loss_output)) + pred <- sample(length(pred_prob), prob = pred_prob, size = 1, replace = T) - 1 + predict <- c(predict, pred) } -message(out) -``` -The result: +predict_txt <- paste0(rev_dic[as.character(predict)], collapse = "") +predict_txt_tot <- paste0(infer_raw, predict_txt, collapse = "") +print(predict_txt_tot) ``` -ah not a drobl greens -Settled asing lately sistering sounted to their hight -``` - -## Other RNN models - -In `mxnet`, other RNN models like custom RNN and gru is also provided. -- For **custom RNN model**, you can replace `mx.lstm` with `mx.rnn` to train rnn model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.rnn.inference` and `mx.rnn.forward` to inference from rnn model and get forward result from the inference model. - -- For **GRU model**, you can replace `mx.lstm` with `mx.gru` to train gru model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to inference from gru model and get forward result from the inference model. + [1] "Thou NAknowledge thee my Comfort and his late she.FRIAR LAURENCE:Nothing a groats waterd forth. The lend he thank that;When she I am brother draw London: and not hear that know.BENVOLIO:How along, makes your " - \ No newline at end of file + diff --git a/R-package/vignettes/MultidimLstm.Rmd b/R-package/vignettes/MultidimLstm.Rmd new file mode 100644 index 000000000000..bd57d12f759c --- /dev/null +++ b/R-package/vignettes/MultidimLstm.Rmd @@ -0,0 +1,302 @@ +LSTM time series example +============================================= + +This tutorial shows how to use an LSTM model with multivariate data, and generate predictions from it. For demonstration purposes, we used an open source [pollution data](https://archive.ics.uci.edu/ml/datasets/Beijing+PM2.5+Data). +The tutorial is an illustration of how to use LSTM models with MXNet-R. We are forecasting the air pollution with data recorded at the US embassy in Beijing, China for five years. + +Dataset Attribution: +"PM2.5 data of US Embassy in Beijing" +We want to predict pollution levels(PM2.5 concentration) in the city given the above dataset. + +```r +Dataset description: +No: row number +year: year of data in this row +month: month of data in this row +day: day of data in this row +hour: hour of data in this row +pm2.5: PM2.5 concentration +DEWP: Dew Point +TEMP: Temperature +PRES: Pressure +cbwd: Combined wind direction +Iws: Cumulated wind speed +Is: Cumulated hours of snow +Ir: Cumulated hours of rain +``` + +We use past PM2.5 concentration, dew point, temperature, pressure, wind speed, snow and rain to predict +PM2.5 concentration levels. + +Load and pre-process the data +--------- +The first step is to load in the data and preprocess it. It is assumed that the data has been downloaded in a .csv file: data.csv from the [pollution dataset](https://archive.ics.uci.edu/ml/datasets/Beijing+PM2.5+Data) + + ```r +## Loading required packages +library("readr") +library("dplyr") +library("mxnet") +library("abind") + ``` + + + + ```r +## Preprocessing steps +Data <- read.csv(file = "/Users/khedia/Downloads/data.csv", + header = TRUE, + sep = ",") + +## Extracting specific features from the dataset as variables for time series We extract +## pollution, temperature, pressue, windspeed, snowfall and rainfall information from dataset +df <- data.frame(Data$pm2.5, + Data$DEWP, + Data$TEMP, + Data$PRES, + Data$Iws, + Data$Is, + Data$Ir) +df[is.na(df)] <- 0 + +## Now we normalise each of the feature set to a range(0,1) +df <- matrix(as.matrix(df), + ncol = ncol(df), + dimnames = NULL) + +rangenorm <- function(x) { + (x - min(x))/(max(x) - min(x)) +} +df <- apply(df, 2, rangenorm) +df <- t(df) + ``` +For using multidimesional data with MXNet-R, we need to convert training data to the form +(n_dim x seq_len x num_samples). For one-to-one RNN flavours labels should be of the form (seq_len x num_samples) while for many-to-one flavour, the labels should be of the form (1 x num_samples). Please note that MXNet-R currently supports only these two flavours of RNN. +We have used n_dim = 7, seq_len = 100, and num_samples = 430 because the dataset has 430 samples, each the length of 100 timestamps, we have seven time series as input features so each input has dimesnion of seven at each time step. + + +```r +n_dim <- 7 +seq_len <- 100 +num_samples <- 430 + +## extract only required data from dataset +trX <- df[1:n_dim, 25:(24 + (seq_len * num_samples))] + +## the label data(next PM2.5 concentration) should be one time step +## ahead of the current PM2.5 concentration +trY <- df[1, 26:(25 + (seq_len * num_samples))] + +## reshape the matrices in the format acceptable by MXNetR RNNs +trainX <- trX +dim(trainX) <- c(n_dim, seq_len, num_samples) +trainY <- trY +dim(trainY) <- c(seq_len, num_samples) +``` + + + +Defining and training the network +--------- + +```r +batch.size <- 32 + +# take first 300 samples for training - remaining 100 for evaluation +train_ids <- 1:300 +eval_ids <- 301:400 + +## The number of samples used for training and evaluation is arbitrary. I have kept aside few +## samples for testing purposes create dataiterators +train.data <- mx.io.arrayiter(data = trainX[, , train_ids, drop = F], + label = trainY[, train_ids], + batch.size = batch.size, shuffle = TRUE) + +eval.data <- mx.io.arrayiter(data = trainX[, , eval_ids, drop = F], + label = trainY[, eval_ids], + batch.size = batch.size, shuffle = FALSE) + +## Create the symbol for RNN +symbol <- rnn.graph(num_rnn_layer = 1, + num_hidden = 5, + input_size = NULL, + num_embed = NULL, + num_decode = 1, + masking = F, + loss_output = "linear", + dropout = 0.2, + ignore_label = -1, + cell_type = "lstm", + output_last_state = T, + config = "one-to-one") + + + +mx.metric.mse.seq <- mx.metric.custom("MSE", function(label, pred) { + label = mx.nd.reshape(label, shape = -1) + pred = mx.nd.reshape(pred, shape = -1) + res <- mx.nd.mean(mx.nd.square(label - pred)) + return(as.array(res)) +}) + + + +ctx <- mx.cpu() + +initializer <- mx.init.Xavier(rnd_type = "gaussian", + factor_type = "avg", + magnitude = 3) + +optimizer <- mx.opt.create("adadelta", + rho = 0.9, + eps = 1e-05, + wd = 1e-06, + clip_gradient = 1, + rescale.grad = 1/batch.size) + +logger <- mx.metric.logger() +epoch.end.callback <- mx.callback.log.train.metric(period = 10, + logger = logger) + +## train the network +system.time(model <- mx.model.buckets(symbol = symbol, + train.data = train.data, + eval.data = eval.data, + num.round = 100, + ctx = ctx, + verbose = TRUE, + metric = mx.metric.mse.seq, + initializer = initializer, + optimizer = optimizer, + batch.end.callback = NULL, + epoch.end.callback = epoch.end.callback)) +``` +Output: +``` +Start training with 1 devices +[1] Train-MSE=0.197570244409144 +[1] Validation-MSE=0.0153861071448773 +[2] Train-MSE=0.0152517843060195 +[2] Validation-MSE=0.0128299412317574 +[3] Train-MSE=0.0124418652616441 +[3] Validation-MSE=0.010827143676579 +[4] Train-MSE=0.0105128229130059 +[4] Validation-MSE=0.00940261723008007 +[5] Train-MSE=0.00914482437074184 +[5] Validation-MSE=0.00830172537826002 +[6] Train-MSE=0.00813581114634871 +[6] Validation-MSE=0.00747016374953091 +[7] Train-MSE=0.00735094994306564 +[7] Validation-MSE=0.00679832429159433 +[8] Train-MSE=0.00672049634158611 +[8] Validation-MSE=0.00623159145470709 +[9] Train-MSE=0.00620287149213254 +[9] Validation-MSE=0.00577476259786636 +[10] Train-MSE=0.00577280316501856 +[10] Validation-MSE=0.00539038667920977 +.......... +.......... +[91] Train-MSE=0.00177705133100972 +[91] Validation-MSE=0.00154715491225943 +[92] Train-MSE=0.00177639147732407 +[92] Validation-MSE=0.00154592350008897 +[93] Train-MSE=0.00177577760769054 +[93] Validation-MSE=0.00154474508599378 +[94] Train-MSE=0.0017752077546902 +[94] Validation-MSE=0.0015436161775142 +[95] Train-MSE=0.00177468206966296 +[95] Validation-MSE=0.00154253660002723 +[96] Train-MSE=0.00177419915562496 +[96] Validation-MSE=0.00154150440357625 +[97] Train-MSE=0.0017737578949891 +[97] Validation-MSE=0.00154051734716631 +[98] Train-MSE=0.00177335749613121 +[98] Validation-MSE=0.00153957353904843 +[99] Train-MSE=0.00177299699280411 +[99] Validation-MSE=0.00153867155313492 +[100] Train-MSE=0.00177267640829086 +[100] Validation-MSE=0.00153781197150238 + + user system elapsed + 21.937 1.914 13.402 +``` +We can see how mean squared error varies with epochs below. + +![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/loss.png?raw=true) + +Inference on the network +--------- +Now we have trained the network. Let's use it for inference. + +```r +## We extract the state symbols for RNN +internals <- model$symbol$get.internals() +sym_state <- internals$get.output(which(internals$outputs %in% "RNN_state")) +sym_state_cell <- internals$get.output(which(internals$outputs %in% "RNN_state_cell")) +sym_output <- internals$get.output(which(internals$outputs %in% "loss_output")) +symbol <- mx.symbol.Group(sym_output, sym_state, sym_state_cell) + +## We will predict 100 timestamps for 401st sample (first sample from the test samples) +pred_length <- 100 +predicted <- numeric() + +## We pass the 400th sample through the network to get the weights and use it for predicting next +## 100 time stamps. +data <- mx.nd.array(trainX[, , 400, drop = F]) +label <- mx.nd.array(trainY[, 400, drop = F]) + + +## We create dataiterators for the input, please note that the label is required to create +## iterator and will not be used in the inference. You can use dummy values too in the label. +infer.data <- mx.io.arrayiter(data = data, + label = label, + batch.size = 1, + shuffle = FALSE) + +infer <- mx.infer.rnn.one(infer.data = infer.data, + symbol = symbol, + arg.params = model$arg.params, + aux.params = model$aux.params, + input.params = NULL, + ctx = ctx) +## Once we get the weights for the above time series, we try to predict the next 100 steps for +## this time series, which is technically our 401st time series. + +actual <- trainY[, 401] + +## Now we iterate one by one to generate each of the next timestamp pollution values + +for (i in 1:pred_length) { + + data <- mx.nd.array(trainX[, i, 401, drop = F]) + label <- mx.nd.array(trainY[i, 401, drop = F]) + infer.data <- mx.io.arrayiter(data = data, + label = label, + batch.size = 1, + shuffle = FALSE) + ## note that we use rnn state values from previous iterations here + infer <- mx.infer.rnn.one(infer.data = infer.data, + symbol = symbol, + ctx = ctx, + arg.params = model$arg.params, + aux.params = model$aux.params, + input.params = list(rnn.state = infer[[2]], + rnn.state.cell = infer[[3]])) + + pred <- infer[[1]] + predicted <- c(predicted, as.numeric(as.array(pred))) + +} + +``` +Now predicted contains the predicted 100 values. We use ggplot to plot the actual and predicted values as shown below. + +![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/sample_401.png?raw=true) + +We also repeated the above experiments to generate the next 100 samples to 301st time series and we got the following results. + +![png](https://github.com/dmlc/web-data/blob/master/mxnet/doc/tutorials/r/images/sample_301.png?raw=true) + +The above tutorial is just for demonstration purposes and has not been tuned extensively for accuracy. + +For more tutorials on MXNet-R, head on to [MXNet-R tutorials](https://mxnet.incubator.apache.org/tutorials/r/index.html) \ No newline at end of file diff --git a/amalgamation/python/mxnet_predict.py b/amalgamation/python/mxnet_predict.py index 091bfbb1cf14..a91d3849b0d2 100644 --- a/amalgamation/python/mxnet_predict.py +++ b/amalgamation/python/mxnet_predict.py @@ -163,7 +163,7 @@ def forward(self, **kwargs): for k, v in kwargs.items(): if not isinstance(v, np.ndarray): raise ValueError("Expect numpy ndarray as input") - v = np.ascontiguousarray(v, dtype=np.float32) + v = np.asarray(v, dtype=np.float32, order='C') _check_call(_LIB.MXPredSetInput( self.handle, c_str(k), v.ctypes.data_as(mx_float_p), diff --git a/benchmark/python/control_flow/rnn.py b/benchmark/python/control_flow/rnn.py index 8a44a9cab174..08498724b1b4 100644 --- a/benchmark/python/control_flow/rnn.py +++ b/benchmark/python/control_flow/rnn.py @@ -32,6 +32,7 @@ _parser.add_argument('--benchmark', choices=["foreach", "while_loop"], required=True) _parser.add_argument('--warmup_rounds', type=int, default=20) _parser.add_argument('--test_rounds', type=int, default=100) +_parser.add_argument('--gpu', type=bool, default=False) args = _parser.parse_args() @@ -66,8 +67,7 @@ def _func(*states): loop_vars=states, max_iterations=self.length, ) - assert len(out) == 1 - return out[0] + return out def _zeros(shape, ctx): @@ -124,7 +124,9 @@ def main(): cell_types = [gluon.rnn.RNNCell, gluon.rnn.GRUCell, gluon.rnn.LSTMCell] - ctxs = [mx.cpu(0)] + [mx.gpu(i) for i in _get_gpus()] + ctxs = [mx.cpu(0)] + if args.gpu: + ctxs = ctxs + [mx.gpu(i) for i in _get_gpus()] seq_lens = [100] batch_sizes = [1, 32] hidden_dims = [512] diff --git a/benchmark/python/gluon/benchmark_gluon.py b/benchmark/python/gluon/benchmark_gluon.py new file mode 100644 index 000000000000..3dbb36404d07 --- /dev/null +++ b/benchmark/python/gluon/benchmark_gluon.py @@ -0,0 +1,164 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import mxnet.gluon.model_zoo.vision as models +import time +import logging +import argparse +import subprocess +import os +import errno + +logging.basicConfig(level=logging.INFO) +parser = argparse.ArgumentParser(description='Gluon modelzoo-based CNN performance benchmark') + +parser.add_argument('--model', type=str, default='all', + choices=['all', 'alexnet', 'densenet121', 'densenet161', + 'densenet169', 'densenet201', 'inceptionv3', 'mobilenet0.25', + 'mobilenet0.5', 'mobilenet0.75', 'mobilenet1.0', 'mobilenetv2_0.25', + 'mobilenetv2_0.5', 'mobilenetv2_0.75', 'mobilenetv2_1.0', 'resnet101_v1', + 'resnet101_v2', 'resnet152_v1', 'resnet152_v2', 'resnet18_v1', + 'resnet18_v2', 'resnet34_v1', 'resnet34_v2', 'resnet50_v1', + 'resnet50_v2', 'squeezenet1.0', 'squeezenet1.1', 'vgg11', + 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', + 'vgg19', 'vgg19_bn']) +parser.add_argument('--batch-size', type=int, default=0, + help='Batch size to use for benchmarking. Example: 32, 64, 128.' + 'By default, runs benchmark for batch sizes - 1, 32, 64, 128, 256') +parser.add_argument('--num-batches', type=int, default=10) +parser.add_argument('--gpus', type=str, default='', + help='GPU IDs to use for this benchmark task. Example: --gpus=0,1,2,3 to use 4 GPUs.' + 'By default, use CPU only.') +parser.add_argument('--type', type=str, default='inference', choices=['all', 'training', 'inference']) + +opt = parser.parse_args() + +num_batches = opt.num_batches +dry_run = 10 # use 10 iterations to warm up +batch_inf = [1, 32, 64, 128, 256] +batch_train = [1, 32, 64, 128, 256] +image_shapes = [(3, 224, 224), (3, 299, 299)] + +def score(network, batch_size, ctx): + assert (batch_size >= len(ctx)), "ERROR: batch size should not be smaller than num of GPUs." + net = models.get_model(network) + if 'inceptionv3' == network: + data_shape = [('data', (batch_size,) + image_shapes[1])] + else: + data_shape = [('data', (batch_size,) + image_shapes[0])] + + data = mx.sym.var('data') + out = net(data) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=ctx) + mod.bind(for_training = False, + inputs_need_grad = False, + data_shapes = data_shape) + mod.init_params(initializer=mx.init.Xavier(magnitude=2.)) + data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx[0]) for _, shape in mod.data_shapes] + batch = mx.io.DataBatch(data, []) + for i in range(dry_run + num_batches): + if i == dry_run: + tic = time.time() + mod.forward(batch, is_train=False) + for output in mod.get_outputs(): + output.wait_to_read() + fwd = time.time() - tic + return fwd + + +def train(network, batch_size, ctx): + assert (batch_size >= len(ctx)), "ERROR: batch size should not be smaller than num of GPUs." + net = models.get_model(network) + if 'inceptionv3' == network: + data_shape = [('data', (batch_size,) + image_shapes[1])] + else: + data_shape = [('data', (batch_size,) + image_shapes[0])] + + data = mx.sym.var('data') + out = net(data) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=ctx) + mod.bind(for_training = True, + inputs_need_grad = False, + data_shapes = data_shape) + mod.init_params(initializer=mx.init.Xavier(magnitude=2.)) + if len(ctx) > 1: + mod.init_optimizer(kvstore='device', optimizer='sgd') + else: + mod.init_optimizer(kvstore='local', optimizer='sgd') + data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx[0]) for _, shape in mod.data_shapes] + batch = mx.io.DataBatch(data, []) + for i in range(dry_run + num_batches): + if i == dry_run: + tic = time.time() + mod.forward(batch, is_train=True) + for output in mod.get_outputs(): + output.wait_to_read() + mod.backward() + mod.update() + bwd = time.time() - tic + return bwd + +if __name__ == '__main__': + runtype = opt.type + bs = opt.batch_size + + if opt.model == 'all': + networks = ['alexnet', 'densenet121', 'densenet161', 'densenet169', 'densenet201', + 'inceptionv3', 'mobilenet0.25', 'mobilenet0.5', 'mobilenet0.75', + 'mobilenet1.0', 'mobilenetv2_0.25', 'mobilenetv2_0.5', 'mobilenetv2_0.75', + 'mobilenetv2_1.0', 'resnet101_v1', 'resnet101_v2', 'resnet152_v1', 'resnet152_v2', + 'resnet18_v1', 'resnet18_v2', 'resnet34_v1', 'resnet34_v2', 'resnet50_v1', + 'resnet50_v2', 'squeezenet1.0', 'squeezenet1.1', 'vgg11', 'vgg11_bn', 'vgg13', + 'vgg13_bn', 'vgg16', 'vgg16_bn', 'vgg19', 'vgg19_bn'] + logging.info('It may take some time to run all models, ' + 'set --network to run a specific one') + else: + networks = [opt.model] + + devs = [mx.gpu(int(i)) for i in opt.gpus.split(',')] if opt.gpus.strip() else [mx.cpu()] + num_gpus = len(devs) + + for network in networks: + logging.info('network: %s', network) + logging.info('device: %s', devs) + if runtype == 'inference' or runtype == 'all': + if bs != 0: + fwd_time = score(network, bs, devs) + fps = (bs * num_batches)/fwd_time + logging.info(network + ' inference perf for BS %d is %f img/s', bs, fps) + else: + logging.info('run batchsize [1, 2, 4, 8, 16, 32] by default, ' + 'set --batch-size to run a specific one') + for batch_size in batch_inf: + fwd_time = score(network, batch_size, devs) + fps = (batch_size * num_batches) / fwd_time + logging.info(network + ' inference perf for BS %d is %f img/s', batch_size, fps) + if runtype == 'training' or runtype == 'all': + if bs != 0: + bwd_time = train(network, bs, devs) + fps = (bs * num_batches) / bwd_time + logging.info(network + ' training perf for BS %d is %f img/s', bs, fps) + else: + logging.info('run batchsize [1, 2, 4, 8, 16, 32] by default, ' + 'set --batch-size to run a specific one') + for batch_size in batch_train: + bwd_time = train(network, batch_size, devs) + fps = (batch_size * num_batches) / bwd_time + logging.info(network + ' training perf for BS %d is %f img/s', batch_size, fps) diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy index 00952d7e1be6..0a85c1498cf5 100644 --- a/ci/Jenkinsfile_utils.groovy +++ b/ci/Jenkinsfile_utils.groovy @@ -26,8 +26,11 @@ def init_git() { // retries as this will increase the amount of requests and worsen the throttling timeout(time: 15, unit: 'MINUTES') { checkout scm - sh 'git submodule update --init --recursive' sh 'git clean -xdff' + sh 'git reset --hard' + sh 'git submodule update --init --recursive' + sh 'git submodule foreach --recursive git clean -ffxd' + sh 'git submodule foreach --recursive git reset --hard' } } catch (exc) { deleteDir() @@ -45,8 +48,11 @@ def init_git_win() { // retries as this will increase the amount of requests and worsen the throttling timeout(time: 15, unit: 'MINUTES') { checkout scm - bat 'git submodule update --init --recursive' bat 'git clean -xdff' + bat 'git reset --hard' + bat 'git submodule update --init --recursive' + bat 'git submodule foreach --recursive git clean -ffxd' + bat 'git submodule foreach --recursive git reset --hard' } } catch (exc) { deleteDir() diff --git a/ci/README.md b/ci/README.md index 693087569434..5b6cc6689781 100644 --- a/ci/README.md +++ b/ci/README.md @@ -5,6 +5,8 @@ Docker containers You need docker and nvidia docker if you have a GPU. +Also you need to run `pip3 install docker` as it uses the [docker python module](https://docker-py.readthedocs.io/en/stable/containers.html#) + If you are in ubuntu an easy way to install Docker CE is executing the following script: @@ -90,3 +92,29 @@ For all builds a directory from the host system is mapped where ccache will stor compiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds significantly. You can set this directory explicitly by setting CCACHE_DIR environment variable. All ccache instances are currently set to be 10 Gigabytes max in size. + + +## Testing with QEMU +To run the unit tests under qemu: +``` +./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu +``` + +To get a shell on the container and debug issues with the emulator itself, we build the container +and then execute it interactively. We can afterwards use port 2222 on the host to connect with SSH. + + +``` +ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti mxnetci/build.test.arm_qemu +``` + +Then from another terminal: + +``` +ssh -o StrictHostKeyChecking=no -p 2222 qemu@localhost +``` + +There are two pre-configured users: `root` and `qemu` both without passwords. + + + diff --git a/ci/build.py b/ci/build.py index b7c86adda715..628f49e706c5 100755 --- a/ci/build.py +++ b/ci/build.py @@ -101,6 +101,8 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + if not registry: + registry = "mxnet_local" return "{0}/build.{1}".format(registry, platform) @@ -112,14 +114,14 @@ def get_docker_binary(use_nvidia_docker: bool) -> str: return "nvidia-docker" if use_nvidia_docker else "docker" -def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, use_cache: bool) -> str: +def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str: """ Build a container for the given platform :param platform: Platform :param docker_binary: docker binary to use (docker/nvidia-docker) :param registry: Dockerhub registry name :param num_retries: Number of retries to build the docker image - :param use_cache: will pass cache_from to docker to use the previously pulled tag + :param no_cache: pass no-cache to docker to rebuild the images :return: Id of the top level image """ tag = get_docker_tag(platform=platform, registry=registry) @@ -144,7 +146,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries: "-f", get_dockerfile(platform), "--build-arg", "USER_ID={}".format(os.getuid()), "--build-arg", "GROUP_ID={}".format(os.getgid())] - if use_cache: + if no_cache: + cmd.append("--no-cache") + elif registry: cmd.extend(["--cache-from", tag]) cmd.extend(["-t", tag, get_dockerfiles_path()]) @@ -281,7 +285,6 @@ def container_run(platform: str, # noinspection PyShadowingNames # runc is default (docker info | grep -i runtime) runtime = 'nvidia' - container = docker_client.containers.run( tag, runtime=runtime, @@ -299,52 +302,55 @@ def container_run(platform: str, {'bind': '/work/ccache', 'mode': 'rw'}, }, environment=environment) - logging.info("Started container: %s", trim_container_id(container.id)) - # Race condition: - # If the previous call is interrupted then it's possible that the container is not cleaned up - # We avoid by masking the signals temporarily - cleanup.add_container(container) - signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM}) - # - ############################# - - stream = container.logs(stream=True, stdout=True, stderr=True) - sys.stdout.flush() - for chunk in stream: - sys.stdout.buffer.write(chunk) - sys.stdout.buffer.flush() - sys.stdout.flush() - stream.close() - try: - logging.info("Waiting for status of container %s for %d s.", - trim_container_id(container.id), - container_wait_s) - wait_result = container.wait(timeout=container_wait_s) - logging.info("Container exit status: %s", wait_result) - ret = wait_result.get('StatusCode', 200) - except Exception as e: - logging.exception(e) - ret = 150 - - # Stop try: - logging.info("Stopping container: %s", trim_container_id(container.id)) - container.stop() - except Exception as e: - logging.exception(e) - ret = 151 + logging.info("Started container: %s", trim_container_id(container.id)) + # Race condition: + # If the previous call is interrupted then it's possible that the container is not cleaned up + # We avoid by masking the signals temporarily + cleanup.add_container(container) + signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM}) + # + ############################# + + stream = container.logs(stream=True, stdout=True, stderr=True) + sys.stdout.flush() + for chunk in stream: + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + sys.stdout.flush() + stream.close() + try: + logging.info("Waiting for status of container %s for %d s.", + trim_container_id(container.id), + container_wait_s) + wait_result = container.wait(timeout=container_wait_s) + logging.info("Container exit status: %s", wait_result) + ret = wait_result.get('StatusCode', 200) + except Exception as e: + logging.exception(e) + ret = 150 - # Remove - try: - logging.info("Removing container: %s", trim_container_id(container.id)) - container.remove() - except Exception as e: - logging.exception(e) - ret = 152 - cleanup.remove_container(container) - containers = docker_client.containers.list() - if containers: - logging.info("Other running containers: %s", [trim_container_id(x.id) for x in containers]) + # Stop + try: + logging.info("Stopping container: %s", trim_container_id(container.id)) + container.stop() + except Exception as e: + logging.exception(e) + ret = 151 + + # Remove + try: + logging.info("Removing container: %s", trim_container_id(container.id)) + container.remove() + except Exception as e: + logging.exception(e) + ret = 152 + cleanup.remove_container(container) + containers = docker_client.containers.list() + if containers: + logging.info("Other running containers: %s", [trim_container_id(x.id) for x in containers]) + except docker.errors.NotFound as e: + logging.info("Container was stopped before cleanup started: %s", e) return ret @@ -394,7 +400,7 @@ def main() -> int: help="platform", type=str) - parser.add_argument("--build-only", + parser.add_argument("-b", "--build-only", help="Only build the container, don't build the project", action='store_true') @@ -420,7 +426,7 @@ def main() -> int: action='store_true') parser.add_argument("-d", "--docker-registry", - help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'", + help="Dockerhub registry name to retrieve cache from.", default='mxnetci', type=str) @@ -429,10 +435,8 @@ def main() -> int: default=1, type=int) - parser.add_argument("-c", "--no-dockerhub-cache", action="store_true", - help="Disables use of --cache-from option on docker build, allowing docker" - " to use local layers for caching. If absent, we use the cache from dockerhub" - " which is the default.") + parser.add_argument("--no-cache", action="store_true", + help="passes --no-cache to docker build") parser.add_argument("command", help="command to run in the container", @@ -445,9 +449,6 @@ def main() -> int: args = parser.parse_args() - def use_cache(): - return not args.no_dockerhub_cache or under_ci() - command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) @@ -470,10 +471,10 @@ def signal_handler(signum, _): elif args.platform: platform = args.platform tag = get_docker_tag(platform=platform, registry=args.docker_registry) - if use_cache(): + if args.docker_registry: load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry, - num_retries=args.docker_build_retries, use_cache=use_cache()) + num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: logging.warning("Container was just built. Exiting due to build-only.") return 0 @@ -510,10 +511,9 @@ def signal_handler(signum, _): logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: tag = get_docker_tag(platform=platform, registry=args.docker_registry) - if use_cache(): - load_docker_cache(tag=tag, docker_registry=args.docker_registry) + load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry, - num_retries=args.docker_build_retries, use_cache=use_cache()) + num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: continue shutil.rmtree(buildir(), ignore_errors=True) diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu old mode 100755 new mode 100644 index 076ef5df911a..e2802aa2fb2b --- a/ci/docker/Dockerfile.build.centos7_cpu +++ b/ci/docker/Dockerfile.build.centos7_cpu @@ -28,6 +28,8 @@ COPY install/centos7_ccache.sh /work/ RUN /work/centos7_ccache.sh COPY install/centos7_python.sh /work/ RUN /work/centos7_python.sh +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson old mode 100755 new mode 100644 index 4be011af068e..15518cd6f22e --- a/ci/docker/Dockerfile.build.jetson +++ b/ci/docker/Dockerfile.build.jetson @@ -52,20 +52,31 @@ COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda ENV TARGET_ARCH aarch64 ENV TARGET_OS linux -# Install ARM depedencies based on Jetpack 3.2.1 -RUN JETPACK_DOWNLOAD_PREFIX=https://developer.download.nvidia.com/devzone/devcenter/mobile/jetpack_l4t/3.2.1/m8u2ki/JetPackL4T_321_b23 && \ +# Install ARM depedencies based on Jetpack 3.3 +RUN JETPACK_DOWNLOAD_PREFIX=https://developer.download.nvidia.com/devzone/devcenter/mobile/jetpack_l4t/3.3/lw.xd42/JetPackL4T_33_b39 && \ + CUDA_REPO_PREFIX=/var/cuda-repo-9-0-local && \ ARM_CUDA_INSTALLER_PACKAGE=cuda-repo-l4t-9-0-local_9.0.252-1_arm64.deb && \ - ARM_CUDNN_INSTALLER_PACKAGE=libcudnn7_7.0.5.15-1+cuda9.0_arm64.deb && \ - ARM_CUDNN_DEV_INSTALLER_PACKAGE=libcudnn7-dev_7.0.5.15-1+cuda9.0_arm64.deb && \ + ARM_CUDNN_INSTALLER_PACKAGE=libcudnn7_7.1.5.14-1+cuda9.0_arm64.deb && \ + ARM_CUDNN_DEV_INSTALLER_PACKAGE=libcudnn7-dev_7.1.5.14-1+cuda9.0_arm64.deb && \ + ARM_LICENSE_INSTALLER=cuda-license-9-0_9.0.252-1_arm64.deb && \ + ARM_CUBLAS_INSTALLER=cuda-cublas-9-0_9.0.252-1_arm64.deb && \ + ARM_NVINFER_INSTALLER_PACKAGE=libnvinfer4_4.1.3-1+cuda9.0_arm64.deb && \ + ARM_NVINFER_DEV_INSTALLER_PACKAGE=libnvinfer-dev_4.1.3-1+cuda9.0_arm64.deb && \ dpkg --add-architecture arm64 && \ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDA_INSTALLER_PACKAGE && \ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_INSTALLER_PACKAGE && \ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_DEV_INSTALLER_PACKAGE && \ + wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_NVINFER_INSTALLER_PACKAGE && \ + wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_NVINFER_DEV_INSTALLER_PACKAGE && \ dpkg -i --force-architecture $ARM_CUDA_INSTALLER_PACKAGE && \ - apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub && \ + apt-key add $CUDA_REPO_PREFIX/7fa2af80.pub && \ dpkg -i --force-architecture $ARM_CUDNN_INSTALLER_PACKAGE && \ dpkg -i --force-architecture $ARM_CUDNN_DEV_INSTALLER_PACKAGE && \ - apt update -y || true && apt install -y cuda-libraries-dev-9-0 libcudnn7-dev + dpkg -i --force-architecture $CUDA_REPO_PREFIX/$ARM_LICENSE_INSTALLER && \ + dpkg -i --force-architecture $CUDA_REPO_PREFIX/$ARM_CUBLAS_INSTALLER && \ + dpkg -i --force-architecture $ARM_NVINFER_INSTALLER_PACKAGE && \ + dpkg -i --force-architecture $ARM_NVINFER_DEV_INSTALLER_PACKAGE && \ + apt update -y || true && apt install -y cuda-libraries-dev-9-0 libcudnn7-dev libnvinfer-dev ENV PATH $PATH:/usr/local/cuda/bin ENV NVCCFLAGS "-m64" ENV CUDA_ARCH "-gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62" diff --git a/example/memcost/Makefile b/ci/docker/Dockerfile.build.test.arm_qemu similarity index 51% rename from example/memcost/Makefile rename to ci/docker/Dockerfile.build.test.arm_qemu index f6d52db597b3..68891a72c284 100644 --- a/example/memcost/Makefile +++ b/ci/docker/Dockerfile.build.test.arm_qemu @@ -1,3 +1,4 @@ +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -13,26 +14,33 @@ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations -# under the License +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work + +RUN apt-get update +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh -.PHONY: no_optimization with_inplace with_sharing with_both +COPY install/ubuntu_arm_qemu.sh /work +RUN /work/ubuntu_arm_qemu.sh -no_optimization: - @echo "Estimating the cost with no optimization..." - @NNVM_EXEC_ENABLE_INPLACE=false NNVM_EXEC_MATCH_RANGE=0 python inception_memcost.py +COPY install/ubuntu_arm_qemu_bin.sh /work +RUN /work/ubuntu_arm_qemu_bin.sh -with_inplace: - @echo "Estimating the cost with inplace optimization..." - @NNVM_EXEC_ENABLE_INPLACE=true MXNET_EXEC_MATCH_RANGE=0 python inception_memcost.py +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -with_sharing: - @echo "Estimating the cost with memory sharing ..." - @NNVM_EXEC_ENABLE_INPLACE=false python inception_memcost.py +COPY runtime_functions.sh /work/ +COPY qemu/* /work/ -with_both: - @echo "Estimating the cost with all optimizations ..." - @python inception_memcost.py +# SSH to the Qemu VM +EXPOSE 2222/tcp -forward_only: - @echo "Estimating the cost of forward only ..." - @python inception_memcost.py 'null' +CMD ["./runtime_functions.py","run_qemu_interactive"] diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu old mode 100755 new mode 100644 index f45c8da4af87..7c7e2240ee61 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -45,6 +45,9 @@ RUN /work/ubuntu_r.sh COPY install/ubuntu_perl.sh /work/ RUN /work/ubuntu_perl.sh +COPY install/ubuntu_julia.sh /work/ +RUN /work/ubuntu_julia.sh + COPY install/ubuntu_clang.sh /work/ RUN /work/ubuntu_clang.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat old mode 100755 new mode 100644 diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh new file mode 100755 index 000000000000..ea46de9b9311 --- /dev/null +++ b/ci/docker/install/centos7_scala.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +yum install -y java-1.8.0-openjdk-devel +# Build from source with Maven +wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ +alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 diff --git a/ci/docker/install/docs_requirements b/ci/docker/install/docs_requirements index 7407223b3eed..4e3ce3e55e0b 100644 --- a/ci/docker/install/docs_requirements +++ b/ci/docker/install/docs_requirements @@ -6,7 +6,7 @@ h5py==2.8.0rc1 mock==2.0.0 nose==1.3.7 nose-timer==0.7.3 -numpy<1.15.0,>=1.8.2 +numpy<=1.15.2,>=1.8.2 pylint==1.8.3 pypandoc==1.4 recommonmark==0.4.0 diff --git a/example/bi-lstm-sort/gen_data.py b/ci/docker/install/ubuntu_arm_qemu.sh old mode 100644 new mode 100755 similarity index 61% rename from example/bi-lstm-sort/gen_data.py rename to ci/docker/install/ubuntu_arm_qemu.sh index 55af1b45554a..79ab67bfdbe6 --- a/example/bi-lstm-sort/gen_data.py +++ b/ci/docker/install/ubuntu_arm_qemu.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,23 +17,21 @@ # specific language governing permissions and limitations # under the License. -import random +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image -vocab = [str(x) for x in range(100, 1000)] -sw_train = open("sort.train.txt", "w") -sw_test = open("sort.test.txt", "w") -sw_valid = open("sort.valid.txt", "w") +set -exuo pipefail -for i in range(1000000): - seq = " ".join([vocab[random.randint(0, len(vocab) - 1)] for j in range(5)]) - k = i % 50 - if k == 0: - sw_test.write(seq + "\n") - elif k == 1: - sw_valid.write(seq + "\n") - else: - sw_train.write(seq + "\n") +apt-get install -y \ + cmake \ + curl \ + wget \ + git \ + qemu \ + qemu-system-arm \ + unzip \ + bzip2 \ + vim-nox \ + toilet -sw_train.close() -sw_test.close() -sw_valid.close() +pip3 install ipython diff --git a/ci/docker/install/ubuntu_arm_qemu_bin.sh b/ci/docker/install/ubuntu_arm_qemu_bin.sh new file mode 100755 index 000000000000..d4f81185c169 --- /dev/null +++ b/ci/docker/install/ubuntu_arm_qemu_bin.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -exuo pipefail + +# +# This disk image and kernels for virtual testing with QEMU is generated with some manual OS +# installation steps with the scripts and documentation found in the ci/qemu/ folder. +# +# The image has a base Debian OS and MXNet runtime dependencies installed. +# The root password is empty and there's a "qemu" user without password. SSH access is enabled as +# well. +# +# See also: ci/qemu/README.md +# + +REMOTE="https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data" +curl -f ${REMOTE}/vda_debian_stretch.qcow2.bz2 | bunzip2 > vda.qcow2 +curl -f ${REMOTE}/vmlinuz -o vmlinuz +curl -f ${REMOTE}/initrd.img -o initrd.img + diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh index 8c13612114bd..eaa8ab9b18ac 100755 --- a/ci/docker/install/ubuntu_caffe.sh +++ b/ci/docker/install/ubuntu_caffe.sh @@ -18,6 +18,7 @@ # under the License. set -ex +apt-get update || true apt-get install -y \ libgflags-dev \ libgoogle-glog-dev \ diff --git a/ci/docker/install/ubuntu_clang.sh b/ci/docker/install/ubuntu_clang.sh index cb0f234a1c15..19aada9b3e88 100755 --- a/ci/docker/install/ubuntu_clang.sh +++ b/ci/docker/install/ubuntu_clang.sh @@ -21,6 +21,8 @@ # the whole docker cache for the image set -ex + +apt-get update || true # Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major release) wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \ diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 8a2ea30cd5cf..64f8af3e0444 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -21,7 +21,7 @@ # the whole docker cache for the image set -ex -apt-get update +apt-get update || true apt-get install -y \ apt-transport-https \ build-essential \ diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh index a709b3de7843..5dc201c15b52 100755 --- a/ci/docker/install/ubuntu_docs.sh +++ b/ci/docker/install/ubuntu_docs.sh @@ -23,6 +23,7 @@ set -ex # Install dependencies echo 'Installing dependencies...' +apt-get update || true apt-get install -y \ doxygen \ pandoc diff --git a/ci/docker/install/ubuntu_emscripten.sh b/ci/docker/install/ubuntu_emscripten.sh index e3d72caf0eee..28ede755f5c7 100755 --- a/ci/docker/install/ubuntu_emscripten.sh +++ b/ci/docker/install/ubuntu_emscripten.sh @@ -25,6 +25,7 @@ set -ex +apt-get update || true apt-get -y install nodejs git clone -b 1.38.6 https://github.com/kripken/emscripten.git diff --git a/ci/docker/install/ubuntu_gcc8.sh b/ci/docker/install/ubuntu_gcc8.sh index 0a9dc605745a..cd31f8213c1a 100755 --- a/ci/docker/install/ubuntu_gcc8.sh +++ b/ci/docker/install/ubuntu_gcc8.sh @@ -19,5 +19,5 @@ sudo add-apt-repository ppa:jonathonf/gcc-8.0 sudo add-apt-repository ppa:jonathonf/gcc-7.3 -sudo apt-get update +sudo apt-get update || true sudo apt-get install -y gcc-8 g++-8 diff --git a/ci/docker/install/ubuntu_julia.sh b/ci/docker/install/ubuntu_julia.sh new file mode 100755 index 000000000000..62013e36d8fd --- /dev/null +++ b/ci/docker/install/ubuntu_julia.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex + +export JLBINARY='julia.tar.gz' +export JULIADIR='/work/julia' +export JULIA="${JULIADIR}/bin/julia" + +mkdir -p $JULIADIR +# The julia version in Ubuntu repo is too old +# We download the tarball from the official link: +# https://julialang.org/downloads/ +wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.2-linux-x86_64.tar.gz +tar xzvf $JLBINARY -C $JULIADIR --strip 1 +rm $JLBINARY + +$JULIA -e 'versioninfo()' diff --git a/ci/docker/install/ubuntu_llvm.sh b/ci/docker/install/ubuntu_llvm.sh index 09e13d3d1ed1..afd881eae9ad 100755 --- a/ci/docker/install/ubuntu_llvm.sh +++ b/ci/docker/install/ubuntu_llvm.sh @@ -23,4 +23,5 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\ >> /etc/apt/sources.list.d/llvm.list wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - -apt-get update && apt-get install -y --force-yes llvm-5.0 \ No newline at end of file +apt-get update || true +apt-get install -y --force-yes llvm-5.0 diff --git a/ci/docker/install/ubuntu_mklml.sh b/ci/docker/install/ubuntu_mklml.sh index 7e17295f4208..862e2846403a 100755 --- a/ci/docker/install/ubuntu_mklml.sh +++ b/ci/docker/install/ubuntu_mklml.sh @@ -21,5 +21,5 @@ # the whole docker cache for the image set -ex -wget -q --no-check-certificate -O /tmp/mklml.tgz https://github.com/intel/mkl-dnn/releases/download/v0.14/mklml_lnx_2018.0.3.20180406.tgz +wget -q --no-check-certificate -O /tmp/mklml.tgz https://github.com/intel/mkl-dnn/releases/download/v0.17-rc/mklml_lnx_2019.0.1.20180928.tgz tar -zxf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_* diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/docker/install/ubuntu_nightly_tests.sh index 68358908bdc9..80028d87c572 100755 --- a/ci/docker/install/ubuntu_nightly_tests.sh +++ b/ci/docker/install/ubuntu_nightly_tests.sh @@ -25,12 +25,12 @@ set -ex # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays. for i in 1 2 3 4 5; do add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || sleep 300; done -apt-get update +apt-get update || true apt-get -y install time # Install for RAT License Check Nightly Test apt-get install -y subversion maven -y #>/dev/null # Packages needed for the Straight Dope Nightly tests. -pip2 install pandas scikit-image -pip3 install pandas scikit-image +pip2 install pandas scikit-image prompt_toolkit +pip3 install pandas scikit-image prompt_toolkit diff --git a/ci/docker/install/ubuntu_npm_blc.sh b/ci/docker/install/ubuntu_npm_blc.sh index 30fcb5a1bb5b..59caa9f88bf4 100755 --- a/ci/docker/install/ubuntu_npm_blc.sh +++ b/ci/docker/install/ubuntu_npm_blc.sh @@ -22,7 +22,7 @@ set -ex echo 'Installing npm...' -apt-get update +apt-get update || true apt-get install -y npm echo "Obtaining NodeJS version 8.x" diff --git a/ci/docker/install/ubuntu_nvidia.sh b/ci/docker/install/ubuntu_nvidia.sh index 7b16ed16f481..3d8de9d0d7dd 100755 --- a/ci/docker/install/ubuntu_nvidia.sh +++ b/ci/docker/install/ubuntu_nvidia.sh @@ -18,6 +18,7 @@ # under the License. set -ex +apt-get update || true apt install -y software-properties-common # Adding ppas frequently fails due to busy gpg servers, retry 5 times with 5 minute delays. diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh index 737c333afb6e..0dad3f9ee431 100755 --- a/ci/docker/install/ubuntu_onnx.sh +++ b/ci/docker/install/ubuntu_onnx.sh @@ -27,8 +27,9 @@ set -e set -x echo "Installing libprotobuf-dev and protobuf-compiler ..." +apt-get update || true apt-get install -y libprotobuf-dev protobuf-compiler echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX and tabulate ..." -pip2 install pytest==3.4.0 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.2.1 Pillow==5.0.0 tabulate==0.7.5 -pip3 install pytest==3.4.0 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.2.1 Pillow==5.0.0 tabulate==0.7.5 +pip2 install pytest==3.4.0 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5 +pip3 install pytest==3.4.0 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.3.0 Pillow==5.0.0 tabulate==0.7.5 diff --git a/ci/docker/install/ubuntu_perl.sh b/ci/docker/install/ubuntu_perl.sh index 4d868f7d5d12..e04141eee322 100755 --- a/ci/docker/install/ubuntu_perl.sh +++ b/ci/docker/install/ubuntu_perl.sh @@ -22,5 +22,6 @@ set -ex # install libraries for mxnet's perl package on ubuntu +apt-get update || true apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl cpanm -q Function::Parameters Hash::Ordered PDL::CCS diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index 0fd91cbf706c..ee05058f227e 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -22,12 +22,13 @@ set -ex # install libraries for mxnet's python package on ubuntu -apt-get install -y python-dev python3-dev virtualenv +apt-get update || true +apt-get install -y python-dev python3-dev virtualenv wget # the version of the pip shipped with ubuntu may be too lower, install a recent version here wget -nv https://bootstrap.pypa.io/get-pip.py python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 -pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_r.sh b/ci/docker/install/ubuntu_r.sh index 0e95601ea9fb..cefc4172f245 100755 --- a/ci/docker/install/ubuntu_r.sh +++ b/ci/docker/install/ubuntu_r.sh @@ -34,7 +34,7 @@ apt-key add r.gpg # Installing the latest version (3.3+) that is compatible with MXNet add-apt-repository 'deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu xenial/' -apt-get update +apt-get update || true apt-get install -y --allow-unauthenticated \ libcairo2-dev \ libssl-dev \ diff --git a/ci/docker/install/ubuntu_rat.sh b/ci/docker/install/ubuntu_rat.sh index b131a0bb5586..2c905fc275c7 100755 --- a/ci/docker/install/ubuntu_rat.sh +++ b/ci/docker/install/ubuntu_rat.sh @@ -20,7 +20,7 @@ set -ex echo "Install dependencies" -apt-get update +apt-get update || true apt-get install -y subversion maven openjdk-8-jdk openjdk-8-jre echo "download RAT" diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index bee0e6bbae62..6ecb8d801186 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -24,17 +24,13 @@ set -ex cd "$(dirname "$0")" # install libraries for mxnet's scala package on ubuntu echo 'Installing Scala...' +apt-get update || true apt-get install -y software-properties-common -apt-get update +apt-get update || true apt-get install -y openjdk-8-jdk apt-get install -y openjdk-8-jre -echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list -# ubuntu keyserver is very flaky -#apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 -#apt-key adv --keyserver keys.gnupg.net --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 -apt-key add sbt.gpg -apt-get update && apt-get install -y \ +apt-get update || true +apt-get install -y \ maven \ - sbt \ scala diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/docker/install/ubuntu_tutorials.sh index 9a236bbf4cca..98774754e9ba 100755 --- a/ci/docker/install/ubuntu_tutorials.sh +++ b/ci/docker/install/ubuntu_tutorials.sh @@ -21,6 +21,7 @@ # the whole docker cache for the image set -ex +apt-get update || true apt-get install graphviz python-opencv pip2 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz tqdm pip3 install jupyter matplotlib Pillow opencv-python scikit-learn graphviz tqdm diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md new file mode 100644 index 000000000000..1dcaa5aeb607 --- /dev/null +++ b/ci/docker/qemu/README.md @@ -0,0 +1 @@ +These are files used in the docker container that runs QEMU diff --git a/ci/docker/qemu/runtime_functions.py b/ci/docker/qemu/runtime_functions.py new file mode 100755 index 000000000000..8b8e5acb503c --- /dev/null +++ b/ci/docker/qemu/runtime_functions.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +"""Runtime functions to use in docker / testing""" + +__author__ = 'Pedro Larroy' +__version__ = '0.1' + +import os +import sys +import subprocess +import argparse +import logging +from subprocess import call, check_call, Popen, DEVNULL, PIPE +import time +import sys +import types +import glob +import vmcontrol +from vmcontrol import qemu_ssh, qemu_provision, qemu_rsync_to_host, VM + +def activate_this(base): + import site + import os + import sys + if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') + else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') + prev_sys_path = list(sys.path) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + + + +def run_ut_py3_qemu(): + """Run unit tests in the emulator and copy the results back to the host through the mounted + volume in /mxnet""" + from vmcontrol import VM + with VM() as vm: + qemu_provision(vm.ssh_port) + logging.info("execute tests") + qemu_ssh(vm.ssh_port, "./runtime_functions.py", "run_ut_python3_qemu_internal") + qemu_rsync_to_host(vm.ssh_port, "*.xml", "mxnet") + logging.info("copied to host") + logging.info("tests finished, vm shutdown.") + vm.shutdown() + +def run_ut_python3_qemu_internal(): + """this runs inside the vm""" + pkg = glob.glob('mxnet_dist/*.whl')[0] + logging.info("=== NOW Running inside QEMU ===") + logging.info("PIP Installing %s", pkg) + check_call(['sudo', 'pip3', 'install', pkg]) + logging.info("PIP Installing mxnet/tests/requirements.txt") + check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/tests/requirements.txt']) + logging.info("Running tests in mxnet/tests/python/unittest/") + check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_engine.py']) + # Example to run a single unit test: + # check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent']) + + + +def run_qemu_interactive(): + vm = VM(interactive=True) + vm.detach() + vm.start() + vm.wait() + logging.info("QEMU finished") + +################################ + +def parsed_args(): + parser = argparse.ArgumentParser(description="""python runtime functions""", epilog="") + parser.add_argument('command',nargs='*', + help="Name of the function to run with arguments") + args = parser.parse_args() + return (args, parser) + +def script_name() -> str: + return os.path.split(sys.argv[0])[1] + +def chdir_to_script_directory(): + # We need to be in the same directory than the script so the commands in the dockerfiles work as + # expected. But the script can be invoked from a different path + base = os.path.split(os.path.realpath(__file__))[0] + os.chdir(base) + +def main(): + logging.getLogger().setLevel(logging.INFO) + logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name())) + chdir_to_script_directory() + + # Run function with name passed as argument + (args, parser) = parsed_args() + logging.info("%s", args.command) + if args.command: + fargs = args.command[1:] + globals()[args.command[0]](*fargs) + return 0 + else: + parser.print_help() + fnames = [x for x in globals() if type(globals()[x]) is types.FunctionType] + print('\nAvailable functions: {}'.format(' '.join(fnames))) + return 1 + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py new file mode 100644 index 000000000000..a7e8c0ff0122 --- /dev/null +++ b/ci/docker/qemu/vmcontrol.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +"""Utilities to control a guest VM, used for virtual testing with QEMU""" + +__author__ = 'Pedro Larroy' +__version__ = '0.1' + +import os +import sys +import subprocess +import argparse +import logging +from subprocess import call, check_call, Popen, DEVNULL, PIPE +import time +import sys +import multiprocessing +import shlex + +################################################### +# +# Virtual testing with QEMU +# +# We start QEMU instances that have a local port in the host redirected to the ssh port. +# +# The VMs are provisioned after boot, tests are run and then they are stopped +# +QEMU_SSH_PORT=2222 +QEMU_RAM=4096 + +QEMU_RUN=""" +qemu-system-arm -M virt -m {ram} \ + -kernel vmlinuz \ + -initrd initrd.img \ + -append 'root=/dev/vda1' \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \ + -device virtio-net-device,netdev=mynet \ + -display none -nographic +""" + +QEMU_RUN_INTERACTIVE=""" +qemu-system-arm -M virt -m {ram} \ + -kernel vmlinuz \ + -initrd initrd.img \ + -append 'root=/dev/vda1' \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \ + -device virtio-net-device,netdev=mynet \ + -nographic +""" + + +class VMError(RuntimeError): + pass + +class VM: + """Control of the virtual machine""" + def __init__(self, ssh_port=QEMU_SSH_PORT, ram=QEMU_RAM, interactive=False): + self.log = logging.getLogger(VM.__name__) + self.ssh_port = ssh_port + self.timeout_s = 300 + self.qemu_process = None + self._detach = False + self._interactive = interactive + self.ram = ram + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if not self._detach: + self.shutdown() + self.terminate() + + def start(self): + sys.stderr.flush() + call(['toilet', '-f', 'smbraille', 'Starting QEMU']) + sys.stdout.flush() + self.log.info("Starting VM, ssh port redirected to localhost:%s (inside docker, not exposed by default)", self.ssh_port) + if self.is_running(): + raise VMError("VM is running, shutdown first") + if self._interactive: + self.qemu_process = Popen(shlex.split(QEMU_RUN_INTERACTIVE.format(ssh_port=self.ssh_port, ram=self.ram))) + return + else: + self.log.info("Starting in non-interactive mode. Terminal output is disabled.") + self.qemu_process = Popen(shlex.split(QEMU_RUN.format(ssh_port=self.ssh_port, ram=self.ram)), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE) + def keep_waiting(): + return self.is_running() + + logging.info("waiting for ssh to be open in the VM (timeout {}s)".format(self.timeout_s)) + ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting, self.timeout_s) + + if not self.is_running(): + (_, stderr) = self.qemu_process.communicate() + raise VMError("VM failed to start, retcode: {}, stderr: {}".format( self.retcode(), stderr.decode())) + + if not ssh_working: + if self.is_running(): + self.log.error("VM running but SSH is not working") + self.terminate() + raise VMError("SSH is not working after {} seconds".format(self.timeout_s)) + self.log.info("VM is online and SSH is up") + + def is_running(self): + return self.qemu_process and self.qemu_process.poll() is None + + def retcode(self): + if self.qemu_process: + return self.qemu_process.poll() + else: + raise RuntimeError('qemu process was not started') + + def terminate(self): + if self.qemu_process: + logging.info("send term signal") + self.qemu_process.terminate() + time.sleep(3) + logging.info("send kill signal") + self.qemu_process.kill() + self.qemu_process.wait() + self.qemu_process = None + else: + logging.warn("VM.terminate: QEMU process not running") + + def detach(self): + self._detach = True + + def shutdown(self): + if self.qemu_process: + logging.info("Shutdown via ssh") + # ssh connection will be closed with an error + call(["ssh", "-o", "StrictHostKeyChecking=no", "-p", str(self.ssh_port), "qemu@localhost", + "sudo", "poweroff"]) + ret = self.qemu_process.wait(timeout=90) + self.log.info("VM on port %s has shutdown (exit code %d)", self.ssh_port, ret) + self.qemu_process = None + + def wait(self): + if self.qemu_process: + self.qemu_process.wait() + + def __del__(self): + if self.is_running and not self._detach: + logging.info("VM destructor hit") + self.terminate() + + +def qemu_ssh(ssh_port=QEMU_SSH_PORT, *args): + check_call(["ssh", "-o", "ServerAliveInterval=5", "-o", "StrictHostKeyChecking=no", "-p{}".format(ssh_port), "qemu@localhost", *args]) + + +def qemu_rsync(ssh_port, local_path, remote_path): + check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no -p{}'.format(ssh_port), '-a', local_path, 'qemu@localhost:{}'.format(remote_path)]) + +def qemu_rsync_to_host(ssh_port, remote_path, local_path): + check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no -p{}'.format(ssh_port), '-va', 'qemu@localhost:{}'.format(remote_path), local_path]) + +def qemu_provision(ssh_port=QEMU_SSH_PORT): + import glob + logging.info("Provisioning the VM with artifacts and sources") + + artifact = glob.glob('/work/mxnet/build/*.whl') + for x in artifact: + qemu_rsync(ssh_port, x, 'mxnet_dist/') + qemu_rsync(ssh_port, '/work/runtime_functions.py','') + qemu_rsync(ssh_port, '/work/vmcontrol.py','') + qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet') + logging.info("Provisioning completed successfully.") + + +def wait_ssh_open(server, port, keep_waiting=None, timeout=None): + """ Wait for network service to appear + @param server: host to connect to (str) + @param port: port (int) + @param timeout: in seconds, if None or 0 wait forever + @return: True of False, if timeout is None may return only True or + throw unhandled network exception + """ + import socket + import errno + import time + log = logging.getLogger('wait_ssh_open') + sleep_s = 1 + if timeout: + from time import time as now + # time module is needed to calc timeout shared between two exceptions + end = now() + timeout + + while True: + log.debug("Sleeping for %s second(s)", sleep_s) + time.sleep(sleep_s) + s = socket.socket() + try: + if keep_waiting and not keep_waiting(): + log.debug("keep_waiting() is set and evaluates to False") + return False + + if timeout: + next_timeout = end - now() + if next_timeout < 0: + log.debug("connect time out") + return False + else: + log.debug("connect timeout %d s", next_timeout) + s.settimeout(next_timeout) + + log.debug("connect %s:%d", server, port) + s.connect((server, port)) + ret = s.recv(1024).decode() + if ret and ret.startswith('SSH'): + s.close() + log.info("wait_ssh_open: port %s:%s is open and ssh is ready", server, port) + return True + else: + log.debug("Didn't get the SSH banner") + s.close() + + except ConnectionError as err: + log.debug("ConnectionError %s", err) + if sleep_s == 0: + sleep_s = 1 + else: + sleep_s *= 2 + + except socket.gaierror as err: + log.debug("gaierror %s",err) + return False + + except socket.timeout as err: + # this exception occurs only if timeout is set + if timeout: + return False + + except TimeoutError as err: + # catch timeout exception from underlying network library + # this one is different from socket.timeout + raise + + +def wait_port_open(server, port, timeout=None): + """ Wait for network service to appear + @param server: host to connect to (str) + @param port: port (int) + @param timeout: in seconds, if None or 0 wait forever + @return: True of False, if timeout is None may return only True or + throw unhandled network exception + """ + import socket + import errno + import time + sleep_s = 0 + if timeout: + from time import time as now + # time module is needed to calc timeout shared between two exceptions + end = now() + timeout + + while True: + logging.debug("Sleeping for %s second(s)", sleep_s) + time.sleep(sleep_s) + s = socket.socket() + try: + if timeout: + next_timeout = end - now() + if next_timeout < 0: + return False + else: + s.settimeout(next_timeout) + + logging.info("connect %s %d", server, port) + s.connect((server, port)) + + except ConnectionError as err: + logging.debug("ConnectionError %s", err) + if sleep_s == 0: + sleep_s = 1 + + except socket.gaierror as err: + logging.debug("gaierror %s",err) + return False + + except socket.timeout as err: + # this exception occurs only if timeout is set + if timeout: + return False + + except TimeoutError as err: + # catch timeout exception from underlying network library + # this one is different from socket.timeout + raise + + else: + s.close() + logging.info("wait_port_open: port %s:%s is open", server, port) + return True + diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d1fc2239a442..39631f9dc7e6 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -23,6 +23,8 @@ set -ex NOSE_COVERAGE_ARGUMENTS="--with-coverage --cover-inclusive --cover-xml --cover-branches --cover-package=mxnet" +CI_CUDA_COMPUTE_CAPABILITIES="-gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_70,code=sm_70" +CI_CMAKE_CUDA_ARCH_BIN="52,70" clean_repo() { set -ex @@ -293,15 +295,16 @@ build_centos7_gpu() { # unfortunately this build has problems in 3rdparty dependencies with ccache and make # build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_LAPACK=1 \ - USE_LAPACK_PATH=/usr/lib64/liblapack.so \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_DIST_KVSTORE=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_LAPACK=1 \ + USE_LAPACK_PATH=/usr/lib64/liblapack.so \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_DIST_KVSTORE=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -520,19 +523,19 @@ build_ubuntu_gpu_tensorrt() { rm -rf build make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_OPENCV=0 \ - USE_DIST_KVSTORE=0 \ - USE_TENSORRT=1 \ - USE_JEMALLOC=0 \ - USE_GPERFTOOLS=0 \ - ONNX_NAMESPACE=onnx \ - CUDA_ARCH="-gencode arch=compute_70,code=compute_70"\ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_OPENCV=0 \ + USE_DIST_KVSTORE=0 \ + USE_TENSORRT=1 \ + USE_JEMALLOC=0 \ + USE_GPERFTOOLS=0 \ + ONNX_NAMESPACE=onnx \ + CUDA_ARCH="-gencode arch=compute_70,code=compute_70" \ -j$(nproc) } @@ -542,14 +545,15 @@ build_ubuntu_gpu_mkldnn() { build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_CPP_PACKAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -559,13 +563,14 @@ build_ubuntu_gpu_mkldnn_nocudnn() { build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_MKLDNN=1 \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=0 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_MKLDNN=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=0 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -574,14 +579,15 @@ build_ubuntu_gpu_cuda91_cudnn7() { # unfortunately this build has problems in 3rdparty dependencies with ccache and make # build_ccache_wrappers make \ - DEV=1 \ - ENABLE_TESTCOVERAGE=1 \ - USE_BLAS=openblas \ - USE_CUDA=1 \ - USE_CUDA_PATH=/usr/local/cuda \ - USE_CUDNN=1 \ - USE_CPP_PACKAGE=1 \ - USE_DIST_KVSTORE=1 \ + DEV=1 \ + ENABLE_TESTCOVERAGE=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=1 \ + USE_DIST_KVSTORE=1 \ + CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \ -j$(nproc) } @@ -608,15 +614,17 @@ build_ubuntu_gpu_cmake_mkldnn() { set -ex cd /work/build cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_TESTCOVERAGE=ON \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=1 \ - -DUSE_MKLDNN=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_TESTCOVERAGE=ON \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_MKLML_MKL=1 \ + -DUSE_MKLDNN=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ + -G Ninja \ /work/mxnet ninja -v @@ -629,16 +637,18 @@ build_ubuntu_gpu_cmake() { set -ex cd /work/build cmake \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_TESTCOVERAGE=ON \ - -DUSE_CUDA=1 \ - -DUSE_CUDNN=1 \ - -DUSE_MKLML_MKL=0 \ - -DUSE_MKLDNN=0 \ - -DUSE_DIST_KVSTORE=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_TESTCOVERAGE=ON \ + -DUSE_CUDA=1 \ + -DUSE_CUDNN=1 \ + -DUSE_MKLML_MKL=0 \ + -DUSE_MKLDNN=0 \ + -DUSE_DIST_KVSTORE=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCUDA_ARCH_NAME=Manual \ + -DCUDA_ARCH_BIN=$CI_CMAKE_CUDA_ARCH_BIN \ + -G Ninja \ /work/mxnet ninja -v @@ -692,38 +702,16 @@ unittest_ubuntu_python2_gpu() { export PYTHONPATH=./python/ export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export CUDNN_VERSION=7.0.3 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } -tutorialtest_ubuntu_python3_gpu() { - set -ex - cd /work/mxnet/docs - export MXNET_DOCS_BUILD_MXNET=0 - make html - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export PYTHONPATH=/work/mxnet/python/ - export MXNET_TUTORIAL_TEST_KERNEL=python3 - cd /work/mxnet/tests/tutorials - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture -} - -tutorialtest_ubuntu_python2_gpu() { - set -ex - cd /work/mxnet/docs - export MXNET_DOCS_BUILD_MXNET=0 - make html - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 - export PYTHONPATH=/work/mxnet/python/ - export MXNET_TUTORIAL_TEST_KERNEL=python2 - cd /work/mxnet/tests/tutorials - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture -} - unittest_ubuntu_python3_gpu() { set -ex export PYTHONPATH=./python/ export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export CUDNN_VERSION=7.0.3 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } @@ -740,8 +728,9 @@ unittest_ubuntu_tensorrt_gpu() { export PYTHONPATH=./python/ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH + export CUDNN_VERSION=7.0.3 python tests/python/tensorrt/lenet5_train.py - nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_trt_gpu.xml --verbose tests/python/tensorrt/ + nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_trt_gpu.xml --verbose --nocapture tests/python/tensorrt/ } # quantization gpu currently only runs on P3 instances @@ -751,6 +740,7 @@ unittest_ubuntu_python2_quantization_gpu() { export PYTHONPATH=./python/ export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export CUDNN_VERSION=7.0.3 nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } @@ -761,6 +751,7 @@ unittest_ubuntu_python3_quantization_gpu() { export PYTHONPATH=./python/ export MXNET_MKLDNN_DEBUG=1 # Ignored if not present export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export CUDNN_VERSION=7.0.3 nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu } @@ -770,6 +761,13 @@ unittest_ubuntu_cpu_scala() { make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 } +unittest_centos7_cpu_scala() { + set -ex + cd /work/mxnet + make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 + make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 +} + unittest_ubuntu_cpu_clojure() { set -ex make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1 @@ -816,6 +814,35 @@ unittest_ubuntu_gpu_R() { make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1 } +unittest_ubuntu_cpu_julia06() { + set -ex + export PATH="/work/julia/bin:$PATH" + export MXNET_HOME='/work/mxnet' + export JULIA_PKGDIR='/work/julia-pkg' + export DEPDIR=`julia -e 'print(Pkg.dir())'` + + julia -e 'versioninfo()' + julia -e 'Pkg.init()' + + # install package + ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet + + # install dependencies + julia -e 'Pkg.resolve()' + + # FIXME + export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' + + # use the prebuilt binary from $MXNET_HOME/lib + julia -e 'Pkg.build("MXNet")' + + # run the script `julia/test/runtests.jl` + julia -e 'Pkg.test("MXNet")' + + # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774 + julia -e 'using MXNet; mx._sig_checker()' +} + unittest_centos7_cpu() { set -ex cd /work/mxnet @@ -826,13 +853,13 @@ unittest_centos7_cpu() { unittest_centos7_gpu() { set -ex cd /work/mxnet + export CUDNN_VERSION=7.0.3 python3.6 -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu } integrationtest_ubuntu_cpu_onnx() { set -ex export PYTHONPATH=./python/ - python example/onnx/super_resolution.py pytest tests/python-pytest/onnx/import/mxnet_backend_test.py pytest tests/python-pytest/onnx/import/onnx_import_test.py pytest tests/python-pytest/onnx/import/gluon_backend_test.py @@ -1079,6 +1106,33 @@ nightly_straight_dope_python3_multi_gpu_tests() { test_notebooks_multi_gpu.py --nologcapture } +nightly_tutorial_test_ubuntu_python3_gpu() { + set -ex + cd /work/mxnet/docs + export BUILD_VER=tutorial + export MXNET_DOCS_BUILD_MXNET=0 + make html + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export PYTHONPATH=/work/mxnet/python/ + export MXNET_TUTORIAL_TEST_KERNEL=python3 + cd /work/mxnet/tests/tutorials + nosetests-3.4 --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture +} + +nightly_tutorial_test_ubuntu_python2_gpu() { + set -ex + cd /work/mxnet/docs + export BUILD_VER=tutorial + export MXNET_DOCS_BUILD_MXNET=0 + make html + export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0 + export PYTHONPATH=/work/mxnet/python/ + export MXNET_TUTORIAL_TEST_KERNEL=python2 + cd /work/mxnet/tests/tutorials + nosetests-3.4 --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture +} + + # Deploy deploy_docs() { @@ -1090,6 +1144,31 @@ deploy_docs() { popd } +deploy_jl_docs() { + set -ex + export PATH="/work/julia/bin:$PATH" + export MXNET_HOME='/work/mxnet' + export JULIA_PKGDIR='/work/julia-pkg' + export DEPDIR=`julia -e 'print(Pkg.dir())'` + + julia -e 'versioninfo()' + julia -e 'Pkg.init()' + ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet + julia -e 'Pkg.resolve()' + + # FIXME + export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so' + + # use the prebuilt binary from $MXNET_HOME/lib + julia -e 'Pkg.build("MXNet")' + # build docs + julia -e 'Pkg.add("Documenter")' + julia -e 'cd(Pkg.dir("MXNet")); include(joinpath("docs", "make.jl"))' + + # TODO: make Jenkins worker push to MXNet.jl ph-pages branch if master build + # ... +} + # broken_link_checker broken_link_checker() { diff --git a/ci/docker_cache.py b/ci/docker_cache.py index bebcb25fb8f8..80fd8834ae3d 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -78,7 +78,7 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]: logging.debug('Building %s as %s', platform, docker_tag) try: # Increase the number of retries for building the cache. - image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, use_cache=True) + image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, no_cache=False) logging.info('Built %s as %s', docker_tag, image_id) # Push cache to registry @@ -125,6 +125,10 @@ def load_docker_cache(registry, docker_tag) -> None: :return: None """ # We don't have to retag the image since it's already in the right format + if not registry: + return + assert docker_tag + logging.info('Loading Docker cache for %s from %s', docker_tag, registry) pull_cmd = ['docker', 'pull', docker_tag] subprocess.call(pull_cmd) # Don't throw an error if the image does not exist diff --git a/ci/qemu/README.md b/ci/qemu/README.md new file mode 100644 index 000000000000..6dde8916b28c --- /dev/null +++ b/ci/qemu/README.md @@ -0,0 +1,71 @@ +# QEMU base image creation + +This folder contains scripts and configuration to create a QEMU virtual drive with a debian system. + +The order of execution is: +- `init.sh` to download the installation kernel and ramdisk +- `preseed.sh` to preseed the debian installer so it doesn't ask questions +- `copy.sh` to extract the kernel and ramdisk from the installed system +- `run.sh` to boot the system and fine tune the image + +# Description of the process: + +# Preparing the base image + +First, an installation is made using installer kernel and initrd by using the scripts above. + +# After installation, we extract initrd and kernel from the installation drive + +The commands look like this: + +`virt-copy-out -a hda.qcow2 /boot/initrd.img-4.15.0-30-generic-lpae .` + +In the same way for the kernel. + +Then we install packages and dependencies on the qemu image: + +apt install -y sudo python3-dev virtualenv wget libgfortran3 libopenblas-base rsync build-essential +libopenblas-dev libomp5 + +We enable sudo and passwordless logins: + +Add file `/etc/sudoers.d/01-qemu` +With content: +``` +qemu ALL=(ALL) NOPASSWD: ALL +``` + +Edit: `/etc/ssh/sshd_config` + +And set the following options: +``` +PermitEmptyPasswords yes +PasswordAuthentication yes +PermitRootLogin yes +``` + +Disable root and user passwords with `passwd -d` + +Edit ` /etc/pam.d/common-auth` + +Replace `auth [success=1 default=ignore] pam_unix.so nullok_secure` by +``` +auth [success=1 default=ignore] pam_unix.so nullok +``` + +As root to install system wide: + +``` +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +apt-get clean +``` + +Afterwards install mxnet python3 deps: + +``` +pip3 install -r mxnet_requirements.txt +``` + + +To access qemu control console from tmux: `ctrl-a a c` diff --git a/ci/qemu/copy.sh b/ci/qemu/copy.sh new file mode 100755 index 000000000000..f39a9d083509 --- /dev/null +++ b/ci/qemu/copy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Extract kernel from image + +set -ex +virt-copy-out -a vda.qcow2 /boot/vmlinuz-3.16.0-6-armmp-lpae /boot/initrd.img-3.16.0-6-armmp-lpae . diff --git a/ci/qemu/init.sh b/ci/qemu/init.sh new file mode 100755 index 000000000000..1698cb10f272 --- /dev/null +++ b/ci/qemu/init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Download the installer and ramdisk for intallation +set -ex +wget -O installer-vmlinuz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/vmlinuz +wget -O installer-initrd.gz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/initrd.gz diff --git a/ci/qemu/initrd_modif/inittab b/ci/qemu/initrd_modif/inittab new file mode 100644 index 000000000000..064512595fbc --- /dev/null +++ b/ci/qemu/initrd_modif/inittab @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# /etc/inittab +# busybox init configuration for debian-installer + +# main rc script +::sysinit:/sbin/reopen-console /sbin/debian-installer-startup + +# main setup program +::respawn:/sbin/reopen-console /sbin/debian-installer + +# convenience shells +tty2::askfirst:-/bin/sh +tty3::askfirst:-/bin/sh + +# logging +#tty4::respawn:/usr/bin/tail -f /var/log/syslog + +# Stuff to do before rebooting +::ctrlaltdel:/sbin/shutdown > /dev/null 2>&1 + +# re-exec init on receipt of SIGHUP/SIGUSR1 +::restart:/sbin/init diff --git a/ci/qemu/install.sh b/ci/qemu/install.sh new file mode 100755 index 000000000000..8531b033d074 --- /dev/null +++ b/ci/qemu/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +rm -f vda.qcow2 +sudo ./preseed.sh +qemu-img create -f qcow2 vda.qcow2 10G +qemu-system-arm -M virt -m 1024 \ + -kernel installer-vmlinuz \ + -append BOOT_DEBUG=2,DEBIAN_FRONTEND=noninteractive \ + -initrd installer-initrd_automated.gz \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet \ + -device virtio-net-device,netdev=mynet \ + -nographic -no-reboot diff --git a/ci/qemu/mxnet_requirements.txt b/ci/qemu/mxnet_requirements.txt new file mode 100644 index 000000000000..a2e485efed19 --- /dev/null +++ b/ci/qemu/mxnet_requirements.txt @@ -0,0 +1,7 @@ +urllib3<1.23,>=1.21.1 +requests<2.19.0,>=2.18.4 +graphviz<0.9.0,>=0.8.1 +numpy<=1.15.0,>=1.8.2 +mock +nose +nose-timer diff --git a/ci/qemu/preseed.cfg b/ci/qemu/preseed.cfg new file mode 100644 index 000000000000..23a8fc3baebf --- /dev/null +++ b/ci/qemu/preseed.cfg @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +d-i debian-installer/locale string en_US +d-i keyboard-configuration/xkb-keymap select us +d-i netcfg/get_hostname string debian-qemu +d-i netcfg/get_domain string lab +d-i passwd/root-login boolean true +d-i passwd/root-password password debian +d-i passwd/root-password-again password debian +d-i clock-setup/utc boolean true +d-i mirror/country string US +d-i mirror/https/proxy string +d-i mirror/http/proxy string +d-i mirror/ftp/proxy string +d-i mirror/http/countries select US +d-i mirror/http/hostname string ftp.us.debian.org +d-i mirror/http/mirror select ftp.us.debian.org +d-i localechooser/preferred-locale select en_US.UTF-8 +apt-mirror-setup apt-setup/use_mirror boolean false +apt-mirror-setup apt-setup/mirror/error select Retry +d-i passwd/username string qemu +d-i passwd/user-password password qemu +d-i passwd/user-password-again password qemu +user-setup-udeb passwd/username string qemu +user-setup-udeb passwd/user-fullname string qemu +d-i time/zone string GMT +d-i partman-auto/choose_recipe select atomic +#partman-auto partman-auto/select_disk select /var/lib/partman/devices/=dev=vda +#partman-auto partman-auto/automatically_partition select +#partman-target partman-target/no_root error +#partman-auto partman-auto/init_automatically_partition select 50some_device__________regular +#partman-auto partman-auto/disk string vda +#partman-auto partman-auto/expert_recipe string \ +# boot-root :: \ +# 100 10000 1000000000 ext4 \ +# $primary{ } \ +# lv_name{ root } \ +# method{ format } \ +# format{ } \ +# use_filesystem{ } \ +# filesystem{ ext4 } \ +# mountpoint{ / } . +# +#d-i partman-partitioning/confirm_write_new_label boolean true +#d-i partman/choose_partition select finish +#d-i partman/confirm boolean true +#d-i partman/confirm_nooverwrite boolean true +#partman-base partman/choose_partition select 90finish__________finish +#partman-basicfilesystems partman-basicfilesystems/swap_check_failed boolean +d-i popularity-contest/participate boolean false +d-i tasksel/first multiselect SSH server, standard system utilities +d-i debian-installer/main-menu select Finish the installation +d-i debian-installer/exit/poweroff boolean true diff --git a/ci/qemu/preseed.sh b/ci/qemu/preseed.sh new file mode 100755 index 000000000000..ad005548fbbe --- /dev/null +++ b/ci/qemu/preseed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +rm -rf initrd +mkdir -p initrd +cd initrd +gunzip -c ../installer-initrd.gz | cpio -i +cp ../preseed.cfg . +cp ../initrd_modif/inittab etc/inittab +cp ../initrd_modif/S10syslog lib/debian-installer-startup.d/S10syslog +find . | cpio --create --format 'newc' | gzip -c > ../installer-initrd_automated.gz +echo "Done!" diff --git a/ci/qemu/run.sh b/ci/qemu/run.sh new file mode 100755 index 000000000000..eeff4e1fdccb --- /dev/null +++ b/ci/qemu/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash -exuo pipefail + + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +disk=${1:-vda.qcow2} +qemu-system-arm -M virt -m 1024 \ + -kernel vmlinuz-3.16.0-6-armmp-lpae \ + -initrd initrd.img-3.16.0-6-armmp-lpae \ + -smp 4 \ + -append 'root=/dev/vda1' \ + -drive if=none,file=$disk,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::2222-:22 \ + -device virtio-net-device,netdev=mynet \ + -nographic +# -display none diff --git a/ci/requirements.txt b/ci/requirements.txt new file mode 100644 index 000000000000..8f21ead27f7c --- /dev/null +++ b/ci/requirements.txt @@ -0,0 +1 @@ +docker==3.5.0 diff --git a/ci/travis/install.sh b/ci/travis/install.sh index 16db601211a5..64128495b489 100644 --- a/ci/travis/install.sh +++ b/ci/travis/install.sh @@ -17,14 +17,10 @@ # specific language governing permissions and limitations # under the License. +# Disable brew auto-update to avoid long running updates while running tests in CI. +export HOMEBREW_NO_AUTO_UPDATE=1 + if [ ${TRAVIS_OS_NAME} == "osx" ]; then - brew update brew install opencv - brew install python3 - brew install fftw - brew install libpng - brew install ImageMagick - brew install swig - python -m pip install --user nose numpy cython scipy requests mock - python3 -m pip install --user nose numpy cython scipy requests mock + python -m pip install --user nose numpy cython scipy requests mock nose-timer nose-exclude mxnet-to-coreml fi diff --git a/cmake/AutoDetectF16C.cmake b/cmake/AutoDetectF16C.cmake new file mode 100644 index 000000000000..04331c3ac710 --- /dev/null +++ b/cmake/AutoDetectF16C.cmake @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Determines whether hardware and compiler support F16C +# instruction set +# +# The following are set after configuration is done: +# SUPPORT_F16C + +if(AUTO_DETECT_F16_CMAKE_INCLUDED) + return() +endif() +set(AUTO_DETECT_F16_CMAKE_INCLUDED True) + +set(SUPPORT_F16C False) +if(MSVC) + message("F16C instruction set is not yet supported for MSVC") + return() +endif() +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-mf16c" COMPILER_SUPPORT_MF16C) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + execute_process(COMMAND cat /proc/cpuinfo + COMMAND grep flags + COMMAND grep f16c + OUTPUT_VARIABLE CPU_SUPPORT_F16C) +elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + execute_process(COMMAND sysctl -a + COMMAND grep machdep.cpu.features + COMMAND grep F16C + OUTPUT_VARIABLE CPU_SUPPORT_F16C) +endif() +if(NOT CPU_SUPPORT_F16C) + message("CPU does not support F16C instructions") + return() +endif() +if(CPU_SUPPORT_F16C AND COMPILER_SUPPORT_MF16C) + set(SUPPORT_F16C TRUE) +endif() diff --git a/cmake/DownloadMKLML.cmake b/cmake/DownloadMKLML.cmake new file mode 100644 index 000000000000..c2c1cd6916f6 --- /dev/null +++ b/cmake/DownloadMKLML.cmake @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script will download MKLML + +message(STATUS "Downloading MKLML...") + +set(MKLDNN_RELEASE v0.17-rc) +set(MKLML_RELEASE_FILE_SUFFIX 2019.0.1.20180928) + +if(MSVC) + set(MKL_NAME "mklml_win_${MKLML_RELEASE_FILE_SUFFIX}") + + file(DOWNLOAD "https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_RELEASE}/${MKL_NAME}.zip" + "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.zip" + EXPECTED_MD5 "443e661bdfd32dbbc99b460b43afceee" SHOW_PROGRESS) + file(DOWNLOAD "https://github.com/apache/incubator-mxnet/releases/download/utils/7z.exe" + "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z2.exe" + EXPECTED_MD5 "E1CF766CF358F368EC97662D06EA5A4C" SHOW_PROGRESS) + + execute_process(COMMAND "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z2.exe" "-o${CMAKE_CURRENT_BINARY_DIR}/mklml/" "-y") + execute_process(COMMAND "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z.exe" + "x" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.zip" "-o${CMAKE_CURRENT_BINARY_DIR}/mklml/" "-y") + + set(MKLROOT "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}") + + message(STATUS "Setting MKLROOT path to ${MKLROOT}") + + include_directories(${MKLROOT}/include) + +elseif(APPLE) + set(MKL_NAME "mklml_mac_${MKLML_RELEASE_FILE_SUFFIX}") + + file(DOWNLOAD "https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_RELEASE}/${MKL_NAME}.tgz" + "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" + EXPECTED_MD5 "95f887af332205b1d15b392260003952" SHOW_PROGRESS) + execute_process(COMMAND "tar" "-xzf" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" + "-C" "${CMAKE_CURRENT_BINARY_DIR}/mklml/") + + set(MKLROOT "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}") + + message(STATUS "Setting MKLROOT path to ${MKLROOT}") + +elseif(UNIX) + set(MKL_NAME "mklml_lnx_${MKLML_RELEASE_FILE_SUFFIX}") + + file(DOWNLOAD "https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_RELEASE}/${MKL_NAME}.tgz" + "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" + EXPECTED_MD5 "a63abf155361322b9c03f8fc50f4f317" SHOW_PROGRESS) + execute_process(COMMAND "tar" "-xzf" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" + "-C" "${CMAKE_CURRENT_BINARY_DIR}/mklml/") + + set(MKLROOT "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}") + + message(STATUS "Setting MKLROOT path to ${MKLROOT}") + +else() + message(FATAL_ERROR "Wrong platform") +endif() diff --git a/cmake/FirstClassLangCuda.cmake b/cmake/FirstClassLangCuda.cmake index 4b44077d2c55..f7ffcbbb8201 100644 --- a/cmake/FirstClassLangCuda.cmake +++ b/cmake/FirstClassLangCuda.cmake @@ -185,6 +185,12 @@ function(mshadow_select_nvcc_arch_flags out_variable) elseif(${arch_name} STREQUAL "Pascal") set(arch_bin 6.0 6.1) set(arch_ptx 6.1) + elseif(${arch_name} STREQUAL "Volta") + set(arch_bin 7.0) + set(arch_ptx 7.0) + elseif(${arch_name} STREQUAL "Turing") + set(arch_bin 7.5) + set(arch_ptx 7.5) else() message(SEND_ERROR "Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS") endif() diff --git a/cmake/MklDnn.cmake b/cmake/MklDnn.cmake deleted file mode 100644 index acaf878b2f45..000000000000 --- a/cmake/MklDnn.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#this file download mklml - -message(STATUS "download mklml") -if(MSVC) - set(MKL_NAME "mklml_win_2018.0.3.20180406") - file(DOWNLOAD "https://github.com/intel/mkl-dnn/releases/download/v0.14/${MKL_NAME}.zip" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.zip" EXPECTED_MD5 "8DD73E7D3F19F004551809824C4E8970" SHOW_PROGRESS) - file(DOWNLOAD "https://github.com/apache/incubator-mxnet/releases/download/utils/7z.exe" "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z2.exe" EXPECTED_MD5 "E1CF766CF358F368EC97662D06EA5A4C" SHOW_PROGRESS) - - execute_process(COMMAND "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z2.exe" "-o${CMAKE_CURRENT_BINARY_DIR}/mklml/" "-y") - execute_process(COMMAND "${CMAKE_CURRENT_BINARY_DIR}/mklml/7z.exe" "x" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.zip" "-o${CMAKE_CURRENT_BINARY_DIR}/mklml/" "-y") - set(MKLROOT "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}") - include_directories(${MKLROOT}/include) - file(COPY ${MKLROOT}/lib/libiomp5md.dll DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - file(COPY ${MKLROOT}/lib/mklml.dll DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - file(COPY ${CMAKE_SOURCE_DIR}/3rdparty/mkldnn/config_template.vcxproj.user DESTINATION ${CMAKE_SOURCE_DIR}) -elseif(UNIX) - set(MKL_NAME "mklml_lnx_2018.0.3.20180406") - file(DOWNLOAD "https://github.com/intel/mkl-dnn/releases/download/v0.14/${MKL_NAME}.tgz" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" EXPECTED_MD5 "DAF7EFC3C1C0036B447213004467A8AE" SHOW_PROGRESS) - execute_process(COMMAND "tar" "-xzf" "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}.tgz" "-C" "${CMAKE_CURRENT_BINARY_DIR}/mklml/") - set(MKLROOT "${CMAKE_CURRENT_BINARY_DIR}/mklml/${MKL_NAME}") - include_directories(${MKLROOT}/include) - file(COPY ${MKLROOT}/lib/libiomp5.so DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - file(COPY ${MKLROOT}/lib/libmklml_gnu.so DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - file(COPY ${MKLROOT}/lib/libmklml_intel.so DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -else() - message(FATAL_ERROR "not support now") -endif() diff --git a/cmake/cmake_options.yml b/cmake/cmake_options.yml new file mode 100644 index 000000000000..6fbf4e1d0617 --- /dev/null +++ b/cmake/cmake_options.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- # CMake configuration +USE_CUDA: OFF # Build with CUDA support +USE_OLDCMAKECUDA: OFF # Build with old cmake cuda +USE_NCCL: OFF # Use NVidia NCCL with CUDA +USE_OPENCV: ON # Build with OpenCV support +USE_OPENMP: ON # Build with Openmp support +USE_CUDNN: ON # Build with cudnn support) # one could set CUDNN_ROOT for search path +USE_SSE: ON # Build with x86 SSE instruction support IF NOT ARM +USE_F16C: ON # Build with x86 F16C instruction support) # autodetects support if ON +USE_LAPACK: ON # Build with lapack support +USE_MKL_IF_AVAILABLE: ON # Use MKL if found +USE_MKLML_MKL: ON # Use MKLDNN variant of MKL (if MKL found) IF USE_MKL_IF_AVAILABLE AND (NOT APPLE) +USE_MKLDNN: ON # Use MKLDNN variant of MKL (if MKL found) IF USE_MKL_IF_AVAILABLE AND (NOT APPLE) +USE_OPERATOR_TUNING: ON # Enable auto-tuning of operators IF NOT MSVC +USE_GPERFTOOLS: ON # Build with GPerfTools support (if found) +USE_JEMALLOC: ON # Build with Jemalloc support +USE_PROFILER: ON # Build with Profiler support +USE_DIST_KVSTORE: OFF # Build with DIST_KVSTORE support +USE_PLUGINS_WARPCTC: OFF # Use WARPCTC Plugins +USE_PLUGIN_CAFFE: OFF # Use Caffe Plugin +USE_CPP_PACKAGE: OFF # Build C++ Package +USE_MXNET_LIB_NAMING: ON # Use MXNet library naming conventions. +USE_GPROF: OFF # Compile with gprof (profiling) flag +USE_CXX14_IF_AVAILABLE: OFF # Build with C++14 if the compiler supports it +USE_VTUNE: OFF # Enable use of Intel Amplifier XE (VTune)) # one could set VTUNE_ROOT for search path +ENABLE_CUDA_RTC: ON # Build with CUDA runtime compilation support +BUILD_CPP_EXAMPLES: ON # Build cpp examples +INSTALL_EXAMPLES: OFF # Install the example source files. +USE_SIGNAL_HANDLER: OFF # Print stack traces on segfaults. +USE_TENSORRT: OFF # Enable infeference optimization with TensorRT. +USE_ASAN: OFF # Enable Clang/GCC ASAN sanitizers. +ENABLE_TESTCOVERAGE: OFF # Enable compilation with test coverage metric output diff --git a/contrib/clojure-package/README.md b/contrib/clojure-package/README.md index 66ba77b76135..d5ee6ccbeb44 100644 --- a/contrib/clojure-package/README.md +++ b/contrib/clojure-package/README.md @@ -1,152 +1,204 @@ # Clojure MXNet -A clojure package to the MXNet Deep Learning library +A Clojure Package Built on the MXNet Deep Learning Library ## Introduction -MXNet is a first class, modern deep learning library. It supports multiple languages on a first class basis and is incubating as an Apache project. +MXNet is a flexible and efficient deep learning library. While its core is built in C++ for maximum performance, support for multiple programming languages in intermediate and high-level APIs is a first-class feature. MXNet is currently an incubating Apache project. -The motivation for creating a Clojure package is to be able to open the deep learning library to the Clojure ecosystem and build bridges for future development and innovation for the community. It provides all the needed tools including low level and high level apis, dynamic graphs, and things like GAN and natural language support. +The motivation for creating a Clojure package was to give Clojurians access to a world-class deep learning platform, thereby building bridges for future development and innovation in the community. The Clojure package provides all the essential tools, including low-level and high-level APIs, dynamic graphs, etc., and enables building advanced architectures like GANs or LSTM to tackle challenging applications such as image recognition or natural language processing. -For high leverage, the Clojure package has been built on the existing Scala package using interop. This has allowed rapid development and close parity with the Scala functionality. This also leaves the door open to directly developing code against the jni-bindings with Clojure in the future in an incremental fashion, using the test suites as a refactoring guide. +To maximize leverage, the Clojure package has been built on the existing Scala package using [Java Interop](https://clojure.org/reference/java_interop). This approach has allowed rapid initial development and close parity with the Scala package functionality. It also leaves the door open to incrementally developing Clojure code that directly interfaces MXNet core using [JNI](https://en.wikipedia.org/wiki/Java_Native_Interface). + +For a **video introduction**, see [Clojure MXNet with Carin Meier - Clojure Virtual Meetup](https://www.crowdcast.io/e/clojure-mxnet-with-carin) (setup instructions from 20:49). ## Current State and Plans -The Clojure package is nearing the end of its first development milestone which is to achieve a close parity with the Scala package. +The Clojure MXNet package is currently treated as *user-contributed code* within MXNet, as can be seen from its placement under `contrib` in the source tree. This means that it should first undergo a stabilization period and receive feedback from users before it can graduate to a fully integrated and supported part of MXNet. -Help is needed testing and generally making the package better. A list of the pacakge status and contribution needs can be found here [Clojure Package Contribution Needs](https://cwiki.apache.org/confluence/display/MXNET/Clojure+Package+Contribution+Needs). Please get involved :) +That said, because it closely tracks the Scala package, Clojure MXNet can be expected to have a similar level of maturity and stability regarding the low-level functionality. It is mostly in the hand-written Java interop part of the Clojure wrapper where bugs are more likely to be encountered. Such bugs tend to be fixed rather quickly once they are known and their origin is clear (see also [Getting Involved](#getting-involved)). -Testing instructions can be found in the testing.md. +For an overview of the development status and open problems, please refer to [Clojure Package Contribution Needs](https://cwiki.apache.org/confluence/display/MXNET/Clojure+Package+Contribution+Needs). -## Getting Started +## Getting Involved + +By far the best way to get involved with this project is to install the Clojure MXNet package, run the examples, play around, build new things with it, and get back to the development team with feedback! Your input can not only help to identify current issues, but also guide the future development of the Clojure package by pointing out must-have features that are currently missing, or by identifying usability or performace problems of high impact. -The following systems are supported: +There are two main ways of reaching out to other users and the package maintainers: -- OSX cpu -- Linux cpu -- Linux gpu +- If you have a question or general feedback, or you encountered a problem but are not sure if it's a bug or a misunderstanding, then the *Apache Slack* (channels `#mxnet` and `#mxnet-scala`) is the best place to turn check out. To join, [ask for an invitation](https://mxnet.apache.org/community/contribute.html#slack) at `dev@mxnet.apache.org`. +- If you found a bug, miss an important feature or want to give feedback directly relevant for development, please head over to the MXNet [GitHub issue page](https://github.com/apache/incubator-mxnet/issues) and create a new issue. If the issue is specific to the Clojure package, consider using a title starting with `[Clojure]` to make it easily discoverable among the many other, mostly generic issues. -There are two ways of getting going. The first way is the easiest and that is to use the pre-built jars from Maven. The second way is to build from source. In both cases, you will need to load the prereqs and dependencies, (like opencv). +Of course, contributions to code or documentation are also more than welcome! Please check out the [Clojure Package Contribution Needs](https://cwiki.apache.org/confluence/display/MXNET/Clojure+Package+Contribution+Needs) to get an idea about where and how to contribute code. +For a more comprehensive overview of different ways to contribute, see [Contributing to MXNet](https://mxnet.apache.org/community/contribute.html). +## Getting Started -### Prerequisites +The Clojure MXNet framework consists of a core C library, a Scala API that talks to the core through [JNI (Java Native Interface)](https://en.wikipedia.org/wiki/Java_Native_Interface) bindings, and finally a Clojure wrapper around the Scala API. +Since the core contains native (compiled) code and is bundled with the language bindings, your hardware and OS matter to the choices to be made during installation. The following combinations of operating system and compute device are supported: -Follow the instructions from https://mxnet.incubator.apache.org/install/osx_setup.html or https://mxnet.incubator.apache.org/install/ubuntu_setup.html -about _Prepare Environment for GPU Installation_ -and _Install MXNet dependencies_ +- Linux CPU +- Linux GPU +- OSX CPU +There are three ways of getting started: -#### Cloning the repo and running from source +1. [Install prebuilt Clojure jars](#option-1-clojure-package-from-prebuilt-jar) with the native dependencies baked in. This the quickest way to get going. +2. [Install the Clojure package from source, but use prebuilt jars for the native dependencies](#option-2-clojure-package-from-source-scala-package-from-jar). Choose this option if you want pre-release features of the Clojure package but don't want to build (compile) native dependencies yourself. +3. [Build everything from source](#option-3-everything-from-source). This option is for developers or advanced users who want cutting-edge features in all parts of the dependency chain. -To use the prebuilt jars (easiest), you will need to replace the native version of the line in the project dependencies with your configuration. +**Note:** This guide assumes that you are familiar with the basics of creating Clojure projects and managing dependencies. See [here](https://github.com/technomancy/leiningen/blob/stable/doc/TUTORIAL.md) for the official Leiningen tutorial. -`[org.apache.mxnet/mxnet-full_2.11-linux-x86_64-gpu "1.3.0"]` -or -`[org.apache.mxnet/mxnet-full_2.11-linux-x86_64-cpu "1.3.0"]` -or -`[org.apache.mxnet/mxnet-full_2.11-osx-x86_64-cpu "1.3.0"]` +### Option 1: Clojure Package from Prebuilt Jar -If you are using the prebuilt jars they may have a slightly different dependencies then building from source: +If you are new to MXNet and just want to try things out, this option is the best way to get started. You will install release versions of MXNet core, MXNet Scala and MXNet Clojure. -*For OSX you will need:* +For reference, the Clojure MXNet jars can be found on [maven.org](https://search.maven.org/search?q=clojure%20mxnet). -`brew install opencv` +#### Installing additional dependencies -*For Ubuntu Linux you will need:* +Depending on your operating system, you will need a couple of packages that are not distributed through Maven: -``` -sudo add-apt-repository ppa:timsc/opencv-3.4 +- [OpenCV](https://opencv.org/) version 3.4 +- [OpenBLAS](https://www.openblas.net/) +- [ATLAS](http://math-atlas.sourceforge.net/) +- [cURL](https://curl.haxx.se/) library version 3 + +##### Linux (Ubuntu) + +As of writing this, OpenCV 3.4 is not available in the default repositories. Therefore, a third-party repository is needed. + +```bash +sudo add-apt-repository ppa:timsc/opencv sudo apt-get update -sudo apt install libopencv-imgcodecs3.4 +sudo apt install libopencv-imgcodecs3.4 libopenblas-base libatlas3-base libcurl3 ``` -*For Arch Linux you will need:* +Note: `libcurl3` may conflict with other packages on your system. [Here](https://github.com/apache/incubator-mxnet/issues/12822) is a possible workaround. -_CPU_ +##### Linux (Arch) -``` +```bash yaourt -S openblas-lapack yaourt -S libcurl-compat export LD_PRELOAD=libcurl.so.3 ``` -_GPU_ -``` +To enable GPU support, you will additionally need the CUDA toolkit: + +```bash wget https://archive.archlinux.org/packages/c/cuda/cuda-9.0.176-4-x86_64.pkg.tar.xz sudo pacman -U cuda-9.0.176-4-x86_64.pkg.tar.xz ``` -If you want to see the exact versions and flags that the jars were built with, look here: -[Scala Release Process](https://cwiki.apache.org/confluence/display/MXNET/MXNet-Scala+Release+Process) +##### OSX +```bash +brew install wget +brew install opencv +``` -Check your installation with `lein test`. If that works alright then, you can try some code! +#### Installing the Clojure package -```clojure +- Create a new project with `lein new my-mxnet` +- Edit your `project.clj` and add one of the following entries to `:dependencies`, based on your system and the compute device you want to use: -(ns tutorial.ndarray - (:require [org.apache.clojure-mxnet.ndarray :as ndarray] - [org.apache.clojure-mxnet.context :as context])) + - `[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-cpu "1.3.0"]` + - `[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-gpu "1.3.0"]` + - `[org.apache.mxnet.contrib.clojure/clojure-mxnet-osx-cpu "1.3.0"]` -;;Create NDArray -(def a (ndarray/zeros [100 50])) ;;all zero arrray of dimension 100 x 50 -(def b (ndarray/ones [256 32 128 1])) ;; all one array of dimension -(def c (ndarray/array [1 2 3 4 5 6] [2 3])) ;; array with contents of a shape 2 x 3 +After making this change and running `lein deps`, you should be able to run example code like this [NDArray Tutorial](https://github.com/apache/incubator-mxnet/blob/master/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj). -;;; There are also ways to convert to a vec or get the shape as an object or vec -(ndarray/->vec c) ;=> [1.0 2.0 3.0 4.0 5.0 6.0] -``` +### Option 2: Clojure package from Source, Scala Package from Jar + +With this option, you will install a Git revision of the Clojure package source and a [Scala package jar from Maven](https://search.maven.org/search?q=g:org.apache.mxnet) with native dependencies baked in. + +- Install additional dependencies as described in [the corresponding section for Option 1](#installing-additional-dependencies), +- Recursively clone the MXNet repository and checkout the desired revision. Here we assume the `1.3.0` tag and a clone into the `~/mxnet` directory: + + ```bash + git clone --recursive https://github.com/apache/incubator-mxnet.git ~/mxnet + cd ~/mxnet + git tag --list # Find the tag that matches the Scala package version + git checkout tags/1.3.0 -b my_mxnet + git submodule update --init --recursive + cd contrib/clojure + ``` + +- Edit `project.clj` to include the desired Scala jar from Maven: -See the examples/tutorial section for more. + [org.apache.mxnet/mxnet-full_2.11-linux-x86_64-cpu "1.3.0”] +- Run `lein test`. All the tests should run without error. +- At this point you can run `lein install` to build and install the Clojure jar locally. -The jars from maven with the needed MXNet native binaries in it. On startup, the native libraries are extracted from the jar and copied into a temporary location on your path. On termination, they are deleted. +To run examples, you can now use `lein run` in any of the example directories, e.g., `examples/imclassification`. You can also specify the compute device, e.g., `lein run :cpu 2` (for 2 CPUs) or `lein run :gpu` (for 1 GPU). +**Note:** Instead of a release tag, you can also use a development version of the Clojure package, e.g., Git `master`, together with the prebuilt Scala jar. In that case, however, breakage can happen at any point, for instance when the Scala development version adds, changes or removes an interface and the Clojure development version moves along. If you really need the most recent version, you should consider [installation option 3](#option-3-everything-from-source). -### Build from MXNET Source +### Option 3: Everything from Source -First, ensure you have JDK 8 on your system. Later versions may produce cryptic build errors mentioning `scala.reflect.internal.MissingRequirementError`. +With this option, you will compile the core MXNet C++ package and jars for both Scala and Clojure language bindings from source. If you intend to make changes to the code in any of the parts, or if you simply want the latest and greatest features, this choice is for you. -Checkout the latest SHA from the main package: +The first step is to recursively clone the MXNet repository and checkout the desired revision. Here we assume a clone into the `~/mxnet` directory: -`git clone --recursive https://github.com/apache/incubator-mxnet.git ~/mxnet` -`cd ~/mxnet` + ```bash + git clone --recursive https://github.com/apache/incubator-mxnet.git ~/mxnet + cd ~/mxnet + git checkout tags/1.3.0 -b my_mxnet # this is optional + git submodule update --init --recursive + ``` -If you need to checkout a particular release you can do it with: +If you have previous builds and other unwanted files lying around in the working directory and would like to clean up, [here](https://gist.github.com/nicktoumpelis/11214362) is a useful script for that task. However, be aware that this recipe will remove any untracked files and reset uncommitted changes in the working directory. -`git checkout tags/1.3.0 -b release-1.3.0` +#### Building the core library -`git submodule update --init --recursive` +Detailed instructions for building MXNet core from source can be found [in the MXNet installation documentation](https://mxnet.incubator.apache.org/install/index.html). The relevant sections are: -Sometimes it useful to use this script to clean hard -https://gist.github.com/nicktoumpelis/11214362 +- For Ubuntu Linux: [CUDA Dependencies](https://mxnet.incubator.apache.org/install/ubuntu_setup.html#cuda-dependencies) and [Building MXNet from Source](https://mxnet.incubator.apache.org/install/ubuntu_setup.html#build-mxnet-from-source) +- For Mac OSX: [Build the Shared Library](https://mxnet.incubator.apache.org/install/osx_setup.html#build-the-shared-library) +In particular, ignore all of the language-interface-specific sections. -Go here to do the base package installation https://mxnet.incubator.apache.org/install/index.html +The outcome of this step will be a shared library `lib/libmxnet.so` that is used in the next step. - Run `make scalapkg` then `make scalainstall` +#### Building the Scala jar -then replace the correct jar for your architecture in the project.clj, example `[org.apache.mxnet/mxnet-full_2.11-osx-x86_64-cpu "1.3.0-SNAPSHOT"]` +- Ensure you have JDK 8 on your system. Later versions may produce cryptic build errors mentioning `scala.reflect.internal.MissingRequirementError`. +- Build and install the Scala package in your local Maven directory using the following commands: -#### Test your installation + ```bash + make scalapkg + make scalainstall + ``` -To test your installation, you should run `lein test`. This will run the test suite (CPU) for the clojure package. +#### Building the Clojure jar + +- Enter the `contrib/clojure` directory and edit the `project.clj` file. Add the Scala jar that was just created and installed, e.g., `[org.apache.mxnet/mxnet-full_2.11-osx-x86_64-cpu "1.3.0-SNAPSHOT"]`, to the `:dependencies`. +- Run `lein test`. All the tests should run without an error. +- Run `lein install` to build and install the Clojure jar locally. +To run examples, you can now use `lein run` in any of the example directories, e.g., `examples/imclassification`. You can also specify the compute device, e.g., `lein run :cpu 2` (for 2 CPUs) or `lein run :gpu` (for 1 GPU). -#### Generation of NDArray and Symbol apis +## Docker Files -The bulk of the ndarray and symbol apis are generated via java reflection into the Scala classes. The files are generated as a compile time step (AOT) in the `dev.generator` namespace. +There are Dockerfiles available as well. -You may also run this manually with the repl functions: +- [Community Provided by Magnet](https://hub.docker.com/u/magnetcoop/) +- [MXNet CI](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/Dockerfile.build.ubuntu_cpu) and the install scripts + - [Ubuntu core](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/install/ubuntu_core.sh) + - [Ubuntu Scala](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/install/ubuntu_scala.sh) + - [Ubuntu Clojure](https://github.com/apache/incubator-mxnet/blob/master/ci/docker/install/ubuntu_clojure.sh) -`(generate-ndarray-file)` -and -`(generate-symbol-file)` +## Need Help? +If you are having trouble getting started or have a question, feel free to reach out at: -These will generate the files under `src/org.apache.clojure-mxnet/gen/` that are loaded by the `src/org.apache.clojure-mxnet/ndarray.clj` and `src/org.apache.clojure-mxnet/symbol.clj` files. +- Clojurian Slack #mxnet channel. To join, go to [http://clojurians.net/](http://clojurians.net/). +- Apache Slack #mxnet and #mxnet-scala channel. To join this slack send an email to dev@mxnet.apache.org. +- Create an Issue on [https://github.com/apache/incubator-mxnet/issues](https://github.com/apache/incubator-mxnet/issues). ## Examples @@ -158,11 +210,11 @@ There are quite a few examples in the examples directory. To use. There are README is every directory outlining instructions. A good place to get started is the module example. -Do `lein run` for the cpu version or `lein run :gpu` for gpu. +Do `lein run` for the CPU version or `lein run :gpu` for GPU. ## Generating documentation -To generate api docs, run `lein codox`. The html docs will be generated in the target/docs directory. +To generate API docs, run `lein codox`. The html docs will be generated in the target/docs directory. ## Code Coverage @@ -180,12 +232,11 @@ before the submit a new pull request so we can keep the style consistent through ## FAQ - **Why build on the Scala package?** The motivation section addresses this, but the main reason is high leverage is using the great work that the Scala package has already done. -**How can I tell if the gpu is being used?** +**How can I tell if the GPU is being used?** CUDA is finding a best algorithm... As long as a Context.gpu() passed in the code as a context, GPU should be used. @@ -195,30 +246,17 @@ This command can be very handy too timestamp, name, utilization.gpu [%], utilization.memory [%], memory.total [MiB], memory.free [MiB], memory.used [MiB]` **Supported APIs** -There are 3 high level apis supported in MXNet: (Model/FeedForward), Module, and Gluon. The Module api is supported in the Clojure package because of the existing support for it in the Scala package. The Module api is very similar to the Gluon api and examples of the usage can be found in the examples directory. The Model/FeedForward Api is deprected. +There are 3 high level APIs supported in MXNet: (Model/FeedForward), Module, and Gluon. The Module API is supported in the Clojure package because of the existing support for it in the Scala package. The Module API is very similar to the Gluon API and examples of the usage can be found in the examples directory. The Model/FeedForward API is deprected. -Gluon support will come later and may or may not be built on the Scala gluon api (when it lands there) +Gluon support will come later and may or may not be built on the Scala gluon API (when it lands there) ## Architecture & Design See the Confluence page: https://cwiki.apache.org/confluence/display/MXNET/MXNet+Clojure ## Building and Deploying Jars -The process to build and deploy the jars currently is a manual process using the `lein` build tool and `Clojars`, the Clojure dependency hosting platform. - -There is one jar for every system supported. - -- Comment out the line in the `project.clj` for the system that you are targeting, (example OSX cpu you would uncomment out ` [org.apache.mxnet/mxnet-full_2.11-osx-x86_64-cpu "1.2.0"]` but leave the linux deps commented) -- Change the `defproject org.apache.mxnet.contrib.clojure/clojure-mxnet "0.1.1-SNAPSHOT"` in the project to reference the correct version number and jar description. For example changing the line to be `org.apache.mxnet.contrib.clojure/mxnet-osx-cpu "0.1.2"` would create a jar with the group id of `org.apache.mxnet.contrib.clojure` and the artifact name of `mxnet-osx-cpu` and the version of `0.1.2` -- Run `lein clean` -- Run `lein jar` to create the jar -- Check that the jar looks alright in the `/target` directory. - -To deploy the jar to Clojars, you do `lein deploy clojars` and it will prompt you for your username and password. - -_Note: Integration with deployment to Nexus can be enabled too for the future [https://help.sonatype.com/repomanager2/maven-and-other-build-tools/leiningen](https://help.sonatype.com/repomanager2/maven-and-other-build-tools/leiningen)_ -You would repeat this process for all the build system types. +The release process for deploying the Clojure jars is on the [Apache MXNet developer wiki](https://cwiki.apache.org/confluence/display/MXNET/Clojure+Release+Process). ## Special Thanks diff --git a/contrib/clojure-package/examples/tutorial/project.clj b/contrib/clojure-package/examples/tutorial/project.clj index 4910886ca54e..7f3254e641eb 100644 --- a/contrib/clojure-package/examples/tutorial/project.clj +++ b/contrib/clojure-package/examples/tutorial/project.clj @@ -19,4 +19,7 @@ :description "MXNET tutorials" :plugins [[lein-cljfmt "0.5.7"]] :dependencies [[org.clojure/clojure "1.9.0"] - [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.1-SNAPSHOT"]]) + ;; Uncomment the one appropriate for your machine & configuration: + #_[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-cpu "1.3.0"] + #_[org.apache.mxnet.contrib.clojure/clojure-mxnet-linux-gpu "1.3.0"] + #_[org.apache.mxnet.contrib.clojure/clojure-mxnet-osx-cpu "1.3.0"]]) diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/kvstore.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/kvstore.clj index 558b21f0aa4c..f35d4a06922f 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/kvstore.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/kvstore.clj @@ -16,35 +16,44 @@ ;; (ns tutorial.kvstore + "A REPL tutorial of the MXNet Clojure API for KVStore, based on + https://mxnet.incubator.apache.org/api/clojure/kvstore.html" (:require [org.apache.clojure-mxnet.kvstore :as kvstore] [org.apache.clojure-mxnet.ndarray :as ndarray] [org.apache.clojure-mxnet.context :as context])) -;;Basic Push and Pull -;;Provides basic operation over multiple devices (GPUs or CPUs) on a single device. -;; Initialization -;; Let’s consider a simple example. It initializes a (int, NDArray) pair into the store, and then pulls the value out. +;;;; Basic Push and Pull -(def kv (kvstore/create "local")) ;; create a local kvstore +;; Provides basic operation over multiple devices (GPUs or CPUs) on a +;; single device. + +;;; Initialization +;; Let’s consider a simple example. It initializes a (`int`, +;; `NDArray`) pair into the store, and then pulls the value out. + +(def kv (kvstore/create "local")) ; create a local kvstore (def shape [2 3]) -;;; init the kvstore with a vector of keys (strings) and ndarrays +;; init the kvstore with a vector of keys (strings) and ndarrays (kvstore/init kv ["3"] [(ndarray/* (ndarray/ones shape) 2)]) (def a (ndarray/zeros shape)) (kvstore/pull kv ["3"] [a]) (ndarray/->vec a) ;=> [2.0 2.0 2.0 2.0 2.0 2.0] -;;Push, Aggregation, and Updater -;;For any key that’s been initialized, you can push a new value with the same shape to the key, as follows: - +;;; Push, Aggregation, and Updater +;; For any key that’s been initialized, you can push a new value with +;; the same shape to the key, as follows: (kvstore/push kv ["3"] [(ndarray/* (ndarray/ones shape) 8)]) (kvstore/pull kv ["3"] [a]) (ndarray/->vec a);=>[8.0 8.0 8.0 8.0 8.0 8.0] -;;The data that you want to push can be stored on any device. Furthermore, you can push multiple values into the same key, where KVStore first sums all of these values, and then pushes the aggregated value, as follows: +;; The data that you want to push can be stored on any +;; device. Furthermore, you can push multiple values into the same +;; key, where KVStore first sums all of these values, and then pushes +;; the aggregated value, as follows: -;; using multiple cpus instead of gpus +;; (Here we use multiple CPUs.) (def cpus [(context/cpu 0) (context/cpu 1) (context/cpu 2)]) (def b [(ndarray/ones shape {:ctx (nth cpus 0)}) (ndarray/ones shape {:ctx (nth cpus 1)}) @@ -53,22 +62,33 @@ (kvstore/pull kv "3" a) (ndarray/->vec a) ;=> [3.0 3.0 3.0 3.0 3.0 3.0] - -;;Pull -;;You’ve already seen how to pull a single key-value pair. Similar to the way that you use the push command, you can pull the value into several devices with a single call. +;;; Pull +;; You’ve already seen how to pull a single key-value pair. Similar to +;; the way that you use the push command, you can pull the value into +;; several devices with a single call. (def b [(ndarray/ones shape {:ctx (context/cpu 0)}) (ndarray/ones shape {:ctx (context/cpu 1)})]) (kvstore/pull kv ["3" "3"] b) (map ndarray/->vec b) ;=> ([3.0 3.0 3.0 3.0 3.0 3.0] [3.0 3.0 3.0 3.0 3.0 3.0]) -;;List Key-Value Pairs -;;All of the operations that we’ve discussed so far are performed on a single key. KVStore also provides the interface for generating a list of key-value pairs. For a single device, use the following: + +;;;; List Key-Value Pairs + +;; All of the operations that we’ve discussed so far are performed on +;; a single key. KVStore also provides the interface for generating a +;; list of key-value pairs. For a single device, use the following: (def ks ["5" "7" "9"]) -(kvstore/init kv ks [(ndarray/ones shape) (ndarray/ones shape) (ndarray/ones shape)]) -(kvstore/push kv ks [(ndarray/ones shape) (ndarray/ones shape) (ndarray/ones shape)]) -(def b [(ndarray/zeros shape) (ndarray/zeros shape) (ndarray/zeros shape)]) +(kvstore/init kv ks [(ndarray/ones shape) + (ndarray/ones shape) + (ndarray/ones shape)]) +(kvstore/push kv ks [(ndarray/ones shape) + (ndarray/ones shape) + (ndarray/ones shape)]) +(def b [(ndarray/zeros shape) + (ndarray/zeros shape) + (ndarray/zeros shape)]) (kvstore/pull kv ks b) -(map ndarray/->vec b);=> ([1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0]) +(map ndarray/->vec b) ;=> ([1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0] [1.0 1.0 1.0 1.0 1.0 1.0]) diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj index 3cef342f0ed2..4ca50ff5cd44 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/module.clj @@ -16,6 +16,8 @@ ;; (ns tutorial.module + "A REPL tutorial of the MXNet Clojure API for Module, based on + https://mxnet.incubator.apache.org/api/clojure/module.html" (:require [clojure.java.io :as io] [clojure.java.shell :refer [sh]] [org.apache.clojure-mxnet.eval-metric :as eval-metric] @@ -24,12 +26,26 @@ [org.apache.clojure-mxnet.symbol :as sym] [org.apache.clojure-mxnet.ndarray :as ndarray])) + +;; The Module API provides an intermediate and high-level interface +;; for performing computation with neural networks in MXNet. Module +;; wraps a Symbol and one or more Executors. It has both a high level +;; and intermediate level API. + + +;;;; Prepare the Data + +;; In this example, we are going to use the MNIST data set. If you +;; start, we can run some helper scripts to download the data for us. + (def data-dir "data/") (when-not (.exists (io/file (str data-dir "train-images-idx3-ubyte"))) (sh "../../scripts/get_mnist_data.sh")) -;;; Load the MNIST datasets +;; MXNet provides function in the `io` namespace to load the MNIST +;; datasets into training and test data iterators that we can use with +;; our module. (def train-data (mx-io/mnist-iter {:image (str data-dir "train-images-idx3-ubyte") :label (str data-dir "train-labels-idx1-ubyte") :label-name "softmax_label" @@ -47,11 +63,13 @@ :flat true :silent false})) -;; The module API provides an intermediate and high-level interface for performing computation with neural networks in MXNet. Module wraps a Symbol and one or more Executors. It has both a high level and intermediate level api -;; Preparing a module for Computation +;;;; Preparing a module for Computation -;; construct a module +;; To construct a module, we need to have a symbol as input. This +;; symbol takes input data in the first layer and then has subsequent +;; layers of fully connected and relu activation layers, ending up in +;; a softmax layer for output. (let [data (sym/variable "data") fc1 (sym/fully-connected "fc1" {:data data :num-hidden 128}) @@ -62,7 +80,7 @@ out (sym/softmax-output "softmax" {:data fc3})] out) ;=>#object[org.apache.mxnet.Symbol 0x1f43a406 "org.apache.mxnet.Symbol@1f43a406"] -;; You can also use as-> for easier threading +;; You can also write this with the `as->` threading macro. (def out (as-> (sym/variable "data") data @@ -75,10 +93,15 @@ ;=> #'tutorial.module/out -;; By default, context is the CPU. If you need data parallelization, you can specify a GPU context or an array of GPU contexts. -;; like this (m/module out {:contexts [(context/gpu)]}) +;; By default, context is the CPU. If you need data parallelization, +;; you can specify a GPU context or an array of GPU contexts, like +;; this: `(m/module out {:contexts [(context/gpu)]})` -;; Before you can compute with a module, you need to call `bind` to allocate the device memory and `initParams` or `set-params` to initialize the parameters. If you simply want to fit a module, you don’t need to call `bind` and `init-params` explicitly, because the `fit` function automatically calls them if they are needed. +;; Before you can compute with a module, you need to call `bind` to +;; allocate the device memory and `initParams` or `set-params` to +;; initialize the parameters. If you simply want to fit a module, you +;; don’t need to call `bind` and `init-params` explicitly, because the +;; `fit` function automatically calls them if they are needed. (let [mod (m/module out)] (-> mod @@ -86,29 +109,46 @@ :label-shapes (mx-io/provide-label train-data)}) (m/init-params))) -;; Now you can compute with the module using functions like `forward`, `backward`, etc. +;; Now you can compute with the module using functions like `forward`, +;; `backward`, etc. -;; Training, Predicting, and Evaluating +;;;; Training and Predicting -;;Modules provide high-level APIs for training, predicting, and evaluating. To fit a module, call the `fit` function with some DataIters: +;; Modules provide high-level APIs for training, predicting, and +;; evaluating. To fit a module, call the `fit` function with some data +;; iterators: -(def mod (m/fit (m/module out) {:train-data train-data :eval-data test-data :num-epoch 1})) +(def mod + (m/fit (m/module out) {:train-data train-data + :eval-data test-data + :num-epoch 1})) +;; => ;; Epoch 0 Train- [accuracy 0.12521666] ;; Epoch 0 Time cost- 8392 ;; Epoch 0 Validation- [accuracy 0.2227] -;; You can pass in batch-end callbacks using batch-end-callback and epoch-end callbacks using epoch-end-callback in the `fit-params`. You can also set parameters using functions like in the fit-params like optimizer and eval-metric. To learn more about the fit-params, see the fit-param function options. To predict with a module, call `predict` with a DataIter: +;; You can pass in batch-end callbacks using batch-end-callback and +;; epoch-end callbacks using epoch-end-callback in the +;; `fit-params`. You can also set parameters using functions like in +;; the fit-params like optimizer and eval-metric. To learn more about +;; the fit-params, see the fit-param function options. To predict with +;; a module, call `predict` with a DataIter: + +(def results + (m/predict mod {:eval-data test-data})) -(def results (m/predict mod {:eval-data test-data})) (first results) ;=>#object[org.apache.mxnet.NDArray 0x3540b6d3 "org.apache.mxnet.NDArray@a48686ec"] (first (ndarray/->vec (first results))) ;=>0.08261358 -;;The module collects and returns all of the prediction results. For more details about the format of the return values, see the documentation for the `predict` function. +;; The module collects and returns all of the prediction results. For +;; more details about the format of the return values, see the +;; documentation for the `predict` function. -;;When prediction results might be too large to fit in memory, use the `predict-every-batch` API +;; When prediction results might be too large to fit in memory, use +;; the `predict-every-batch` API. (let [preds (m/predict-every-batch mod {:eval-data test-data})] (mx-io/reduce-batches test-data @@ -118,23 +158,33 @@ ;;; do something (inc i)))) -;;If you need to evaluate on a test set and don’t need the prediction output, call the `score` function with a DataIter and an EvalMetric: +;; If you need to evaluate on a test set and don’t need the prediction +;; output, call the `score` function with a data iterator and an eval +;; metric: -(m/score mod {:eval-data test-data :eval-metric (eval-metric/accuracy)}) ;=>["accuracy" 0.2227] +(m/score mod {:eval-data test-data + :eval-metric (eval-metric/accuracy)}) ;=>["accuracy" 0.2227] -;;This runs predictions on each batch in the provided DataIter and computes the evaluation score using the provided EvalMetric. The evaluation results are stored in metric so that you can query later. +;; This runs predictions on each batch in the provided DataIter and +;; computes the evaluation score using the provided EvalMetric. The +;; evaluation results are stored in metric so that you can query +;; later. -;;Saving and Loading Module Parameters -;;To save the module parameters in each training epoch, use a `checkpoint` function +;;;; Saving and Loading + +;; To save the module parameters in each training epoch, use the +;; `save-checkpoint` function: (let [save-prefix "my-model"] (doseq [epoch-num (range 3)] (mx-io/do-batches train-data (fn [batch ;; do something -])) - (m/save-checkpoint mod {:prefix save-prefix :epoch epoch-num :save-opt-states true}))) + ])) + (m/save-checkpoint mod {:prefix save-prefix + :epoch epoch-num + :save-opt-states true}))) ;; INFO org.apache.mxnet.module.Module: Saved checkpoint to my-model-0000.params ;; INFO org.apache.mxnet.module.Module: Saved optimizer state to my-model-0000.states @@ -144,20 +194,22 @@ ;; INFO org.apache.mxnet.module.Module: Saved optimizer state to my-model-0002.states -;;To load the saved module parameters, call the `load-checkpoint` function: +;; To load the saved module parameters, call the `load-checkpoint` +;; function: (def new-mod (m/load-checkpoint {:prefix "my-model" :epoch 1 :load-optimizer-states true})) new-mod ;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 "org.apache.mxnet.module.Module@5304d0f4"] -;;To initialize parameters, Bind the symbols to construct executors first with bind function. Then, initialize the parameters and auxiliary states by calling `init-params` function. - +;; To initialize parameters, bind the symbols to construct executors +;; first with the `bind` function. Then, initialize the parameters and +;; auxiliary states by calling the `init-params` function.\ (-> new-mod - (m/bind {:data-shapes (mx-io/provide-data train-data) :label-shapes (mx-io/provide-label train-data)}) + (m/bind {:data-shapes (mx-io/provide-data train-data) + :label-shapes (mx-io/provide-label train-data)}) (m/init-params)) -;;To get current parameters, use `params` - +;; To get current parameters, use `params` (let [[arg-params aux-params] (m/params new-mod)] {:arg-params arg-params :aux-params aux-params}) @@ -178,20 +230,24 @@ new-mod ;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 "org.apache.mxnet. ;; :aux-params {}} -;;To assign parameter and aux state values, use `set-params` function. +;; To assign parameter and aux state values, use the `set-params` +;; function: +(m/set-params new-mod {:arg-params (m/arg-params new-mod) + :aux-params (m/aux-params new-mod)}) -(m/set-params new-mod {:arg-params (m/arg-params new-mod) :aux-params (m/aux-params new-mod)}) -;=> #object[org.apache.mxnet.module.Module 0x5304d0f4 "org.apache.mxnet.module.Module@5304d0f4"] -;;To resume training from a saved checkpoint, instead of calling `set-params`, directly call `fit`, passing the loaded parameters, so that `fit` knows to start from those parameters instead of initializing randomly: +;; To resume training from a saved checkpoint, pass the loaded +;; parameters to the `fit` function. This will prevent `fit` from +;; initializing randomly. -;; reset the training data before calling fit or you will get an error +;; (First, reset the training data before calling `fit` or you will +;; get an error) (mx-io/reset train-data) (mx-io/reset test-data) -(m/fit new-mod {:train-data train-data :eval-data test-data :num-epoch 2 - :fit-params (-> (m/fit-params {:begin-epoch 1}))}) - -;;Create fit-params, and then use it to set `begin-epoch` so that fit() knows to resume from a saved epoch. - - +;; Create `fit-params` and then use it to set `begin-epoch` so that +;; `fit` knows to resume from a saved epoch. +(m/fit new-mod {:train-data train-data + :eval-data test-data + :num-epoch 2 + :fit-params (m/fit-params {:begin-epoch 1})}) diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj index 858316eefdc4..8e51de215157 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/ndarray.clj @@ -16,42 +16,53 @@ ;; (ns tutorial.ndarray + "A REPL tutorial of the MXNet Clojure API for NDArray, based on + https://mxnet.incubator.apache.org/api/clojure/ndarray.html" (:require [org.apache.clojure-mxnet.ndarray :as ndarray] [org.apache.clojure-mxnet.context :as context])) -;;The NDArray package (mxnet.ndarray) contains tensor operations similar to numpy.ndarray. The syntax is also similar, except for some additional calls for dealing with I/O and multiple devices. +;; The NDArray API contains tensor operations similar to +;; `numpy.ndarray`. The syntax is also similar, except for some +;; additional calls for dealing with I/O and multiple devices. -;;Create NDArray -;;Create mxnet.ndarray as follows: -(def a (ndarray/zeros [100 50])) ;;all zero arrray of dimension 100 x 50 -(def b (ndarray/ones [256 32 128 1])) ;; all one array of dimension -(def c (ndarray/array [1 2 3 4 5 6] [2 3])) ;; array with contents of a shape 2 x 3 +;;;; Create NDArray -;;; There are also ways to convert to a vec or get the shape as an object or vec +;; Create an MXNet NDArray as follows: +(def a (ndarray/zeros [100 50])) ; all-zero array of dimension 100 x 50 +(def b (ndarray/ones [256 32 128 1])) ; all-one array of given dimensions +(def c (ndarray/array [1 2 3 4 5 6] [2 3])) ; array with given contents in shape 2 x 3 + +;;; There are also ways to convert an NDArray to a vec or to get the +;;; shape as an object or vec: (ndarray/->vec c) ;=> [1.0 2.0 3.0 4.0 5.0 6.0] (ndarray/shape c) ;=> #object[org.apache.mxnet.Shape 0x583c865 "(2,3)"] (ndarray/shape-vec c) ;=> [2 3] -;; NDArray Operations -;; Arithmtic Operations +;; There are some basic NDArray operations, like arithmetic and slice +;; operations. + + +;;;; NDArray Operations: Arithmetic + (def a (ndarray/ones [1 5])) (def b (ndarray/ones [1 5])) -(-> (ndarray/+ a b) (ndarray/->vec)) ;=> [2.0 2.0 2.0 2.0 2.0] +(ndarray/->vec (ndarray/+ a b)) ;=> [2.0 2.0 2.0 2.0 2.0] ;; original ndarrays are unchanged (ndarray/->vec a) ;=> [1.0 1.0 1.0 1.0 1.0] (ndarray/->vec b) ;=> [1.0 1.0 1.0 1.0 1.0] -;;inplace operators +;; inplace operators (ndarray/+= a b) (ndarray/->vec a) ;=> [2.0 2.0 2.0 2.0 2.0] -;; other arthimetic operations are similar +;; Other arthimetic operations are similar. + -;; Slice operations +;;;; NDArray Operations: Slice (def a (ndarray/array [1 2 3 4 5 6] [3 2])) (def a1 (ndarray/slice a 1)) @@ -62,7 +73,8 @@ (ndarray/shape-vec a2) ;=>[2 2] (ndarray/->vec a2) ;=> [3.0 4.0 5.0 6.0] -;; Dot Product + +;;;; NDArray Operations: Dot Product (def arr1 (ndarray/array [1 2] [1 2])) (def arr2 (ndarray/array [3 4] [2 1])) @@ -70,23 +82,40 @@ (ndarray/shape-vec res) ;=> [1 1] (ndarray/->vec res) ;=> [11.0] -;;Save and Load NDArray -;;You can use MXNet functions to save and load a map of NDArrays from file systems, as follows: + +;;;; Save and Load NDArray + +;; You can use MXNet functions to save and load a map of NDArrays from +;; file systems, as follows: (ndarray/save "filename" {"arr1" arr1 "arr2" arr2}) -;; you can also do "s3://path" or "hdfs" +;; (you can also do "s3://path" or "hdfs") + +(ndarray/save "/Users/daveliepmann/src/coursework/mxnet-clj-tutorials/abc" + {"arr1" arr1 "arr2" arr2}) -;; to load +;; To load: (def from-file (ndarray/load "filename")) + from-file ;=>{"arr1" #object[org.apache.mxnet.NDArray 0x6115ba61 "org.apache.mxnet.NDArray@43d85753"], "arr2" #object[org.apache.mxnet.NDArray 0x374b5eff "org.apache.mxnet.NDArray@5c93def4"]} -;;Multi-Device Support +;; The good thing about using the `save` and `load` interface is that +;; you can use the format across all `mxnet` language bindings. They +;; also already support Amazon S3 and HDFS. + + +;;;; Multi-Device Support -;;Device information is stored in the mxnet.Context structure. When creating NDArray in MXNet, you can use the context argument (the default is the CPU context) to create arrays on specific devices as follows: +;; Device information is stored in the `mxnet.Context` structure. When +;; creating NDArray in MXNet, you can use the context argument (the +;; default is the CPU context) to create arrays on specific devices as +;; follows: (def cpu-a (ndarray/zeros [100 200])) (ndarray/context cpu-a) ;=> #object[org.apache.mxnet.Context 0x3f376123 "cpu(0)"] (def gpu-b (ndarray/zeros [100 200] {:ctx (context/gpu 0)})) ;; to use with gpu -;;Currently, we do not allow operations among arrays from different contexts. To manually enable this, use the copyto function to copy the content to different devices, and continue computation: +;; Currently, we do not allow operations among arrays from different +;; contexts. To manually enable this, use the `copy-to` function to +;; copy the content to different devices, and continue computation. diff --git a/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj b/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj index bec71dee81f5..ebf4f7e96797 100644 --- a/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj +++ b/contrib/clojure-package/examples/tutorial/src/tutorial/symbol.clj @@ -16,79 +16,66 @@ ;; (ns tutorial.symbol + "A REPL tutorial of the MXNet Clojure Symbolic API, based on + https://mxnet.incubator.apache.org/api/clojure/symbol.html" (:require [org.apache.clojure-mxnet.executor :as executor] [org.apache.clojure-mxnet.ndarray :as ndarray] [org.apache.clojure-mxnet.symbol :as sym] [org.apache.clojure-mxnet.context :as context])) -;; How to compose symbols -;;The symbolic API provides a way to configure computation graphs. You can configure the graphs either at the level of neural network layer operations or as fine-grained operations. -;;The following example configures a two-layer neural network. +;;;; How to Compose Symbols +;; The symbolic API provides a way to configure computation +;; graphs. You can configure the graphs either at the level of neural +;; network layer operations or as fine-grained operations. + +;; The following example configures a two-layer neural network. (def data (sym/variable "data")) (def fc1 (sym/fully-connected "fc1" {:data data :num-hidden 128})) (def act1 (sym/activation "act1" {:data fc1 :act-type "relu"})) (def fc2 (sym/fully-connected "fc2" {:data act1 :num-hidden 64})) (def net (sym/softmax-output "out" {:data fc2})) -;; you could also combine this more dynamically with +;; This can also be combined more dynamically with the `as->` Clojure +;; threading form. (as-> (sym/variable "data") data (sym/fully-connected "fc1" {:data data :num-hidden 128}) - (sym/activation "act1" {:data data :act-type "relu"}) + (sym/activation "act1" {:data data :act-type "relu"}) (sym/fully-connected "fc2" {:data data :num-hidden 64}) - (sym/softmax-output "out" {:data data})) + (sym/softmax-output "out" {:data data})) net ;=> #object[org.apache.mxnet.Symbol 0x5c78c8c2 "org.apache.mxnet.Symbol@5c78c8c2"] - -;;The basic arithmetic operators (plus, minus, div, multiplication) - -;;The following example creates a computation graph that adds two inputs together. +;; The basic arithmetic operators (plus, minus, div, multiplication) +;; work as expected. The following example creates a computation graph +;; that adds two inputs together. (def a (sym/variable "a")) (def b (sym/variable "b")) (def c (sym/+ a b)) -;; Each symbol takes a (unique) string name. NDArray and Symbol both represent a single tensor. Operators represent the computation between tensors. Operators take symbol (or NDArray) as inputs and might also additionally accept other hyperparameters such as the number of hidden neurons (num_hidden) or the activation type (act_type) and produce the output. - -;; We can view a symbol simply as a function taking several arguments. And we can retrieve those arguments with the following method call: - -;;We can view a symbol simply as a function taking several arguments. And we can retrieve those arguments with the following method call: - -(sym/list-arguments net) - ;=> ["data" "fc1_weight" "fc1_bias" "fc2_weight" "fc2_bias" "out_label"] - -;; These arguments are the parameters and inputs needed by each symbol: - -;; data: Input data needed by the variable data. -;; fc1_weight and fc1_bias: The weight and bias for the first fully connected layer fc1. -;; fc2_weight and fc2_bias: The weight and bias for the second fully connected layer fc2. -;; out_label: The label needed by the loss. - -;;We can also specify the names explicitly: -(def net (sym/variable "data")) -(def w (sym/variable "myweight")) -(def net (sym/fully-connected "fc1" {:data net :weight w :num-hidden 128})) -(sym/list-arguments net) - ;=> ["data" "fc1_weight" "fc1_bias" "fc2_weight" "fc2_bias" "out_label" "myweight" "fc1_bias"] +;;;; More Complicated Compositions - -;;In the above example, FullyConnected layer has 3 inputs: data, weight, bias. When any input is not specified, a variable will be automatically generated for it. - - -;; More complicated composition - -;;MXNet provides well-optimized symbols for layers commonly used in deep learning (see src/operator). We can also define new operators in Python. The following example first performs an element-wise add between two symbols, then feeds them to the fully connected operator: +;; MXNet provides well-optimized symbols for layers commonly used in +;; deep learning (see src/operator). We can also define new operators +;; in Python. The following example first performs an element-wise add +;; between two symbols, then feeds them to the fully connected +;; operator: (def lhs (sym/variable "data1")) (def rhs (sym/variable "data2")) -(def net (sym/fully-connected "fc1" {:data (sym/+ lhs rhs) :num-hidden 128})) +(def net (sym/fully-connected "fc1" {:data (sym/+ lhs rhs) + :num-hidden 128})) (sym/list-arguments net) ;=> ["data1" "data2" "fc1_weight" "fc1_bias"] -;; Group Multiple Symbols -;;To construct neural networks with multiple loss layers, we can use mxnet.sym.Group to group multiple symbols together. The following example groups two outputs: + +;;;; Group Multiple Symbols + +;; To construct neural networks with multiple loss layers, we can use +;; `group` to group multiple symbols together. The following example +;; groups two outputs: (def net (sym/variable "data")) (def fc1 (sym/fully-connected {:data net :num-hidden 128})) @@ -96,56 +83,49 @@ net ;=> #object[org.apache.mxnet.Symbol 0x5c78c8c2 "org.apache.mxnet.Symbol@5c78 (def out1 (sym/softmax-output {:data net2})) (def out2 (sym/linear-regression-output {:data net2})) (def group (sym/group [out1 out2])) -(sym/list-outputs group);=> ["softmaxoutput0_output" "linearregressionoutput0_output"] +(sym/list-outputs group) ;=> ["softmaxoutput0_output" "linearregressionoutput0_output"] -;; Symbol Manipulation -;; One important difference of Symbol compared to NDArray is that we first declare the computation and then bind the computation with data to run. +;;;; Serialization -;; In this section, we introduce the functions to manipulate a symbol directly. But note that, most of them are wrapped by the module package. +;; You can use the `save` and `load` functions to serialize Symbol +;; objects as JSON. These functions have the advantage of being +;; language-agnostic and cloud-friendly. You can also get a JSON +;; string directly using `to-json`. -;; Shape and Type Inference -;; For each symbol, we can query its arguments, auxiliary states and outputs. We can also infer the output shape and type of the symbol given the known input shape or type of some arguments, which facilitates memory allocation. -(sym/list-arguments fc1) ;=> ["data" "fullyconnected1_weight" "fullyconnected1_bias"] -(sym/list-outputs fc1) ;=> ["fullyconnected1_output"] +;; The following example shows how to save a symbol to a file, load it +;; back, and compare two symbols using a JSON string. You can also +;; save to S3 as well. -;; infer the shapes given the shape of the input arguments -(let [[arg-shapes out-shapes] (sym/infer-shape fc1 {:data [2 1]})] - {:arg-shapes arg-shapes - :out-shapes out-shapes}) ;=> {:arg-shapes ([2 1] [128 1] [128]), :out-shapes ([2 128])} +(def a (sym/variable "a")) +(def b (sym/variable "b")) +(def c (sym/+ a b)) +(sym/save c "symbol-c.json") +(def c2 (sym/load "symbol-c.json")) +(= (sym/to-json c) (sym/to-json c2)) ;=>true -;; Bind with Data and Evaluate -;; The symbol c constructed above declares what computation should be run. To evaluate it, we first need to feed the arguments, namely free variables, with data. -;; We can do it by using the bind method, which accepts device context and a dict mapping free variable names to NDArrays as arguments and returns an executor. The executor provides forward method for evaluation and an attribute outputs to get all the results. +;;;; Executing Symbols + +;; To execute symbols, first we need to define the data that they +;; should run on. We can do this with the `bind` function, which +;; returns an executor. We then use `forward` to evaluate and +;; `outputs` to get the results. (def a (sym/variable "a")) (def b (sym/variable "b")) (def c (sym/+ a b)) -(def ex (sym/bind c {"a" (ndarray/ones [2 2]) "b" (ndarray/ones [2 2])})) +(def ex + (sym/bind c {"a" (ndarray/ones [2 2]) + "b" (ndarray/ones [2 2])})) + (-> (executor/forward ex) (executor/outputs) (first) (ndarray/->vec));=> [2.0 2.0 2.0 2.0] -;;We can evaluate the same symbol on GPU with different data. -;; To do this you must have the correct native library jar defined as a dependency - -;;Note In order to execute the following section on a cpu set gpu_device to (cpu). - - -(def ex (sym/bind c (context/gpu 0) {"a" (ndarray/ones [2 2]) "b" (ndarray/ones [2 2])})) - -;; Serialization -;; There are two ways to save and load the symbols. You can use the mxnet.Symbol.save and mxnet.Symbol.load functions to serialize the Symbol objects. The advantage of using save and load functions is that it is language agnostic and cloud friendly. The symbol is saved in JSON format. You can also get a JSON string directly using mxnet.Symbol.toJson. Refer to API documentation for more details. - -;; The following example shows how to save a symbol to a file, load it back, and compare two symbols using a JSON string. You can also save to S3 as well - -(def a (sym/variable "a")) -(def b (sym/variable "b")) -(def c (sym/+ a b)) -(sym/save c "symbol-c.json") -(def c2 (sym/load "symbol-c.json")) -(= (sym/to-json c) (sym/to-json c2)) ;=>true - +;; We can evaluate the same symbol on GPU with different data. +;; (To do this you must have the correct native library jar defined as a dependency.) +(def ex (sym/bind c (context/gpu 0) {"a" (ndarray/ones [2 2]) + "b" (ndarray/ones [2 2])})) diff --git a/contrib/clojure-package/project.clj b/contrib/clojure-package/project.clj index 3779c82cf6d5..3e81dbe28379 100644 --- a/contrib/clojure-package/project.clj +++ b/contrib/clojure-package/project.clj @@ -17,6 +17,9 @@ (defproject org.apache.mxnet.contrib.clojure/clojure-mxnet "1.3.1-SNAPSHOT" :description "Clojure package for MXNet" + :url "https://github.com/apache/incubator-mxnet" + :license {:name "Apache License" + :url "http://www.apache.org/licenses/LICENSE-2.0"} :dependencies [[org.clojure/clojure "1.9.0"] [t6/from-scala "0.3.0"] diff --git a/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj new file mode 100644 index 000000000000..6e726eba9da6 --- /dev/null +++ b/contrib/clojure-package/src/org/apache/clojure_mxnet/image.clj @@ -0,0 +1,139 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns org.apache.clojure-mxnet.image + (:require [t6.from-scala.core :refer [$ $$] :as $] + [org.apache.clojure-mxnet.dtype :as dtype] + [org.apache.clojure-mxnet.ndarray :as ndarray] + [org.apache.clojure-mxnet.util :as util] + [clojure.spec.alpha :as s]) + (:import (org.apache.mxnet Image NDArray) + (java.io InputStream))) + +;; Flags for conversion of images +(def GRAYSCALE 0) +(def COLOR 1) + +(s/def ::input-stream #(instance? InputStream %)) +(s/def ::color-flag #{GRAYSCALE COLOR}) +(s/def ::to-rgb boolean?) +(s/def ::ndarray #(instance? NDArray %)) +(s/def ::output (s/or :empty nil? :ndarray ::ndarray)) +(s/def ::decode-image-opts + (s/keys :opt-un [::color-flag ::to-rgb ::output])) + +(defn decode-image + "Decodes an image from an input stream" + ([input-stream {:keys [color-flag to-rgb output] + :or {color-flag COLOR to-rgb true output nil} + :as opts}] + (util/validate! ::input-stream input-stream "Invalid input stream") + (util/validate! ::decode-image-opts opts "Invalid options for decoding") + (Image/imDecode input-stream color-flag to-rgb ($/option output))) + ([input-stream] + (decode-image input-stream {}))) + +(s/def ::filename string?) +(s/def ::optional-color-flag + (s/or :none nil? :some ::color-flag)) +(s/def ::optional-to-rgb + (s/or :none nil? :some ::to-rgb)) + +(defn read-image + "Reads an image file and returns an ndarray" + ([filename {:keys [color-flag to-rgb output] + :or {color-flag nil to-rgb nil output nil} + :as opts}] + (util/validate! ::filename filename "Invalid filename") + (util/validate! ::optional-color-flag color-flag "Invalid color flag") + (util/validate! ::optional-to-rgb to-rgb "Invalid conversion flag") + (util/validate! ::output output "Invalid output") + (Image/imRead + filename + ($/option color-flag) + ($/option to-rgb) + ($/option output))) + ([filename] + (read-image filename {}))) + +(s/def ::int int?) +(s/def ::optional-int (s/or :none nil? :some int?)) + +(defn resize-image + "Resizes the image array to (width, height)" + ([input w h {:keys [interpolation output] + :or {interpolation nil output nil} + :as opts}] + (util/validate! ::ndarray input "Invalid input array") + (util/validate! ::int w "Invalid width") + (util/validate! ::int h "Invalid height") + (util/validate! ::optional-int interpolation "Invalid interpolation") + (util/validate! ::output output "Invalid output") + (Image/imResize input w h ($/option interpolation) ($/option output))) + ([input w h] + (resize-image input w h {}))) + +(defn apply-border + "Pad image border" + ([input top bottom left right + {:keys [fill-type value values output] + :or {fill-type nil value nil values nil output nil} + :as opts}] + (util/validate! ::ndarray input "Invalid input array") + (util/validate! ::int top "Invalid top margin") + (util/validate! ::int bottom "Invalid bottom margin") + (util/validate! ::int left "Invalid left margin") + (util/validate! ::int right "Invalid right margin") + (util/validate! ::optional-int fill-type "Invalid fill type") + (util/validate! ::output output "Invalid output") + (Image/copyMakeBorder input top bottom left right + ($/option fill-type) + ($/option value) + ($/option values) + ($/option output))) + ([input top bottom left right] + (apply-border input top bottom left right {}))) + +(defn fixed-crop + "Return a fixed crop of the image" + [input x0 y0 w h] + (util/validate! ::ndarray input "Invalid input array") + (util/validate! ::int x0 "Invalid starting x coordinate") + (util/validate! ::int y0 "Invalid starting y coordinate") + (util/validate! ::int w "Invalid width") + (util/validate! ::int h "Invalid height") + (Image/fixedCrop input x0 y0 w h)) + +(defn rgb-array? + "Returns whether the ndarray is in the RGB format" + [input] + (util/validate! ::ndarray input "Invalid input array") + (let [shape (ndarray/shape-vec input)] + (and + (= 3 (count shape)) + (= 3 (shape 2))))) + +(s/def ::all-bytes #(= dtype/UINT8 (ndarray/dtype %))) +(s/def ::rgb-array rgb-array?) +(s/def ::to-image-ndarray + (s/and ::ndarray ::all-bytes ::rgb-array)) + +(defn to-image + "Convert a NDArray image in RGB format to a real image" + [input] + (util/validate! ::to-image-ndarray input "Invalid input array") + (Image/toImage input)) diff --git a/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj new file mode 100644 index 000000000000..38ab11c86012 --- /dev/null +++ b/contrib/clojure-package/test/org/apache/clojure_mxnet/image_test.clj @@ -0,0 +1,79 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + +(ns org.apache.clojure-mxnet.image-test + (:require [org.apache.clojure-mxnet.image :as image] + [org.apache.clojure-mxnet.ndarray :as ndarray] + [clojure.java.io :as io] + [clojure.test :refer :all]) + (:import (javax.imageio ImageIO))) + +(def tmp-dir (System/getProperty "java.io.tmpdir")) +(def image-path (.getAbsolutePath (io/file tmp-dir "Pug-Cookie.jpg"))) + +(defn download-image [] + (with-open [in (io/input-stream "https://s3.amazonaws.com/model-server/inputs/Pug-Cookie.jpg") + out (io/output-stream (io/file image-path))] + (io/copy in out))) + +(defn delete-image [] + (io/delete-file image-path)) + +(defn with-downloaded-image [f] + (download-image) + (f) + (delete-image)) + +(use-fixtures :once with-downloaded-image) + +(deftest test-decode-image + (let [img-arr (image/decode-image + (io/input-stream image-path)) + img-arr-2 (image/decode-image + (io/input-stream image-path) + {:color-flag image/GRAYSCALE})] + (is (= [576 1024 3] (ndarray/shape-vec img-arr))) + (is (= [576 1024 1] (ndarray/shape-vec img-arr-2))))) + +(deftest test-read-image + (let [img-arr (image/read-image image-path) + img-arr-2 (image/read-image + image-path + {:color-flag image/GRAYSCALE})] + (is (= [576 1024 3] (ndarray/shape-vec img-arr))) + (is (= [576 1024 1] (ndarray/shape-vec img-arr-2))))) + +(deftest test-resize-image + (let [img-arr (image/read-image image-path) + resized-arr (image/resize-image img-arr 224 224)] + (is (= [224 224 3] (ndarray/shape-vec resized-arr))))) + +(deftest test-crop-image + (let [img-arr (image/read-image image-path) + cropped-arr (image/fixed-crop img-arr 0 0 224 224)] + (is (= [224 224 3] (ndarray/shape-vec cropped-arr))))) + +(deftest test-apply-border + (let [img-arr (image/read-image image-path) + padded-arr (image/apply-border img-arr 1 1 1 1)] + (is (= [578 1026 3] (ndarray/shape-vec padded-arr))))) + +(deftest test-to-image + (let [img-arr (image/read-image image-path) + resized-arr (image/resize-image img-arr 224 224) + new-img (image/to-image resized-arr)] + (is (= true (ImageIO/write new-img "png" (io/file tmp-dir "out.png")))))) diff --git a/cpp-package/README.md b/cpp-package/README.md index 2b6e0e39f0fd..c4fe63c9ec58 100644 --- a/cpp-package/README.md +++ b/cpp-package/README.md @@ -1,21 +1,46 @@ # MXNet C++ Package -To build the C++ package, please refer to [this guide](). +The MXNet C++ Package provides C++ API bindings to the users of MXNet. Currently, these bindings are not available as standalone package. +The users of these bindings are required to build this package as mentioned below. -A basic tutorial can be found at . +## Building C++ Package -The example directory contains examples for you to get started. +The cpp-package directory contains the implementation of C++ API. As mentioned above, users are required to build this directory or package before using it. +**The cpp-package is built while building the MXNet shared library, *libmxnet.so*.** + +###Steps to build the C++ package: +1. Building the MXNet C++ package requires building MXNet from source. +2. Clone the MXNet GitHub repository **recursively** to ensure the code in submodules is available for building MXNet. + ``` + git clone --recursive https://github.com/apache/incubator-mxnet mxnet + ``` + +3. Install the [prerequisites](), desired [BLAS libraries]() and optional [OpenCV, CUDA, and cuDNN]() for building MXNet from source. +4. There is a configuration file for make, [make/config.mk]() that contains all the compilation options. You can edit this file and set the appropriate options prior to running the **make** command. +5. Please refer to [platform specific build instructions]() and available [build configurations](https://mxnet.incubator.apache.org/install/build_from_source#build-configurations) for more details. +5. For enabling the build of C++ Package, set the **USE\_CPP\_PACKAGE = 1** in [make/config.mk](). Optionally, the compilation flag can also be specified on **make** command line as follows. + ``` + make -j USE_CPP_PACKAGE=1 + ``` + +## Usage -## Building C++ examples in examples folder +In order to consume the C++ API please follow the steps below. -From cpp-package/examples directory -- Build all examples in release mode: **make all** -- Build all examples in debug mode : **make debug** +1. Ensure that the MXNet shared library is built from source with the **USE\_CPP\_PACKAGE = 1**. +2. Include the [MxNetCpp.h]() in the program that is going to consume MXNet C++ API. + ``` + #include + ``` +3. While building the program, ensure that the correct paths to the directories containing header files and MXNet shared library. +4. The program links the MXNet shared library dynamically. Hence the library needs to be accessible to the program during runtime. This can be achieved by including the path to the shared library in the environment variable **LD\_LIBRARY\_PATH** for Linux, Mac. and Ubuntu OS and **PATH** for Windows OS. -By default, the examples are build to be run on GPU. -To build examples to run on CPU: -- Release: **make all MXNET_USE_CPU=1** -- Debug: **make debug MXNET_USE_CPU=1** +## Tutorial + +A basic tutorial can be found at . + +## Examples + +The example directory contains examples for you to get started. -The makefile will also download the necessary data files and store in data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.) diff --git a/cpp-package/example/Makefile b/cpp-package/example/Makefile index eb0676cedf5b..58d4c82bdb35 100644 --- a/cpp-package/example/Makefile +++ b/cpp-package/example/Makefile @@ -16,6 +16,7 @@ # under the License. prebuild : + @mkdir -p build $(shell ./get_data.sh) $(shell cp -r ../../lib ./) CPPEX_SRC = $(wildcard *.cpp) @@ -38,8 +39,9 @@ debug: CPPEX_CFLAGS += -DDEBUG -g debug: prebuild all + $(CPPEX_EXE):% : %.cpp - $(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -o $@ $(filter %.cpp %.a, $^) $(CPPEX_EXTRA_LDFLAGS) + $(CXX) -std=c++11 $(CFLAGS) $(CPPEX_CFLAGS) -o build/$@ $(filter %.cpp %.a, $^) $(CPPEX_EXTRA_LDFLAGS) clean: - rm -f $(CPPEX_EXE) + @rm -rf build diff --git a/cpp-package/example/README.md b/cpp-package/example/README.md new file mode 100644 index 000000000000..06ea17bbc1f1 --- /dev/null +++ b/cpp-package/example/README.md @@ -0,0 +1,106 @@ +# MXNet C++ Package Examples + +## Building C++ examples + +The examples are built while building the MXNet library and cpp-package from source . However, they can be built manually as follows + +From cpp-package/examples directory + +- Build all examples in release mode: **make all** +- Build all examples in debug mode: **make debug** + +By default, the examples are built to be run on GPU. To build examples to run on CPU: + +- Release: **make all MXNET\_USE\_CPU=1** +- Debug: **make debug MXNET\_USE\_CPU=1** + +The examples that are built to be run on GPU may not work on the non-GPU machines. +The makefile will also download the necessary data files and store in a data folder. (The download will take couple of minutes, but will be done only once on a fresh installation.) + + +## Examples + +This directory contains following examples. In order to run the examples, ensure that the path to the MXNet shared library is added to the OS specific environment variable viz. **LD\_LIBRARY\_PATH** for Linux, Mac and Ubuntu OS and **PATH** for Windows OS. For example `export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/home/ubuntu/incubator-mxnet/lib` on ubuntu using gpu. + +### [alexnet.cpp]() + +The example implements the C++ version of AlexNet. The networks trains on MNIST data. The number of epochs can be specified as a command line argument. For example to train with 10 epochs use the following: + +``` +build/alexnet 10 +``` + +### [googlenet.cpp]() + +The code implements a GoogLeNet/Inception network using the C++ API. The example uses MNIST data to train the network. By default, the example trains the model for 100 epochs. The number of epochs can also be specified in the command line. For example, to train the model for 10 epochs use the following: + +``` +build/googlenet 10 +``` + +### [mlp.cpp]() + +The code implements a multilayer perceptron from scratch. The example creates its own dummy data to train the model. The example does not require command line parameters. It trains the model for 20,000 epochs. +To run the example use the following command: + +``` +build/mlp +``` + +### [mlp_cpu.cpp]() + +The code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of "SimpleBind" C++ API and MNISTIter. The example is designed to work on CPU. The example does not require command line parameters. +To run the example use the following command: + +``` +build/mlp_cpu +``` + +### [mlp_gpu.cpp]() + +The code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of the "SimpleBind" C++ API and MNISTIter. The example is designed to work on GPU. The example does not require command line arguments. To run the example execute following command: + +``` +build/mlp_gpu +``` + +### [mlp_csv.cpp]() + +The code implements a multilayer perceptron to train the MNIST data. The code demonstrates the use of the "SimpleBind" C++ API and CSVIter. The CSVIter can iterate data that is in CSV format. The example can be run on CPU or GPU. The example usage is as follows: + +``` +build/mlp_csv --train mnist_training_set.csv --test mnist_test_set.csv --epochs 10 --batch_size 100 --hidden_units "128,64,64 [--gpu]" +``` + +### [resnet.cpp]() + +The code implements a resnet model using the C++ API. The model is used to train MNIST data. The number of epochs for training the model can be specified on the command line. By default, model is trained for 100 epochs. For example, to train with 10 epochs use the following command: + +``` +build/resnet 10 +``` + +### [lenet.cpp]() + +The code implements a lenet model using the C++ API. It uses MNIST training data in CSV format to train the network. The example does not use built-in CSVIter to read the data from CSV file. The number of epochs can be specified on the command line. By default, the mode is trained for 100,000 epochs. For example, to train with 10 epochs use the following command: + +``` +build/lenet 10 +``` +### [lenet\_with\_mxdataiter.cpp]() + +The code implements a lenet model using the C++ API. It uses MNIST training data to train the network. The example uses built-in MNISTIter to read the data. The number of epochs can be specified on the command line. By default, the mode is trained for 100 epochs. For example, to train with 10 epochs use the following command: + +``` +build/lenet_with_mxdataiter 10 +``` + +In addition, there is `run_lenet_with_mxdataiter.sh` that downloads the mnist data and run `lenet_with_mxdataiter` example. + +### [inception_bn.cpp]() + +The code implements an Inception network using the C++ API with batch normalization. The example uses MNIST data to train the network. The model trains for 100 epochs. The example can be run by executing the following command: + +``` +build/inception_bn +``` diff --git a/cpp-package/example/example.mk b/cpp-package/example/example.mk index 4914b31ba840..ef99d7426414 100644 --- a/cpp-package/example/example.mk +++ b/cpp-package/example/example.mk @@ -18,7 +18,7 @@ CPPEX_SRC = $(wildcard cpp-package/example/*.cpp) CPPEX_EXE = $(patsubst cpp-package/example/%.cpp, build/cpp-package/example/%, $(CPPEX_SRC)) -CPPEX_CFLAGS += -Icpp-package/include -Ibuild/cpp-package/include +CPPEX_CFLAGS += -Icpp-package/include CPPEX_EXTRA_LDFLAGS := -L$(ROOTDIR)/lib -lmxnet EXTRA_PACKAGES += cpp-package-example-all @@ -30,8 +30,8 @@ cpp-package-example-all: cpp-package-all $(CPPEX_EXE) build/cpp-package/example/% : cpp-package/example/%.cpp lib/libmxnet.so $(CPP_PACKAGE_OP_H_FILE) @mkdir -p $(@D) - $(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -MM -MT cpp-package/example/$* $< >build/cpp-package/example//$*.d - $(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -o $@ $(filter %.cpp %.a, $^) $(LDFLAGS) $(CPPEX_EXTRA_LDFLAGS) + $(CXX) -std=c++11 $(CFLAGS) $(CPPEX_CFLAGS) -MM -MT cpp-package/example/$* $< >build/cpp-package/example//$*.d + $(CXX) -std=c++11 $(CFLAGS) $(CPPEX_CFLAGS) -o $@ $(filter %.cpp %.a, $^) $(LDFLAGS) $(CPPEX_EXTRA_LDFLAGS) cpp-package-example-clean: rm -rf build/cpp-package/example/* diff --git a/cpp-package/example/feature_extract/Makefile b/cpp-package/example/feature_extract/Makefile index f598183bd167..193eaa7e850b 100644 --- a/cpp-package/example/feature_extract/Makefile +++ b/cpp-package/example/feature_extract/Makefile @@ -27,12 +27,12 @@ LDFLAGS=$(COMMFLAGS) -L ../../../lib -lmxnet $(BLAS) $(CUDA) -lgomp -pthread all: feature_extract prepare_data_with_opencv feature_extract: ./feature_extract.cpp - $(CXX) -c -std=c++0x $(CFLAGS) $^ + $(CXX) -c -std=c++11 $(CFLAGS) $^ $(CXX) $(basename $@).o -o $@ $(LDFLAGS) -rm -f $(basename $@).o prepare_data_with_opencv: ./prepare_data_with_opencv.cpp - $(CXX) -c -std=c++0x $(OPENCV_CFLAGS) $^ + $(CXX) -c -std=c++11 $(OPENCV_CFLAGS) $^ $(CXX) $(basename $@).o -o $@ $(OPENCV_LDFLAGS) -rm -f $(basename $@).o diff --git a/cpp-package/example/mlp_csv.cpp b/cpp-package/example/mlp_csv.cpp new file mode 100644 index 000000000000..8aec4b76d917 --- /dev/null +++ b/cpp-package/example/mlp_csv.cpp @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Example: mlp_csv + * Description: + * The following example demonstrates how to use CSVIter. This example creates + * mlp (multi-layer perceptron) model and trains the MNIST data which is in + * CSV format. + */ +#include +#include "utils.h" +#include "mxnet-cpp/MxNetCpp.h" + +using namespace mxnet::cpp; + +/* + * Implementing the mlp symbol with given hidden units configuration. + */ +Symbol mlp(const std::vector &hidden_units) { + auto data = Symbol::Variable("data"); + auto label = Symbol::Variable("label"); + + std::vector weights(hidden_units.size()); + std::vector biases(hidden_units.size()); + std::vector outputs(hidden_units.size()); + + for (size_t i = 0; i < hidden_units.size(); ++i) { + weights[i] = Symbol::Variable("w" + std::to_string(i)); + biases[i] = Symbol::Variable("b" + std::to_string(i)); + Symbol fc = FullyConnected( + i == 0? data : outputs[i-1], // data + weights[i], + biases[i], + hidden_units[i]); + outputs[i] = i == hidden_units.size()-1 ? fc : Activation(fc, ActivationActType::kRelu); + } + return SoftmaxOutput(outputs.back(), label); +} + +/* + * Convert the input string of number of hidden units into the vector of integers. + */ +std::vector getLayers(const std::string& hidden_units_string) { + std::vector hidden_units; + char *pNext; + int num_unit = strtol(hidden_units_string.c_str(), &pNext, 10); + hidden_units.push_back(num_unit); + while (*pNext) { + num_unit = strtol(pNext, &pNext, 10); + hidden_units.push_back(num_unit); + } + return hidden_units; +} + +void printUsage() { + std::cout << "Usage:" << std::endl; + std::cout << "mlp_csv --train mnist_training_set.csv --test mnist_test_set.csv --epochs 10 " + << "--batch_size 100 --hidden_units \"128 64 64\" [--gpu]" << std::endl; + std::cout << "The example uses mnist data in CSV format. The MNIST data in CSV format assumes " + << "the column 0 to be label and the rest 784 column to be data." << std::endl; + std::cout << "By default, the example uses 'cpu' context. If '--gpu' is specified, " + << "program uses 'gpu' context." <