[v1.x] Backport edge pipeline (apache#18375)

* Update edge toolchain

* Support platforms without rand_r

* Fix the URL to the IUS repository

* compiler warnings

* Use a pre-c++17 way of distinguishing between device types

* Greatly simplify qemu setup

* Request the C++ standard library and extensions

* Upgrade dmlc-core to resolve build errors

* Remove leftovers from C++17 dev type check

* Fix CPU-only RRNOp Forward

* Change the ARM8 build to work like the ARM7 build

* Revert "Fix CPU-only RRNOp Forward"

This reverts commit 0a921a4.

* Hack around the lack of constexpr if

* Adjust the list of files to be packed in ARM jobs

Co-authored-by: Leonard Lausen <[email protected]>

3rdparty/mshadow/mshadow/logging.h

@@ -223,7 +223,12 @@ class LogMessageFatal {
   ~LogMessageFatal() MSHADOW_THROW_EXCEPTION {
     // throwing out of destructor is evil
     // hopefully we can do it here
+#pragma GCC diagnostic push
+#if __GNUC__ >= 7
+#pragma GCC diagnostic ignored "-Wterminate"
+#endif
     throw Error(log_stream_.str());
+#pragma GCC diagnostic pop
   }
 
  private:

3rdparty/mshadow/mshadow/packet-inl.h

@@ -93,7 +93,11 @@ inline void* AlignedMallocPitch(size_t *out_pitch,
   if (res == NULL) {
     LOG(FATAL) << "AlignedMallocPitch failed";
   }
+#if __GNUC__ >= 6
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
   return res;
+#pragma GCC diagnostic pop
 }
 
 /*!

CMakeLists.txt

@@ -7,6 +7,9 @@ if(CMAKE_CROSSCOMPILING)
 endif()
 
 project(mxnet C CXX)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS ON)
 
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/config.cmake)
   # Load config.cmake only if mxnet is not compiled as a dependency of another project
@@ -202,7 +205,7 @@ endif(MSVC)
 
 if(NOT mxnet_LINKER_LIBS)
   set(mxnet_LINKER_LIBS "")
-endif(NOT mxnet_LINKER_LIBS)
+endif()
 
 if(USE_GPROF)
   message(STATUS "Using GPROF")

Makefile

@@ -99,7 +99,9 @@ CFLAGS += -DDMLC_LOG_STACK_TRACE_SIZE=0
 CFLAGS += -DDMLC_LOG_FATAL_THROW=1
 
 ifeq ($(DEV), 1)
-	CFLAGS += -g -Werror
+  # Excluded from Werror:
+  # 1) variables used in '#pragma omp parallel' are considered unused
+	CFLAGS += -g -Werror -Wno-error=unused-variable -Wno-error=maybe-uninitialized -Wno-error=unused-function
 	NVCCFLAGS += -Werror cross-execution-space-call
 endif
 

ci/README.md

@@ -111,90 +111,37 @@ significantly. You can set this directory explicitly by setting CCACHE_DIR envir
 variable. All ccache instances are currently set to be 10 Gigabytes max in size.
 
 
-## Testing with QEMU
-To run the unit tests under qemu:
-```
-./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu
-```
-
-To get a shell on the container and debug issues with the emulator itself, we build the container
-and then execute it interactively. We can afterwards use port 2222 on the host to connect with SSH.
-
-
-```
-ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti mxnetci/build.test.arm_qemu
-```
+## Testing with ARM / Edge devices with QEMU
 
-Then from another terminal:
+We build on [QEMU](https://www.qemu.org/) and Linux [Kernel Support for
+miscellaneous Binary
+Formats](https://www.kernel.org/doc/html/v5.6/admin-guide/binfmt-misc.html) for
+testing MXNet on edge devices. Test can be invoked with the same syntax as for
+non-virtualized platforms:
 
 ```
-ssh -o StrictHostKeyChecking=no -p 2222 qemu@localhost
+./build.py -p armv7
+./build.py -p test.armv7 /work/runtime_functions.sh unittest_ubuntu_python3_armv7
 ```
 
-There are two pre-configured users: `root` and `qemu` both without passwords.
-
-
-### Example of reproducing a test result with QEMU on ARM
-
-
-You might want to enable a debug build first:
-
-```
-$ git diff
-diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
-index 39631f9..666ceea 100755
---- a/ci/docker/runtime_functions.sh
-+++ b/ci/docker/runtime_functions.sh
-@@ -172,6 +172,7 @@ build_armv7() {
-         -DUSE_LAPACK=OFF \
-         -DBUILD_CPP_EXAMPLES=OFF \
-         -Dmxnet_LINKER_LIBS=-lgfortran \
-+        -DCMAKE_BUILD_TYPE=Debug \
-         -G Ninja /work/mxnet
-
-     ninja -v
+For the test step to succeed, you must run Linux kernel 4.8 or later and have qemu installed.
 
+On Debian and Ubuntu systems, run the following command to install the dependencies:
 ```
+sudo apt install binfmt-support qemu-user-static
 
-Then we build the project for armv7, the test container and start QEMU inside docker:
-
-```
-ci/build.py -p armv7
-ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti mxnetci/build.test.arm_qemu
+# Use qemu-binfmt-conf.sh to register all binary types with the kernel
+wget https://raw.githubusercontent.com/qemu/qemu/stable-4.1/scripts/qemu-binfmt-conf.sh
+chmod +x qemu-binfmt-conf.sh
+sudo ./qemu-binfmt-conf.sh --persistent yes --qemu-suffix "-static" --qemu-path "/usr/bin" --systemd ALL
 ```
 
-
-
-At this point we copy artifacts and sources to the VM, in another terminal (host) do the following:
+If you run into segmentation faults at the beginning of the emulated tests, you
+probably have a ancient version of Qemu on your system (or found a bug in
+upstream Qemu). In that situation, you can rely on the
+`multiarch/qemu-user-static` Docker project to register a set of up-to-date Qemu
+binaries from their Docker image with your kernel:
 
 ```
-# Copy mxnet sources to the VM
-rsync --delete -e 'ssh -p2222' --exclude='.git/' -zvaP ./ qemu@localhost:mxnet
-
-
-# Ssh into the vm
-ssh -p2222 qemu@localhost
-
-cd mxnet
-
-# Execute a single failing C++ test
-build/tests/mxnet_unit_tests --gtest_filter="ACTIVATION_PERF.ExecuteBidirectional"
-
-# To install MXNet:
-sudo pip3 install --upgrade --force-reinstall build/mxnet-1.3.1-py2.py3-none-any.whl
-
-# Execute a single python test:
-
-nosetests-3.4 -v -s tests/python/unittest/test_ndarray.py
-
-
-# Debug with cgdb
-sudo apt install -y libstdc++6-6-dbg
-cgdb build/tests/mxnet_unit_tests
-
-(gdb) !pwd
-/home/qemu/mxnet
-(gdb) set substitute-path /work /home/qemu
-(gdb) set substitute-path /build/gcc-6-6mK9AW/gcc-6-6.3.0/build/arm-linux-gnueabihf/libstdc++-v3/include/ /usr/include/c++/6/
-(gdb) r --gtest_filter="ACTIVATION_PERF.ExecuteBidirectional"
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 ```

ci/build.py

@@ -70,7 +70,8 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
     return "nvidia-docker" if use_nvidia_docker else "docker"
 
 
-def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str:
+def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool,
+                 cache_intermediate: bool) -> str:
     """
     Build a container for the given platform
     :param platform: Platform
@@ -104,6 +105,8 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
            "--build-arg", "GROUP_ID={}".format(os.getgid())]
     if no_cache:
         cmd.append("--no-cache")
+    if cache_intermediate:
+        cmd.append("--rm=false")
     elif registry:
         cmd.extend(["--cache-from", tag])
     cmd.extend(["-t", tag, get_dockerfiles_path()])
@@ -330,6 +333,9 @@ def main() -> int:
     parser.add_argument("--no-cache", action="store_true",
                         help="passes --no-cache to docker build")
 
+    parser.add_argument("--cache-intermediate", action="store_true",
+                        help="passes --rm=false to docker build")
+
     parser.add_argument("-e", "--environment", nargs="*", default=[],
                         help="Environment variables for the docker container. "
                         "Specify with a list containing either names or name=value")
@@ -361,7 +367,8 @@ def main() -> int:
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
         if not args.run_only:
             build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry,
-                         num_retries=args.docker_build_retries, no_cache=args.no_cache)
+                         num_retries=args.docker_build_retries, no_cache=args.no_cache,
+                         cache_intermediate=args.cache_intermediate)
         else:
             logging.info("Skipping docker build step.")
 

ci/dev_menu.py

@@ -167,7 +167,7 @@ def provision_virtualenv(venv_path=DEFAULT_PYENV):
     ('[Docker] Python3 ARMv7 unittests (QEMU)',
     [
         "ci/build.py -p armv7",
-        "ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
+        "ci/build.py -p test.armv7 /work/runtime_functions.sh unittest_ubuntu_python3_armv7"
     ]),
     ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',
        Confirm("ci/docker/runtime_functions.sh clean_repo"))

ci/docker/Dockerfile.build.android_armv7

@@ -18,68 +18,46 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM dockcross/base
-MAINTAINER Pedro Larroy "[email protected]"
-
-# The cross-compiling emulator
-RUN apt-get update && apt-get install -y \
-  unzip
-
-ENV CROSS_TRIPLE=arm-linux-androideabi
-ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
-ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
-    AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
-    CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
-    CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
-    CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
-    LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld
-
-ENV ANDROID_NDK_REVISION 17b
-ENV ANDROID_NDK_API 27
-ENV ANDROID_NDK_ARCH arm
-WORKDIR /work/deps
-COPY install/android_ndk.sh /work/deps
-RUN /work/deps/android_ndk.sh
-
-ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
-
-# Build-time metadata as defined at http://label-schema.org
-ARG BUILD_DATE
-ARG IMAGE
-ARG VCS_REF
-ARG VCS_URL
-LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.name=$IMAGE \
-      org.label-schema.vcs-ref=$VCS_REF \
-      org.label-schema.vcs-url=$VCS_URL \
-      org.label-schema.schema-version="1.0"
-
-
-ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
-ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
-
-WORKDIR /work/deps
-
-COPY install/deb_ubuntu_ccache.sh /work/
-RUN /work/deb_ubuntu_ccache.sh
-WORKDIR /work
-COPY install/ubuntu_arm.sh /work/
-RUN /work/ubuntu_arm.sh
-
-COPY install/arm_openblas.sh /work/
-COPY install/android_armv7_openblas.sh /work/deps
-RUN /work/deps/android_armv7_openblas.sh
-
-ENV OpenBLAS_HOME=${CROSS_ROOT}
-ENV OpenBLAS_DIR=${CROSS_ROOT}
-
-WORKDIR /work
+FROM ubuntu:20.04
+
+ENV ARCH=armv7l \
+    HOSTCC=gcc \
+    HOSTCXX=g++ \
+    TARGET=ARMV7
+
+WORKDIR /usr/local
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    build-essential \
+    ninja-build \
+    cmake \
+    ccache \
+    git \
+    curl \
+    unzip \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip && \
+    unzip android-ndk-r19-linux-x86_64.zip && \
+    rm android-ndk-r19-linux-x86_64.zip
+ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake
+
+RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
+    mkdir /usr/local/openblas-android && \
+    cd /usr/local/OpenBLAS && \
+    export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \
+    make NOFORTRAN=1 ARM_SOFTFP_ABI=1 NO_SHARED=1 \
+        LDFLAGS="-L/usr/local/android-ndk-r19/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x -lm" \
+        CC=$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang AR=$TOOLCHAIN/bin/arm-linux-androideabi-ar && \
+    make PREFIX=/usr/local/openblas-android NO_SHARED=1 install && \
+    cd /usr/local && \
+    rm -rf OpenBLAS
+ENV OpenBLAS_HOME=/usr/local/openblas-android
 
 ARG USER_ID=0
 ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
-WORKDIR /work/mxnet
-
+WORKDIR /work/build