Add minimal forwarding RPC server for host driven python execution on…

… Hexagon (apache#9526)

* Minimal proxy RPC server for hexagon. Functions by routing from
Android to Hexagon via QTI FastRPC calls. Interim solution until
Hexagon on-device RPC server is ready.

* Apply clang-format.

* Fix build to support building alongside Hexagon Launcher.

* Add readme.

* src/runtime/hexagon/rpc -> src/runtime/hexagon/proxy_rpc

* Added small refactors to hexagon test_matmul.py.

* Add skipif on additional env vars.

* Fix IOS build.

* Rename USE_HEXAGON_PROXY_RPC and add tvm_options entry.

* Add NDArray::Container deleters.

Co-authored-by: Eric Lunderberg <[email protected]>

CMakeLists.txt

@@ -30,6 +30,8 @@ tvm_option(USE_ROCM "Build with ROCM" OFF)
 tvm_option(ROCM_PATH "The path to rocm" /opt/rocm)
 tvm_option(USE_HEXAGON_DEVICE "Build with Hexagon device support in TVM runtime" OFF)
 tvm_option(USE_HEXAGON_SDK "Path to the Hexagon SDK root (required for Hexagon support in TVM runtime or for building TVM runtime for Hexagon)" /path/to/sdk)
+tvm_option(USE_HEXAGON_LAUNCHER "Build the Hexagon graph launcher application" OFF)
+tvm_option(USE_HEXAGON_PROXY_RPC "Build the Hexagon Proxy RPC server application" OFF)
 tvm_option(USE_RPC "Build with RPC" ON)
 tvm_option(USE_THREADS "Build with thread support" ON)
 tvm_option(USE_LLVM "Build with LLVM, can be set to specific llvm-config path" OFF)

apps/hexagon_launcher/README.md

@@ -63,10 +63,10 @@ The Hexagon launcher application is an android binary and thus requires the use
 of an android toolchain for compilation. Similarly, the Hexagon tvm runtime
 requires the use of the Hexagon toolchain and depends on the Hexagon SDK. The
 resulting hexagon launcher binaries can be found in the `apps_hexagon_launcher`
-subdirectory of the cmake build directory. Please note that the above command
-will not build support for Hexagon codegen in the TVM library, for that please
-additionally define the `USE_HEXAGON_DEVICE` variable. Also, the LLVM used in
-`USE_LLVM` should have Hexagon target built in.
+subdirectory of the cmake build directory. The above command
+will build support for Hexagon codegen in the TVM library that requires 
+`USE_LLVM` to be set to an llvm-config that has the Hexagon target built in.
+
 
 ### Manual compilation
 

apps/hexagon_proxy_rpc/Readme.md

@@ -0,0 +1,82 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+# Hexagon Proxy RPC server
+
+The proxy RPC server for Hexagon is a wrapper which takes standard TVM RPC calls from a python host
+to a remote Android device and forwards them across FastRPC to Hexagon. This RPC flow will be replaced 
+by running a minimal RPC server directly on Hexagon. For now we provide a prototype forwarding RPC server 
+for host driven execution on Hexagon.
+
+## Compilation
+
+Project inventory: 
+* Android
+  * libtvm_runtime.so (containing HexagonHostDeviceAPI src/runtime/Hexagon/proxy_rpc/device_api.cc)
+  * tvm_rpc (C++ RPC server)
+  * librpc_env (Hexagon specific RPC proxy environment)
+
+* Hexagon
+  * libhexagon_proxy_rpc_skel.so (Hexagon device code containing FastRPC endpoints for the Hexagon Proxy RPC server)
+
+All Android and Hexagon device artifacts will be placed in `apps_hexagon_proxy_rpc` from which they can be pushed
+to an attached `adb` device.
+
+### Prerequisites
+
+1. Android NDK version r19c or later.
+2. Hexagon SDK version 4.0.0 or later.
+
+Android NDK can be downloaded from https://developer.android.com/ndk.
+Hexagon SDK is available at //developer.qualcomm.com/software/Hexagon-dsp-sdk.
+
+### Compilation with TVM
+
+Building the Hexagon Proxy RPC as a component of the main TVM build
+used for Hexagon codegen can be achieved by setting `USE_HEXAGON_PROXY_RPC=ON`.
+A minimal  example invocation for compiling TVM along with the Hexagon Proxy RPC server 
+is included below:
+
+```
+cmake -DCMAKE_C_COMPILER=/path/to/clang \
+      -DCMAKE_CXX_COMPILER=/path/to/clang++ \
+      -DCMAKE_CXX_FLAGS='-stdlib=libc++' \
+      -DCMAKE_CXX_STANDARD=14 \
+      -DUSE_RPC=ON \
+      -DUSE_LLVM=/path/to/llvm/bin/llvm-config \
+      -DUSE_HEXAGON_PROXY_RPC=ON \
+      -DANDROID_ABI=arm64-v8a \
+      -DANDROID_PLATFORM=android-28 \
+      -DUSE_ANDROID_TOOLCHAIN=/path/to/android-ndk/build/cmake/android.toolchain.cmake \
+      -DUSE_HEXAGON_ARCH=v65|v66|v68 \
+      -DUSE_HEXAGON_SDK=/path/to/Hexagon/SDK \
+      -DUSE_HEXAGON_TOOLCHAIN=/path/to/Hexagon/toolchain/ ..
+```
+
+where `v65|v66|v68` means "one of" these architecture versions.
+The Hexagon proxy RPC application (tvm_rpc) is an android binary and thus requires the use 
+of an android toolchain for compilation. Similarly, the Hexagon tvm runtime 
+requires the use of the Hexagon toolchain and depends on the Hexagon SDK. The 
+resulting Hexagon launcher binaries can be found in the `apps_Hexagon_launcher`
+subdirectory of the cmake build directory. The above command
+will build support for Hexagon codegen in the TVM library that requires 
+`USE_LLVM` to be set to an llvm-config that has the Hexagon target built in.
+
+
+# Disclaimer
+
+The Hexagon proxy RPC is intended for use with prototyping and does not utilize any
+performance acceleration, as such the measured performance may be very poor.

apps/hexagon_proxy_rpc/cmake/HexagonRPC.cmake

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(NOT DEFINED USE_HEXAGON_SDK)
+  message(SEND_ERROR "Please set USE_HEXAGON_SDK to the location of Hexagon SDK")
+endif()
+if (NOT DEFINED USE_HEXAGON_ARCH)
+  message(SEND_ERROR "Please set USE_HEXAGON_ARCH to the Hexagon architecture version")
+endif()
+
+set(TVM_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../")
+
+include(ExternalProject)
+include("${TVM_SOURCE_DIR}/cmake/modules/HexagonSDK.cmake")
+
+find_hexagon_sdk_root("${USE_HEXAGON_SDK}" "${USE_HEXAGON_ARCH}")
+
+include_directories(SYSTEM ${HEXAGON_SDK_INCLUDES} ${HEXAGON_REMOTE_ROOT})
+
+set(QAIC_EXE "${HEXAGON_QAIC_EXE}")
+foreach(INCDIR IN LISTS HEXAGON_SDK_INCLUDES HEXAGON_REMOTE_ROOT)
+  list(APPEND QAIC_FLAGS "-I${INCDIR}")
+endforeach()
+
+set(HEXAGON_PROXY_RPC_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../")
+set(CMAKE_SKIP_RPATH TRUE)
+
+# Qaic for the domain header.
+#
+# Don't add paths to these filenames, or otherwise cmake may spontaneously
+# add -o option to the qaic invocation (with an undesirable path).
+set(HEXAGON_PROXY_RPC_IDL "hexagon_proxy_rpc.idl")
+set(HEXAGON_PROXY_RPC_H "hexagon_proxy_rpc.h")
+set(HEXAGON_PROXY_RPC_SKEL_C "hexagon_proxy_rpc_skel.c")
+set(HEXAGON_PROXY_RPC_STUB_C "hexagon_proxy_rpc_stub.c")
+
+include_directories(
+  "${HEXAGON_PROXY_RPC_SRC}"
+  "${TVM_SOURCE_DIR}/include"
+  "${TVM_SOURCE_DIR}/3rdparty/dlpack/include"
+  "${TVM_SOURCE_DIR}/3rdparty/dmlc-core/include"
+)

apps/hexagon_proxy_rpc/cmake/android/CMakeLists.txt

@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.2)
+project(HexagonAndroidRPC C CXX)
+
+include("${CMAKE_CURRENT_SOURCE_DIR}/../HexagonRPC.cmake")
+
+add_custom_command(
+  OUTPUT ${HEXAGON_PROXY_RPC_STUB_C} ${HEXAGON_PROXY_RPC_H}
+  COMMAND ${QAIC_EXE} ${QAIC_FLAGS} "${HEXAGON_PROXY_RPC_SRC}/${HEXAGON_PROXY_RPC_IDL}"
+  MAIN_DEPENDENCY "${HEXAGON_PROXY_RPC_SRC}/${HEXAGON_PROXY_RPC_IDL}"
+)
+
+include_directories(SYSTEM
+  "${HEXAGON_SDK_INCLUDES}"
+  "${HEXAGON_RPCMEM_ROOT}/inc"
+  "${CMAKE_CURRENT_BINARY_DIR}"   # Output of qaic will go here
+)
+
+link_directories(${HEXAGON_REMOTE_ROOT})
+
+add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+set(TVM_RPC_ENV_SOURCES
+  ${HEXAGON_PROXY_RPC_SRC}/rpc_env.cc
+)
+
+add_library(rpc_env SHARED
+  ${TVM_RPC_ENV_SOURCES}
+  ${HEXAGON_PROXY_RPC_H}
+  ${HEXAGON_PROXY_RPC_STUB_C}
+)
+
+ExternalProject_Add(android_tvm_runtime
+  SOURCE_DIR "${TVM_SOURCE_DIR}"
+  BUILD_COMMAND $(MAKE) runtime
+  CMAKE_ARGS
+  "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+  "-DANDROID_PLATFORM=${ANDROID_PLATFORM}"
+  "-DANDROID_ABI=${ANDROID_ABI}"
+  "-DCMAKE_CXX_STANDARD=14"
+  "-DUSE_LIBBACKTRACE=OFF"
+  "-DUSE_LLVM=OFF"
+  "-DUSE_RPC=ON"
+  "-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
+  "-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
+  INSTALL_COMMAND ""
+  BUILD_ALWAYS ON
+)
+ExternalProject_Get_Property(android_tvm_runtime BINARY_DIR)
+ExternalProject_Add_Step(android_tvm_runtime copy_binaries
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${BINARY_DIR}/libtvm_runtime.so
+    ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDEES install
+)
+
+add_dependencies(rpc_env android_tvm_runtime)
+add_library(a_tvm_runtime SHARED IMPORTED)
+set_target_properties(a_tvm_runtime PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/libtvm_runtime.so")
+
+target_link_libraries(rpc_env cdsprpc log a_tvm_runtime)
+
+# TVM CPP RPC build
+set(TVM_RPC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../cpp_rpc")
+
+
+set(TVM_RPC_SOURCES
+  ${TVM_RPC_DIR}/main.cc
+  ${TVM_RPC_DIR}/rpc_server.cc
+)
+
+# Set output to same directory as the other TVM libs
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+add_executable(tvm_rpc ${TVM_RPC_SOURCES})
+
+
+target_include_directories(
+  tvm_rpc
+  PUBLIC "${TVM_RPC_DIR}../../include"
+  PUBLIC "${TVM_RPC_DIR}../../3rdparty/dlpack"
+  PUBLIC "${TVM_RPC_DIR}../../3rdparty/dmlc-core"
+)
+
+add_dependencies(rpc_env android_tvm_runtime)
+target_link_libraries(rpc_env a_tvm_runtime)
+
+add_dependencies(tvm_rpc android_tvm_runtime rpc_env)
+target_link_libraries(tvm_rpc a_tvm_runtime rpc_env)

apps/hexagon_proxy_rpc/cmake/hexagon/CMakeLists.txt

@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.2)
+project(HexagonRPCSkel C CXX)
+
+include("${CMAKE_CURRENT_SOURCE_DIR}/../HexagonRPC.cmake")
+
+add_custom_command(
+  OUTPUT ${HEXAGON_PROXY_RPC_SKEL_C} ${HEXAGON_PROXY_RPC_H}
+  COMMAND ${QAIC_EXE} ${QAIC_FLAGS} "${HEXAGON_PROXY_RPC_SRC}/${HEXAGON_PROXY_RPC_IDL}"
+  MAIN_DEPENDENCY "${HEXAGON_PROXY_RPC_SRC}/${HEXAGON_PROXY_RPC_IDL}"
+)
+
+include_directories(SYSTEM
+  ${HEXAGON_QURT_INCLUDES}
+  ${CMAKE_CURRENT_BINARY_DIR}   # Output of qaic will go here
+)
+
+link_directories(${HEXAGON_QURT_LIBS})
+
+add_definitions(-D_MACH_I32=int)
+add_definitions(-DDMLC_CXX11_THREAD_LOCAL=0)
+add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+# Extra compile flags (both C and C++).
+set(EXTRA_COMP_FLAGS
+  "-O3"
+  "-m${USE_HEXAGON_ARCH}"
+)
+string(REGEX REPLACE ";" " " EXTRA_COMP_FLAGS_STR "${EXTRA_COMP_FLAGS}")
+set(CMAKE_C_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_C_FLAGS}")
+set(CMAKE_CXX_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_CXX_FLAGS}")
+
+set(SKEL_SRCS
+  "${HEXAGON_PROXY_RPC_SRC}/hexagon_core.cc"
+)
+
+add_library(hexagon_proxy_rpc_skel SHARED
+  "${HEXAGON_PROXY_RPC_H}"
+  "${HEXAGON_PROXY_RPC_SKEL_C}"
+  "${SKEL_SRCS}"
+)
+
+ExternalProject_Add(static_hexagon_tvm_runtime
+  SOURCE_DIR "${TVM_SOURCE_DIR}"
+  BUILD_COMMAND $(MAKE) runtime
+  CMAKE_ARGS
+  "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+  "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+  "-DUSE_HEXAGON_ARCH=${USE_HEXAGON_ARCH}"
+  "-DCMAKE_CXX_STANDARD=14"
+  "-DUSE_LIBBACKTRACE=OFF"
+  "-DUSE_LLVM=OFF"
+  "-DUSE_RPC=OFF"
+  "-DBUILD_STATIC_RUNTIME=ON"
+  "-DUSE_HEXAGON_SDK=${USE_HEXAGON_SDK}"
+  INSTALL_COMMAND ""
+  BUILD_ALWAYS ON
+)
+ExternalProject_Get_Property(static_hexagon_tvm_runtime BINARY_DIR)
+
+add_dependencies(hexagon_proxy_rpc_skel static_hexagon_tvm_runtime)
+add_library(h_tvm_runtime STATIC IMPORTED)
+set_target_properties(h_tvm_runtime PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/libtvm_runtime.a")
+
+target_link_libraries(hexagon_proxy_rpc_skel -Wl,--whole-archive h_tvm_runtime -Wl,--no-whole-archive)