[microTVM][RVM] Add Arduino RVM (apache#8748)

* Functioning Arduino Vagrant VM

Begin building Arduino Vagrant VM

Mostly working Vagrant VM

Changes for debugging

Add ignored json file

Fix venv path

* Generalize parts of RVM for multiple platforms

cwd hack

Add unit tests from apps directory to task_python_microtvm.sh

Generalize parts of RVM for multiple platforms

* Add Vagrantfile lint exceptions

* Address PR comments

Address Mehrdad's PR comments

More PR comments

Documentation tweaks

Add dialout group to user

* Rerun tests

* Spresense fix

* Rerun CI tests

* Rerun tests

apps/microtvm/arduino/template_project/microtvm_api_server.py

@@ -65,6 +65,11 @@ class BoardAutodetectFailed(Exception):
         "architecture": "esp32",
         "board": "feathers2",
     },
+    "metrom4": {
+        "package": "adafruit",
+        "architecture": "samd",
+        "board": "adafruit_metro_m4",
+    },
     # Spresense only works as of its v2.3.0 sdk
     "spresense": {
         "package": "SPRESENSE",

apps/microtvm/reference-vm/arduino/.gitignore

@@ -0,0 +1 @@
+/.vagrant

apps/microtvm/reference-vm/arduino/README.md

@@ -0,0 +1,44 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+# microTVM Arduino Reference Virtual Machine
+
+This directory contains setup files for Arduino virtual machine used for testing
+microTVM platforms that are supported by [Arduino](https://www.arduino.cc/).
+
+## VM Information for Developers
+Arduino VM is published under [tlcpack](https://app.vagrantup.com/tlcpack).
+Here is a list of different release versions and their tools.
+
+(none currently)
+
+## Supported Arduino Boards
+This RVM has been tested and is known to work with these boards:
+- Adafruit Metro M4
+- Adafruit Pybadge
+- Arduino Due
+- Arduino Nano 33 BLE
+- Feather S2
+- Sony Spresense
+- Wio Terminal
+
+However, the RVM *should* work with any Arduino with sufficient memory, provided
+its core is installed in `base-box/base_box_provision.sh`.
+
+Note that this RVM does not work with the Teensy boards, even though they are
+supported by microTVM. This is because arduino-cli does not support Teensy
+boards (https://github.com/arduino/arduino-cli/issues/700)/).

apps/microtvm/reference-vm/arduino/Vagrantfile

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "tlcpack/microtvm-arduino-0.18.3"
+
+  if ENV.has_key?("TVM_RVM_NUM_CORES")
+    num_cores = ENV["TVM_RVM_NUM_CORES"]
+  else
+    num_cores = 2
+  end
+
+  if ENV.has_key?("TVM_RVM_RAM_BYTES")
+    ram_bytes = ENV["TVM_RVM_RAM_BYTES"]
+  else
+    ram_bytes = 2048
+  end
+
+  tvm_home = "../../../.."
+  dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())]
+  if ENV.has_key?("TVM_PROJECT_DIR") then
+    dirs_to_mount.append(ENV["TVM_PROJECT_DIR"])
+    puts "NOTE: also configuring project dir: %s" % [dirs_to_mount[-1]]
+  end
+
+  git_file = Pathname.new(tvm_home + "/.git")
+  if git_file.ftype() == "file" then
+    gitdir_match = Regexp.new('^gitdir: (?<gitdir>.*/.git).*\n$', Regexp::MULTILINE).match(git_file.read())
+    if !gitdir_match.nil? then
+      dirs_to_mount.append(Pathname.new(tvm_home).realpath.join(gitdir_match.named_captures["gitdir"]))
+      puts "NOTE: also configuring git-worktree gitdir: %s" % [dirs_to_mount[-1]]
+    end
+  end
+
+  config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false
+
+  # Enable USB Controller on VirtualBox
+  vm_name = "microtvm-arduino-#{Time.now.tv_sec}"
+  config.vm.provider "virtualbox" do |vb, overrides|
+    vb.name = vm_name
+    vb.cpus = num_cores
+    vb.memory = ram_bytes
+    vb.customize ["modifyvm", :id, "--usb", "on"]
+    vb.customize ["modifyvm", :id, "--usbehci", "on"]
+    vb.customize ["modifyvm", :id, "--usbxhci", "on"]
+    vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000]
+    dirs_to_mount.each do |d|
+      overrides.vm.synced_folder d.to_s, d.to_s
+    end
+  end
+
+end

apps/microtvm/reference-vm/arduino/base-box/.gitignore

@@ -0,0 +1,4 @@
+*.box
+.vagrant
+/output-packer-*
+/packer.json

apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+Vagrant.configure("2") do |config|
+  # From hashicorp default template:
+  # https://github.com/hashicorp/packer/blob/master/builder/vagrant/step_create_vagrantfile.go#L23-L37
+
+  config.vm.define "source" do |source|
+    source.vm.box = "{{.SourceBox}}"
+    config.ssh.insert_key = {{.InsertKey}}
+  end
+
+  config.vm.define "output" do |output|
+    output.vm.box = "{{.BoxName}}"
+    output.vm.box_url = "file://package.box"
+    config.ssh.insert_key = {{.InsertKey}}
+  end
+
+  {{ if ne .SyncedFolder "" -}}
+    config.vm.synced_folder "{{.SyncedFolder}}", "/vagrant"
+  {{- else -}}
+    config.vm.synced_folder ".", "/vagrant", disabled: true
+  {{- end}}
+
+
+  {{ if eq .BoxName "microtvm-base-vmware_desktop" -}}
+  config.vm.provision "shell", inline: "touch ~/skip_zeroing_disk", privileged: false
+  {{- end}}
+
+  # NOTE: base_box_setup.sh resides in the parent directory (../) because this template is expanded into a
+  # sub-directory of base-box (output-packer-*).
+  config.vm.provision "shell", path: "../base_box_setup.sh", privileged: false
+end

apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh

@@ -0,0 +1,77 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#   Using this script we can reuse docker/install scripts to configure the reference 
+#   virtual machine similar to CI QEMU setup.
+#
+
+set -e
+set -x
+
+source ~/.profile
+
+# Init Arduino
+cd ~
+
+sudo apt-get install -y ca-certificates
+
+# Install Arduino-CLI (latest version)
+export PATH="/home/vagrant/bin:$PATH"
+wget -O - https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | sh -s
+
+# Arduino (the CLI and GUI) require the dialout permission for uploading
+sudo usermod -a -G dialout $USER
+
+# ubuntu_init_arduino.sh only installs a few officially
+# supported architectures, so we don't use it here
+
+# 3rd party board URLs
+ADAFRUIT_BOARDS_URL="https://adafruit.github.io/arduino-board-index/package_adafruit_index.json"
+ESP32_BOARDS_URL="https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json"
+SPARKFUN_BOARDS_URL="https://raw.githubusercontent.com/sparkfun/Arduino_Boards/master/IDE_Board_Manager/package_sparkfun_index.json"
+SEEED_BOARDS_URL="https://files.seeedstudio.com/arduino/package_seeeduino_boards_index.json"
+SPRESENSE_BOARDS_URL="https://github.com/sonydevworld/spresense-arduino-compatible/releases/download/generic/package_spresense_index.json"
+arduino-cli core update-index --additional-urls $ADAFRUIT_BOARDS_URL,$ESP32_BOARDS_URL,$SPARKFUN_BOARDS_URL,$SEEED_BOARDS_URL,$SPRESENSE_BOARDS_URL
+
+# Install supported cores from those URLS
+arduino-cli core install arduino:mbed_nano
+arduino-cli core install arduino:sam
+arduino-cli core install adafruit:samd --additional-urls $ADAFRUIT_BOARDS_URL
+arduino-cli core install esp32:esp32 --additional-urls $ESP32_BOARDS_URL
+arduino-cli core install Seeeduino:samd --additional-urls $SEEED_BOARDS_URL
+arduino-cli core install SPRESENSE:spresense --additional-urls $SPRESENSE_BOARDS_URL
+
+# The Sony Spresense SDK has a major bug that breaks TVM. It's scheduled to be fixed in
+# release 2.3.0, but until that's published we need to use the below hack. This ONLY
+# fixes the bug in the main core release SDK - the subcore release SDK and both
+# the main and subcore debug SDKs will continue to fail until an official fix is made.
+# https://github.com/sonydevworld/spresense/issues/200
+SPRESENSE_NUTTX_BUGFIX_PATH=~/.arduino15/packages/SPRESENSE/tools/spresense-sdk/2.2.1/spresense/release/nuttx/include/sys/types.h
+sed -i 's/#ifndef CONFIG_WCHAR_BUILTIN/#if !defined(__cplusplus)/g' $SPRESENSE_NUTTX_BUGFIX_PATH
+
+# There's also a bug in arduino-cli where {runtime.os} is not properly templated in
+# platform.txt. This bug only seems to appear with the SPRESENSE SDK. A fix has been
+# merged and will be part of arduino-cli 0.18.4, but that has yet to be published.
+# This change is only needed to upload code (not compile) for the Spresense.
+# https://github.com/arduino/arduino-cli/issues/1198
+SPRESENSE_FLASH_WRITER_BUGFIX_PATH=~/.arduino15/packages/SPRESENSE/hardware/spresense/2.2.1/platform.txt
+sed -i 's/tools.spresense-tools.cmd.path={path}\/flash_writer\/{runtime.os}\/flash_writer/tools.spresense-tools.cmd.path={path}\/flash_writer\/linux\/flash_writer/g' $SPRESENSE_FLASH_WRITER_BUGFIX_PATH
+sed -i 's/tools.spresense-tools.cmd.path.linux={path}\/flash_writer\/{runtime.os}\/flash_writer/tools.spresense-tools.cmd.path.linux={path}\/flash_writer\/linux\/flash_writer/g' $SPRESENSE_FLASH_WRITER_BUGFIX_PATH
+
+# Cleanup
+rm -f *.sh

apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh

@@ -0,0 +1,75 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -x
+
+skip_zeroing_disk=0
+if [ -e "$HOME/skip_zeroing_disk" ]; then
+    echo "NOTE: will not zero disk at the end due to VMWare Fusion bug"
+    echo "See: https://communities.vmware.com/t5/VMware-Fusion-Discussions/VMWare-Fusion-Pro-11-15-6-16696540-causes-macOS-crash-during/m-p/2284011#M139190"
+    skip_zeroing_disk=1
+fi
+
+sudo apt update
+sudo apt install -y build-essential
+sudo apt-get --purge remove modemmanager  # required to access serial ports.
+
+sudo apt install -y --no-install-recommends git \
+     cmake cmake-data \
+     ninja-build gperf ccache dfu-util device-tree-compiler wget \
+     python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \
+     make gcc gcc-multilib g++-multilib libsdl2-dev
+
+OLD_HOSTNAME=$(hostname)
+sudo hostnamectl set-hostname microtvm
+sudo sed -i.bak "s/${OLD_HOSTNAME}/microtvm.localdomain/g" /etc/hosts
+
+# Poetry deps
+sudo apt install -y python3-venv
+
+# TVM deps
+sudo apt install -y llvm
+
+# ONNX deps
+sudo apt install -y protobuf-compiler libprotoc-dev
+
+# TODO do we need this?
+echo 'export PATH=$HOME/vagrant/bin:"$PATH"' >> ~/.profile
+source ~/.profile
+echo PATH=$PATH
+
+# Poetry
+curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3
+sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc
+sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc
+
+# Clean box for packaging as a base box
+sudo apt-get clean
+if [ $skip_zeroing_disk -eq 0 ]; then
+    echo "Zeroing disk..."
+    EMPTY_FILE="$HOME/EMPTY"
+    dd if=/dev/zero "of=${EMPTY_FILE}" bs=1M || /bin/true
+    if [ ! -e "${EMPTY_FILE}" ]; then
+        echo "failed to zero empty sectors on disk"
+        exit 2
+    fi
+    rm -f "${EMPTY_FILE}"
+else
+    echo "NOTE: skipping zeroing disk due to command-line argument."
+fi

apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Usage: base_box_test.sh <MICROTVM_PLATFORM>
+#     Execute microTVM Arduino tests.
+#
+
+set -e
+set -x
+
+if [ "$#" -lt 1 ]; then
+    echo "Usage: base_box_test.sh <MICROTVM_PLATFORM>"
+    exit -1
+fi
+
+microtvm_platform=$1
+
+pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-platforms=${microtvm_platform}
+
+if [ $microtvm_platform == "nano33ble" ]; then
+    # https://github.com/apache/tvm/issues/8730
+    echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_platform -- known failure"
+else
+    pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-platforms=${microtvm_platform}
+fi