From 544724439efb9a795c92bd7ec9f7929e41c843c6 Mon Sep 17 00:00:00 2001 From: Gavin Uberti Date: Wed, 1 Sep 2021 09:51:49 -0400 Subject: [PATCH] [microTVM][RVM] Add Arduino RVM (#8748) * Functioning Arduino Vagrant VM Begin building Arduino Vagrant VM Mostly working Vagrant VM Changes for debugging Add ignored json file Fix venv path * Generalize parts of RVM for multiple platforms cwd hack Add unit tests from apps directory to task_python_microtvm.sh Generalize parts of RVM for multiple platforms * Add Vagrantfile lint exceptions * Address PR comments Address Mehrdad's PR comments More PR comments Documentation tweaks Add dialout group to user * Rerun tests * Spresense fix * Rerun CI tests * Rerun tests --- .../template_project/microtvm_api_server.py | 5 + apps/microtvm/reference-vm/arduino/.gitignore | 1 + apps/microtvm/reference-vm/arduino/README.md | 44 +++++++ .../microtvm/reference-vm/arduino/Vagrantfile | 66 +++++++++++ .../reference-vm/arduino/base-box/.gitignore | 4 + .../base-box/Vagrantfile.packer-template | 47 ++++++++ .../arduino/base-box/base_box_provision.sh | 77 +++++++++++++ .../arduino/base-box/base_box_setup.sh | 75 ++++++++++++ .../arduino/base-box/base_box_test.sh | 40 +++++++ .../arduino/base-box/test-config.json | 30 +++++ .../reference-vm/arduino/provision_setup.sh | 48 ++++++++ apps/microtvm/reference-vm/base-box-tool.py | 109 ++++++++++++------ apps/microtvm/reference-vm/rebuild-tvm.sh | 43 +++++++ .../reference-vm/zephyr/provision_setup.sh | 2 +- docker/install/ubuntu_install_arduino.sh | 2 +- tests/lint/check_file_type.py | 2 + tests/micro/arduino/conftest.py | 1 + 17 files changed, 561 insertions(+), 35 deletions(-) create mode 100644 apps/microtvm/reference-vm/arduino/.gitignore create mode 100644 apps/microtvm/reference-vm/arduino/README.md create mode 100644 apps/microtvm/reference-vm/arduino/Vagrantfile create mode 100644 apps/microtvm/reference-vm/arduino/base-box/.gitignore create mode 100644 apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template create mode 100644 apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh create mode 100644 apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh create mode 100755 apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh create mode 100644 apps/microtvm/reference-vm/arduino/base-box/test-config.json create mode 100644 apps/microtvm/reference-vm/arduino/provision_setup.sh create mode 100755 apps/microtvm/reference-vm/rebuild-tvm.sh diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index 91beaf558249..57177179bcd0 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -65,6 +65,11 @@ class BoardAutodetectFailed(Exception): "architecture": "esp32", "board": "feathers2", }, + "metrom4": { + "package": "adafruit", + "architecture": "samd", + "board": "adafruit_metro_m4", + }, # Spresense only works as of its v2.3.0 sdk "spresense": { "package": "SPRESENSE", diff --git a/apps/microtvm/reference-vm/arduino/.gitignore b/apps/microtvm/reference-vm/arduino/.gitignore new file mode 100644 index 000000000000..dace7081e3f2 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/.gitignore @@ -0,0 +1 @@ +/.vagrant diff --git a/apps/microtvm/reference-vm/arduino/README.md b/apps/microtvm/reference-vm/arduino/README.md new file mode 100644 index 000000000000..3fa1d8bfb4e3 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/README.md @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + +# microTVM Arduino Reference Virtual Machine + +This directory contains setup files for Arduino virtual machine used for testing +microTVM platforms that are supported by [Arduino](https://www.arduino.cc/). + +## VM Information for Developers +Arduino VM is published under [tlcpack](https://app.vagrantup.com/tlcpack). +Here is a list of different release versions and their tools. + +(none currently) + +## Supported Arduino Boards +This RVM has been tested and is known to work with these boards: +- Adafruit Metro M4 +- Adafruit Pybadge +- Arduino Due +- Arduino Nano 33 BLE +- Feather S2 +- Sony Spresense +- Wio Terminal + +However, the RVM *should* work with any Arduino with sufficient memory, provided +its core is installed in `base-box/base_box_provision.sh`. + +Note that this RVM does not work with the Teensy boards, even though they are +supported by microTVM. This is because arduino-cli does not support Teensy +boards (https://github.com/arduino/arduino-cli/issues/700)/). diff --git a/apps/microtvm/reference-vm/arduino/Vagrantfile b/apps/microtvm/reference-vm/arduino/Vagrantfile new file mode 100644 index 000000000000..2511a6ae296e --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/Vagrantfile @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +Vagrant.configure("2") do |config| + config.vm.box = "tlcpack/microtvm-arduino-0.18.3" + + if ENV.has_key?("TVM_RVM_NUM_CORES") + num_cores = ENV["TVM_RVM_NUM_CORES"] + else + num_cores = 2 + end + + if ENV.has_key?("TVM_RVM_RAM_BYTES") + ram_bytes = ENV["TVM_RVM_RAM_BYTES"] + else + ram_bytes = 2048 + end + + tvm_home = "../../../.." + dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())] + if ENV.has_key?("TVM_PROJECT_DIR") then + dirs_to_mount.append(ENV["TVM_PROJECT_DIR"]) + puts "NOTE: also configuring project dir: %s" % [dirs_to_mount[-1]] + end + + git_file = Pathname.new(tvm_home + "/.git") + if git_file.ftype() == "file" then + gitdir_match = Regexp.new('^gitdir: (?.*/.git).*\n$', Regexp::MULTILINE).match(git_file.read()) + if !gitdir_match.nil? then + dirs_to_mount.append(Pathname.new(tvm_home).realpath.join(gitdir_match.named_captures["gitdir"])) + puts "NOTE: also configuring git-worktree gitdir: %s" % [dirs_to_mount[-1]] + end + end + + config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false + + # Enable USB Controller on VirtualBox + vm_name = "microtvm-arduino-#{Time.now.tv_sec}" + config.vm.provider "virtualbox" do |vb, overrides| + vb.name = vm_name + vb.cpus = num_cores + vb.memory = ram_bytes + vb.customize ["modifyvm", :id, "--usb", "on"] + vb.customize ["modifyvm", :id, "--usbehci", "on"] + vb.customize ["modifyvm", :id, "--usbxhci", "on"] + vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000] + dirs_to_mount.each do |d| + overrides.vm.synced_folder d.to_s, d.to_s + end + end + +end diff --git a/apps/microtvm/reference-vm/arduino/base-box/.gitignore b/apps/microtvm/reference-vm/arduino/base-box/.gitignore new file mode 100644 index 000000000000..e4406c4f61e2 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/.gitignore @@ -0,0 +1,4 @@ +*.box +.vagrant +/output-packer-* +/packer.json diff --git a/apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template b/apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template new file mode 100644 index 000000000000..b43596bb83c1 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +Vagrant.configure("2") do |config| + # From hashicorp default template: + # https://github.com/hashicorp/packer/blob/master/builder/vagrant/step_create_vagrantfile.go#L23-L37 + + config.vm.define "source" do |source| + source.vm.box = "{{.SourceBox}}" + config.ssh.insert_key = {{.InsertKey}} + end + + config.vm.define "output" do |output| + output.vm.box = "{{.BoxName}}" + output.vm.box_url = "file://package.box" + config.ssh.insert_key = {{.InsertKey}} + end + + {{ if ne .SyncedFolder "" -}} + config.vm.synced_folder "{{.SyncedFolder}}", "/vagrant" + {{- else -}} + config.vm.synced_folder ".", "/vagrant", disabled: true + {{- end}} + + + {{ if eq .BoxName "microtvm-base-vmware_desktop" -}} + config.vm.provision "shell", inline: "touch ~/skip_zeroing_disk", privileged: false + {{- end}} + + # NOTE: base_box_setup.sh resides in the parent directory (../) because this template is expanded into a + # sub-directory of base-box (output-packer-*). + config.vm.provision "shell", path: "../base_box_setup.sh", privileged: false +end diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh new file mode 100644 index 000000000000..996d303d48fb --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh @@ -0,0 +1,77 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Using this script we can reuse docker/install scripts to configure the reference +# virtual machine similar to CI QEMU setup. +# + +set -e +set -x + +source ~/.profile + +# Init Arduino +cd ~ + +sudo apt-get install -y ca-certificates + +# Install Arduino-CLI (latest version) +export PATH="/home/vagrant/bin:$PATH" +wget -O - https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | sh -s + +# Arduino (the CLI and GUI) require the dialout permission for uploading +sudo usermod -a -G dialout $USER + +# ubuntu_init_arduino.sh only installs a few officially +# supported architectures, so we don't use it here + +# 3rd party board URLs +ADAFRUIT_BOARDS_URL="https://adafruit.github.io/arduino-board-index/package_adafruit_index.json" +ESP32_BOARDS_URL="https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json" +SPARKFUN_BOARDS_URL="https://raw.githubusercontent.com/sparkfun/Arduino_Boards/master/IDE_Board_Manager/package_sparkfun_index.json" +SEEED_BOARDS_URL="https://files.seeedstudio.com/arduino/package_seeeduino_boards_index.json" +SPRESENSE_BOARDS_URL="https://github.com/sonydevworld/spresense-arduino-compatible/releases/download/generic/package_spresense_index.json" +arduino-cli core update-index --additional-urls $ADAFRUIT_BOARDS_URL,$ESP32_BOARDS_URL,$SPARKFUN_BOARDS_URL,$SEEED_BOARDS_URL,$SPRESENSE_BOARDS_URL + +# Install supported cores from those URLS +arduino-cli core install arduino:mbed_nano +arduino-cli core install arduino:sam +arduino-cli core install adafruit:samd --additional-urls $ADAFRUIT_BOARDS_URL +arduino-cli core install esp32:esp32 --additional-urls $ESP32_BOARDS_URL +arduino-cli core install Seeeduino:samd --additional-urls $SEEED_BOARDS_URL +arduino-cli core install SPRESENSE:spresense --additional-urls $SPRESENSE_BOARDS_URL + +# The Sony Spresense SDK has a major bug that breaks TVM. It's scheduled to be fixed in +# release 2.3.0, but until that's published we need to use the below hack. This ONLY +# fixes the bug in the main core release SDK - the subcore release SDK and both +# the main and subcore debug SDKs will continue to fail until an official fix is made. +# https://github.com/sonydevworld/spresense/issues/200 +SPRESENSE_NUTTX_BUGFIX_PATH=~/.arduino15/packages/SPRESENSE/tools/spresense-sdk/2.2.1/spresense/release/nuttx/include/sys/types.h +sed -i 's/#ifndef CONFIG_WCHAR_BUILTIN/#if !defined(__cplusplus)/g' $SPRESENSE_NUTTX_BUGFIX_PATH + +# There's also a bug in arduino-cli where {runtime.os} is not properly templated in +# platform.txt. This bug only seems to appear with the SPRESENSE SDK. A fix has been +# merged and will be part of arduino-cli 0.18.4, but that has yet to be published. +# This change is only needed to upload code (not compile) for the Spresense. +# https://github.com/arduino/arduino-cli/issues/1198 +SPRESENSE_FLASH_WRITER_BUGFIX_PATH=~/.arduino15/packages/SPRESENSE/hardware/spresense/2.2.1/platform.txt +sed -i 's/tools.spresense-tools.cmd.path={path}\/flash_writer\/{runtime.os}\/flash_writer/tools.spresense-tools.cmd.path={path}\/flash_writer\/linux\/flash_writer/g' $SPRESENSE_FLASH_WRITER_BUGFIX_PATH +sed -i 's/tools.spresense-tools.cmd.path.linux={path}\/flash_writer\/{runtime.os}\/flash_writer/tools.spresense-tools.cmd.path.linux={path}\/flash_writer\/linux\/flash_writer/g' $SPRESENSE_FLASH_WRITER_BUGFIX_PATH + +# Cleanup +rm -f *.sh diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh new file mode 100644 index 000000000000..d02518c538b4 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +skip_zeroing_disk=0 +if [ -e "$HOME/skip_zeroing_disk" ]; then + echo "NOTE: will not zero disk at the end due to VMWare Fusion bug" + echo "See: https://communities.vmware.com/t5/VMware-Fusion-Discussions/VMWare-Fusion-Pro-11-15-6-16696540-causes-macOS-crash-during/m-p/2284011#M139190" + skip_zeroing_disk=1 +fi + +sudo apt update +sudo apt install -y build-essential +sudo apt-get --purge remove modemmanager # required to access serial ports. + +sudo apt install -y --no-install-recommends git \ + cmake cmake-data \ + ninja-build gperf ccache dfu-util device-tree-compiler wget \ + python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ + make gcc gcc-multilib g++-multilib libsdl2-dev + +OLD_HOSTNAME=$(hostname) +sudo hostnamectl set-hostname microtvm +sudo sed -i.bak "s/${OLD_HOSTNAME}/microtvm.localdomain/g" /etc/hosts + +# Poetry deps +sudo apt install -y python3-venv + +# TVM deps +sudo apt install -y llvm + +# ONNX deps +sudo apt install -y protobuf-compiler libprotoc-dev + +# TODO do we need this? +echo 'export PATH=$HOME/vagrant/bin:"$PATH"' >> ~/.profile +source ~/.profile +echo PATH=$PATH + +# Poetry +curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 +sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc +sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc + +# Clean box for packaging as a base box +sudo apt-get clean +if [ $skip_zeroing_disk -eq 0 ]; then + echo "Zeroing disk..." + EMPTY_FILE="$HOME/EMPTY" + dd if=/dev/zero "of=${EMPTY_FILE}" bs=1M || /bin/true + if [ ! -e "${EMPTY_FILE}" ]; then + echo "failed to zero empty sectors on disk" + exit 2 + fi + rm -f "${EMPTY_FILE}" +else + echo "NOTE: skipping zeroing disk due to command-line argument." +fi diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh new file mode 100755 index 000000000000..3d8597f19b64 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Usage: base_box_test.sh +# Execute microTVM Arduino tests. +# + +set -e +set -x + +if [ "$#" -lt 1 ]; then + echo "Usage: base_box_test.sh " + exit -1 +fi + +microtvm_platform=$1 + +pytest tests/micro/arduino/test_arduino_workflow.py --microtvm-platforms=${microtvm_platform} + +if [ $microtvm_platform == "nano33ble" ]; then + # https://github.com/apache/tvm/issues/8730 + echo "NOTE: skipped test_arduino_rpc_server.py on $microtvm_platform -- known failure" +else + pytest tests/micro/arduino/test_arduino_rpc_server.py --microtvm-platforms=${microtvm_platform} +fi diff --git a/apps/microtvm/reference-vm/arduino/base-box/test-config.json b/apps/microtvm/reference-vm/arduino/base-box/test-config.json new file mode 100644 index 000000000000..80cc17f56847 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/base-box/test-config.json @@ -0,0 +1,30 @@ +{ + "due": { + "vid_hex": "2341", + "pid_hex": "003d" + }, + "feathers2": { + "vid_hex": "303a", + "pid_hex": "0002" + }, + "nano33ble": { + "vid_hex": "2341", + "pid_hex": "805a" + }, + "spresense": { + "vid_hex": "10c4", + "pid_hex": "ea60" + }, + "teensy40": { + "vid_hex": "16c0", + "pid_hex": "0478" + }, + "teensy41": { + "vid_hex": "16c0", + "pid_hex": "0478" + }, + "wioterminal": { + "vid_hex": "2886", + "pid_hex": "802d" + } +} diff --git a/apps/microtvm/reference-vm/arduino/provision_setup.sh b/apps/microtvm/reference-vm/arduino/provision_setup.sh new file mode 100644 index 000000000000..aeb46a8f7649 --- /dev/null +++ b/apps/microtvm/reference-vm/arduino/provision_setup.sh @@ -0,0 +1,48 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +# TVM +# NOTE: TVM is presumed to be mounted already by Vagrantfile. +cd "${TVM_HOME}" + +apps/microtvm/reference-vm/rebuild-tvm.sh + +# Build poetry +cd apps/microtvm/reference-vm/arduino + +poetry env use 3.6 +# NOTE: due to https://github.com/python-poetry/poetry/issues/2247, download torch here. +poetry run pip3 install torch==1.4.0 torchvision==0.5.0 + +# importers +poetry install -E importer-onnx +poetry install -E importer-tflite + +echo "------------------------------[ TVM Message ]------------------------------" +echo "WARNING: running 'poetry lock', which could take several minutes (depending" +echo "on your network connection and the state of PyPI) as dependencies are" +echo "downloaded and cached for future use." +echo "------------------------------[ TVM Message ]------------------------------" +poetry lock -vvv +poetry install + +echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm" >>~/.profile +echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/arduino && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile +echo "source \$VENV_PATH/bin/activate" >>~/.profile diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index be9c5173de73..f32885433c2b 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -41,14 +41,43 @@ "vmware_desktop", ) -# List of microTVM platforms for testing. -ALL_MICROTVM_PLATFORMS = ( - "stm32f746xx_nucleo", - "stm32f746xx_disco", - "nrf5340dk", - "mps2_an521", +# List of supported electronics platforms. Each must correspond +# to a sub-directory of this directory. +ALL_PLATFORMS = ( + "arduino", + "zephyr", ) +# List of identifying strings for microTVM platforms for testing. +# Must match PLATFORMS as defined in tvm/tests/micro/[platform]/conftest.py +# TODO add a way to declare supported platforms to ProjectAPI +ALL_MICROTVM_PLATFORMS = { + "arduino": ( + "due", + "feathers2", + "metrom4", + "nano33ble", + "pybadge", + "spresense", + "teensy40", + "teensy41", + "wioterminal", + ), + "zephyr": ( + "stm32f746xx_nucleo", + "stm32f746xx_disco", + "nrf5340dk", + "mps2_an521", + ), +} + +# Extra scripts required to execute on provisioning +# in [platform]/base-box/base_box_provision.sh +EXTRA_SCRIPTS = { + "arduino": (), + "zephyr": ("docker/install/ubuntu_init_zephyr_project.sh",), +} + PACKER_FILE_NAME = "packer.json" @@ -176,12 +205,8 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None): "vmware_desktop": attach_vmware, } -# Extra scripts required to execute on provisioning -# in zephyr/base-box/base_box_provision.sh -EXTRA_SCRIPTS = ("docker/install/ubuntu_init_zephyr_project.sh",) - -def generate_packer_config(file_path, providers): +def generate_packer_config(platform, file_path, providers): builders = [] provisioners = [] for provider_name in providers: @@ -199,9 +224,9 @@ def generate_packer_config(file_path, providers): ) repo_root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], cwd=os.path.dirname(__file__), encoding="utf-8" + ["git", "rev-parse", "--show-toplevel"], encoding="utf-8" ).strip() - for script in EXTRA_SCRIPTS: + for script in EXTRA_SCRIPTS[platform]: script_path = os.path.join(repo_root, script) filename = os.path.basename(script_path) provisioners.append({"type": "file", "source": script_path, "destination": f"~/{filename}"}) @@ -227,6 +252,7 @@ def generate_packer_config(file_path, providers): def build_command(args): generate_packer_config( + args.platform, os.path.join(THIS_DIR, args.platform, "base-box", PACKER_FILE_NAME), args.provider or ALL_PROVIDERS, ) @@ -311,7 +337,7 @@ def do_build_release_test_vm(release_test_dir, user_box_dir, base_box_dir, provi return True -def do_run_release_test(release_test_dir, provider_name, test_config, test_device_serial): +def do_run_release_test(release_test_dir, platform, provider_name, test_config, test_device_serial): with open( os.path.join(release_test_dir, ".vagrant", "machines", "default", provider_name, "id") ) as f: @@ -335,7 +361,7 @@ def _quote_cmd(cmd): + " && " + _quote_cmd( [ - "apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh", + f"apps/microtvm/reference-vm/{platform}/base-box/base_box_test.sh", test_config["microtvm_platform"], ] ) @@ -377,7 +403,11 @@ def test_command(args): release_test_dir, user_box_dir, base_box_dir, provider_name ) do_run_release_test( - release_test_dir, provider_name, microtvm_test_platform, args.test_device_serial + release_test_dir, + args.platform, + provider_name, + microtvm_test_platform, + args.test_device_serial, ) provider_passed[provider_name] = True @@ -439,26 +469,27 @@ def parse_args(): "--provider", choices=ALL_PROVIDERS, action="append", - help="Name of the provider or providers to act on; if not specified, act on all.", + required=True, + help="Name of the provider or providers to act on", ) - parser.add_argument( - "platform", - help="Name of the platform VM to act on. Must be a sub-directory of this directory.", - ) + # "test" has special options for different platforms, and "build", "release" might + # in the future, so we'll add the platform argument to each one individually. + platform_help_str = "Platform to use (e.g. Arduino, Zephyr)" + # Options for build subcommand parser_build = subparsers.add_parser("build", help="Build a base box.") parser_build.set_defaults(func=build_command) - parser_test = subparsers.add_parser("test", help="Test a base box before release.") - parser_test.set_defaults(func=test_command) - parser_release = subparsers.add_parser("release", help="Release base box to cloud.") - parser_release.set_defaults(func=release_command) - + parser_build.add_argument("platform", help=platform_help_str, choices=ALL_PLATFORMS) parser_build.add_argument( "--debug-packer", action="store_true", help=("Run packer in debug mode, and write log to the base-box directory."), ) + + # Options for test subcommand + parser_test = subparsers.add_parser("test", help="Test a base box before release.") + parser_test.set_defaults(func=test_command) parser_test.add_argument( "--skip-build", action="store_true", @@ -475,12 +506,21 @@ def parse_args(): "iSerial field from `lsusb -v` output." ), ) - parser_test.add_argument( - "--microtvm-platform", - choices=ALL_MICROTVM_PLATFORMS, - required=True, - help="MicroTVM platfrom used for testing.", - ) + parser_test_platform_subparsers = parser_test.add_subparsers(help=platform_help_str) + for platform in ALL_PLATFORMS: + platform_specific_parser = parser_test_platform_subparsers.add_parser(platform) + platform_specific_parser.set_defaults(platform=platform) + platform_specific_parser.add_argument( + "--microtvm-platform", + choices=ALL_MICROTVM_PLATFORMS[platform], + required=True, + help="MicroTVM platfrom used for testing.", + ) + + # Options for release subcommand + parser_release = subparsers.add_parser("release", help="Release base box to cloud.") + parser_release.set_defaults(func=release_command) + parser_release.add_argument("platform", help=platform_help_str, choices=ALL_PLATFORMS) parser_release.add_argument( "--release-version", required=True, @@ -494,7 +534,10 @@ def parse_args(): parser_release.add_argument( "--platform-version", required=True, - help="Platform version to release, in the form 'x.y'.", + help=( + "For Zephyr, the platform version to release, in the form 'x.y'. " + "For Arduino, the version of arduino-cli that's being used, in the form 'x.y.z'." + ), ) return parser.parse_args() diff --git a/apps/microtvm/reference-vm/rebuild-tvm.sh b/apps/microtvm/reference-vm/rebuild-tvm.sh new file mode 100755 index 000000000000..1cebcf7166af --- /dev/null +++ b/apps/microtvm/reference-vm/rebuild-tvm.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# Get number of cores for build +if [ -n "${TVM_CI_NUM_CORES}" ]; then + num_cores=${TVM_CI_NUM_CORES} +else + # default setup for Vagrantfile + num_cores=2 +fi + +cd "$(dirname $0)" +cd "$(git rev-parse --show-toplevel)" +BUILD_DIR=build-microtvm + +if [ ! -e "${BUILD_DIR}" ]; then + mkdir "${BUILD_DIR}" +fi +cp cmake/config.cmake "${BUILD_DIR}" +cd "${BUILD_DIR}" +sed -i 's/USE_MICRO OFF/USE_MICRO ON/' config.cmake +sed -i 's/USE_GRAPH_EXECUTOR_DEBUG OFF/USE_GRAPH_EXECUTOR_DEBUG ON/' config.cmake +sed -i 's/USE_LLVM OFF/USE_LLVM ON/' config.cmake +cmake .. +rm -rf standalone_crt host_standalone_crt # remove stale generated files +make -j${num_cores} diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/zephyr/provision_setup.sh index fcefc1176821..2ee2350b377a 100644 --- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh +++ b/apps/microtvm/reference-vm/zephyr/provision_setup.sh @@ -22,7 +22,7 @@ set -ex # NOTE: TVM is presumed to be mounted already by Vagrantfile. cd "${TVM_HOME}" -apps/microtvm/reference-vm/zephyr/rebuild-tvm.sh +apps/microtvm/reference-vm/rebuild-tvm.sh # Build poetry cd apps/microtvm/reference-vm/zephyr diff --git a/docker/install/ubuntu_install_arduino.sh b/docker/install/ubuntu_install_arduino.sh index d5c4303f211b..c374850aa1df 100644 --- a/docker/install/ubuntu_install_arduino.sh +++ b/docker/install/ubuntu_install_arduino.sh @@ -26,7 +26,7 @@ apt-get install -y ca-certificates # Install arduino-cli latest version wget -O - https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh | sh -s -# Install supported cores from those URLS +# Install the cores we want to test on arduino-cli core install arduino:mbed_nano arduino-cli core install arduino:sam diff --git a/tests/lint/check_file_type.py b/tests/lint/check_file_type.py index ed7288ef00d4..0677292371ae 100644 --- a/tests/lint/check_file_type.py +++ b/tests/lint/check_file_type.py @@ -145,6 +145,8 @@ "apps/microtvm/zephyr/template_project/qemu-hack/qemu-system-riscv32", "apps/microtvm/zephyr/template_project/qemu-hack/qemu-system-riscv64", # microTVM Virtual Machines + "apps/microtvm/reference-vm/arduino/Vagrantfile", + "apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template", "apps/microtvm/reference-vm/zephyr/Vagrantfile", "apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template", } diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index bcb2bddf2cab..aea1381a43f8 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -26,6 +26,7 @@ PLATFORMS = { "due": ("sam3x8e", "due"), "feathers2": ("esp32", "feathers2"), + "metrom4": ("atsamd51", "metrom4"), "nano33ble": ("nrf52840", "nano33ble"), "pybadge": ("atsamd51", "pybadge"), "spresense": ("cxd5602gg", "spresense"),