[RFC] [VTA] [TSIM] Enabling Cycle-Accurate Hardware Simulation for VTA …

…#3009 (#3010)

* merge files

* move verilator to the right place

* change name to tsim

* add default rule to be build and run

* add README for tsim

* Update README.md

* add some structural feedback

* change name of VTASim to VTADPISim

* more renaming

* update comment

* add license

* fix indentation

* add switch for vta-tsim

* add more licenses

* update readme

* address some of the new feedback

* add some feedback from cpplint

* add one more whitespace

* pass pointer so linter is happy

* pass pointer so linter is happy

* README moved to vta documentation

* create types for dpi functions, so they can be handle easily

* fix pointer style

* add feedback from docs

* parametrize width data and pointers

* fix comments

* fix comment

* add comment to class

* add missing parameters

* move README back to tsim example

* add feedback

* add more comments and remove un-necessary argument in finish

* update comments

* fix cpplint

* fix doc

cmake/config.cmake

@@ -131,3 +131,6 @@ set(USE_SORT ON)
 
 # Build ANTLR parser for Relay text format
 set(USE_ANTLR OFF)
+
+# Build TSIM for VTA
+set(USE_VTA_TSIM OFF)

cmake/modules/VTA.cmake

@@ -60,6 +60,13 @@ elseif(PYTHON)
     find_library(__cma_lib NAMES cma PATH /usr/lib)
     target_link_libraries(vta ${__cma_lib})
   endif()
+
+  if(NOT USE_VTA_TSIM STREQUAL "OFF")
+    include_directories("vta/include")
+    file(GLOB RUNTIME_DPI_SRCS vta/src/dpi/module.cc)
+    list(APPEND RUNTIME_SRCS ${RUNTIME_DPI_SRCS})
+  endif()
+
 else()
   message(STATUS "Cannot found python in env, VTA build is skipped..")
 endif()

vta/apps/tsim_example/CMakeLists.txt

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.2)
+project(tsim C CXX)
+
+set(TVM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../)
+set(VTA_DIR ${TVM_DIR}/vta)
+
+include_directories("${TVM_DIR}/include")
+include_directories("${TVM_DIR}/3rdparty/dlpack/include")
+include_directories("${TVM_DIR}/3rdparty/dmlc-core/include")
+include_directories("${TVM_DIR}/vta/src/dpi")
+
+set(CMAKE_C_FLAGS "-O2 -Wall -fPIC -fvisibility=hidden")
+set(CMAKE_CXX_FLAGS "-O2 -Wall -fPIC -fvisibility=hidden -std=c++11")
+
+if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND
+    CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
+  set(CMAKE_CXX_FLAGS "-faligned-new ${CMAKE_CXX_FLAGS}")
+endif()
+
+# Module rules
+include(cmake/modules/tsim.cmake)
+include(cmake/modules/driver.cmake)

vta/apps/tsim_example/Makefile

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export PYTHONPATH:=$(PWD)/python:$(PYTHONPATH)
+
+BUILD_DIR = $(shell python python/tsim/config.py --get-build-name)
+
+TVM_DIR = $(abspath ../../../)
+
+TSIM_TARGET = verilog
+TSIM_TOP_NAME = TestAccel
+TSIM_BUILD_NAME = build
+
+# optional
+TSIM_TRACE_NAME = trace.vcd
+
+default: cmake run
+
+.PHONY: cmake
+
+cmake: | $(BUILD_DIR)
+	cd $(BUILD_DIR) && cmake .. && make
+
+$(BUILD_DIR):
+	mkdir -p $@
+
+run:
+	python3 tests/python/test_tsim.py | grep PASS
+
+clean:
+	-rm -rf $(BUILD_DIR)

vta/apps/tsim_example/README.md

@@ -0,0 +1,71 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+VTA TSIM Installation
+======================
+
+*TSIM* is a cycle-accurate hardware simulation environment that can be invoked and managed directly from TVM. It aims to enable cycle accurate simulation of deep learning accelerators including VTA.
+This simulation environment can be used in both OSX and Linux.
+There are two dependencies required to make *TSIM* works: [Verilator](https://www.veripool.org/wiki/verilator) and [sbt](https://www.scala-sbt.org/) for accelerators designed in [Chisel3](https://github.com/freechipsproject/chisel3).
+
+## OSX Dependencies
+
+Install `sbt` and `verilator` using [Homebrew](https://brew.sh/).
+
+```bash
+brew install verilator sbt
+```
+
+## Linux Dependencies
+
+Add `sbt` to package manager (Ubuntu).
+
+```bash
+echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+sudo apt-get update
+```
+
+Install `sbt` and `verilator`.
+
+```bash
+sudo apt install verilator sbt
+```
+
+## Setup in TVM
+
+1. Install `verilator` and `sbt` as described above
+2. Enable VTA TSIM by turning on the switch `USE_VTA_TSIM` in config.cmake
+3. Build tvm
+
+## How to run VTA TSIM examples
+
+There are two sample VTA accelerators (add-by-one) designed in Chisel3 and Verilog to show how *TSIM* works.
+These examples are located at `<tvm-root>/vta/apps/tsim_example`.
+
+* Instructions
+    * Open `<tvm-root>/vta/apps/tsim_example/python/tsim/config.json`
+    * Change `TARGET` from `verilog` to `chisel`, depending on what language backend you would like to test
+    * Go to `tvm/vta/apps/tsim`
+    * Run `make`
+
+* Some pointers
+    * Build cmake script for driver `<tvm-root>/vta/apps/tsim_example/cmake/modules/driver.cmake`
+    * Build cmake script for tsim `<tvm-root>/vta/apps/tsim_example/cmake/modules/tsim.cmake`
+    * Software driver that handles the VTA accelerator `<tvm-root>/vta/apps/tsim_example/src/driver.cc`
+    * VTA add-by-one accelerator (Verilog) `<tvm-root>/vta/apps/tsim_example/hardware/verilog`
+    * VTA add-by-one accelerator (Chisel) `<tvm-root>/vta/apps/tsim_example/hardware/chisel`

vta/apps/tsim_example/cmake/modules/driver.cmake

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+file(GLOB TSIM_SW_SRC src/driver.cc)
+add_library(driver SHARED ${TSIM_SW_SRC})
+target_include_directories(driver PRIVATE ${VTA_DIR}/include)
+
+if(APPLE)
+  set_target_properties(driver PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+endif(APPLE)

vta/apps/tsim_example/cmake/modules/tsim.cmake

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(MSVC)
+  message(STATUS "TSIM build is skipped in Windows..")
+else()
+  find_program(PYTHON NAMES python python3 python3.6)
+  find_program(VERILATOR NAMES verilator)
+
+  if (VERILATOR AND PYTHON)
+
+    if (TSIM_TOP_NAME STREQUAL "")
+      message(FATAL_ERROR "TSIM_TOP_NAME should be defined")
+    endif()
+
+    if (TSIM_BUILD_NAME STREQUAL "")
+      message(FATAL_ERROR "TSIM_BUILD_NAME should be defined")
+    endif()
+
+    set(TSIM_CONFIG ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/python/tsim/config.py)
+
+    execute_process(COMMAND ${TSIM_CONFIG} --get-target OUTPUT_VARIABLE __TSIM_TARGET)
+    execute_process(COMMAND ${TSIM_CONFIG} --get-top-name OUTPUT_VARIABLE __TSIM_TOP_NAME)
+    execute_process(COMMAND ${TSIM_CONFIG} --get-build-name OUTPUT_VARIABLE __TSIM_BUILD_NAME)
+    execute_process(COMMAND ${TSIM_CONFIG} --get-use-trace OUTPUT_VARIABLE __TSIM_USE_TRACE)
+    execute_process(COMMAND ${TSIM_CONFIG} --get-trace-name OUTPUT_VARIABLE __TSIM_TRACE_NAME)
+
+    string(STRIP ${__TSIM_TARGET} TSIM_TARGET)
+    string(STRIP ${__TSIM_TOP_NAME} TSIM_TOP_NAME)
+    string(STRIP ${__TSIM_BUILD_NAME} TSIM_BUILD_NAME)
+    string(STRIP ${__TSIM_USE_TRACE} TSIM_USE_TRACE)
+    string(STRIP ${__TSIM_TRACE_NAME} TSIM_TRACE_NAME)
+
+    set(TSIM_BUILD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${TSIM_BUILD_NAME})
+
+    if (TSIM_TARGET STREQUAL "chisel")
+
+      find_program(SBT NAMES sbt)
+
+      if (SBT)
+
+        # Install Chisel VTA package for DPI modules
+        set(VTA_CHISEL_DIR ${VTA_DIR}/hardware/chisel)
+
+        execute_process(WORKING_DIRECTORY ${VTA_CHISEL_DIR}
+          COMMAND ${SBT} publishLocal RESULT_VARIABLE RETCODE)
+
+        if (NOT RETCODE STREQUAL "0")
+          message(FATAL_ERROR "[TSIM] sbt failed to install VTA scala package")
+        endif()
+
+        # Chisel - Scala to Verilog compilation
+        set(TSIM_CHISEL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/hardware/chisel)
+        set(CHISEL_TARGET_DIR ${TSIM_BUILD_DIR}/chisel)
+        set(CHISEL_OPT "test:runMain test.Elaborate --target-dir ${CHISEL_TARGET_DIR} --top-name ${TSIM_TOP_NAME}")
+
+        execute_process(WORKING_DIRECTORY ${TSIM_CHISEL_DIR} COMMAND ${SBT} ${CHISEL_OPT} RESULT_VARIABLE RETCODE)
+
+        if (NOT RETCODE STREQUAL "0")
+          message(FATAL_ERROR "[TSIM] sbt failed to compile from Chisel to Verilog.")
+        endif()
+
+        file(GLOB VERILATOR_RTL_SRC ${CHISEL_TARGET_DIR}/*.v)
+
+      else()
+        message(FATAL_ERROR "[TSIM] sbt should be installed for Chisel")
+      endif() # sbt
+
+    elseif (TSIM_TARGET STREQUAL "verilog")
+
+      set(VTA_VERILOG_DIR ${VTA_DIR}/hardware/chisel/src/main/resources/verilog)
+      set(TSIM_VERILOG_DIR ${CMAKE_CURRENT_SOURCE_DIR}/hardware/verilog)
+      file(GLOB VERILATOR_RTL_SRC ${VTA_VERILOG_DIR}/*.v ${TSIM_VERILOG_DIR}/*.v)
+
+    else()
+      message(STATUS "[TSIM] target language can be only verilog or chisel...")
+    endif() # TSIM_TARGET
+
+    if (TSIM_TARGET STREQUAL "chisel" OR TSIM_TARGET STREQUAL "verilog")
+
+      # Check if tracing can be enabled
+      if (NOT TSIM_USE_TRACE STREQUAL "OFF")
+        message(STATUS "[TSIM] Verilog enable tracing")
+      else()
+        message(STATUS "[TSIM] Verilator disable tracing")
+      endif()
+
+      # Verilator - Verilog to C++ compilation
+      set(VERILATOR_TARGET_DIR ${TSIM_BUILD_DIR}/verilator)
+      set(VERILATOR_OPT +define+RANDOMIZE_GARBAGE_ASSIGN +define+RANDOMIZE_REG_INIT)
+      list(APPEND VERILATOR_OPT +define+RANDOMIZE_MEM_INIT --x-assign unique)
+      list(APPEND VERILATOR_OPT --output-split 20000 --output-split-cfuncs 20000)
+      list(APPEND VERILATOR_OPT --top-module ${TSIM_TOP_NAME} -Mdir ${VERILATOR_TARGET_DIR})
+      list(APPEND VERILATOR_OPT --cc ${VERILATOR_RTL_SRC})
+
+      if (NOT TSIM_USE_TRACE STREQUAL "OFF")
+        list(APPEND VERILATOR_OPT --trace)
+      endif()
+
+      execute_process(COMMAND ${VERILATOR} ${VERILATOR_OPT} RESULT_VARIABLE RETCODE)
+
+      if (NOT RETCODE STREQUAL "0")
+        message(FATAL_ERROR "[TSIM] Verilator failed to compile Verilog to C++...")
+      endif()
+
+      # Build shared library (.so)
+      set(VTA_HW_DPI_DIR ${VTA_DIR}/hardware/dpi)
+      set(VERILATOR_INC_DIR /usr/local/share/verilator/include)
+      set(VERILATOR_LIB_SRC ${VERILATOR_INC_DIR}/verilated.cpp ${VERILATOR_INC_DIR}/verilated_dpi.cpp)
+
+      if (NOT TSIM_USE_TRACE STREQUAL "OFF")
+        list(APPEND VERILATOR_LIB_SRC ${VERILATOR_INC_DIR}/verilated_vcd_c.cpp)
+      endif()
+
+      file(GLOB VERILATOR_GEN_SRC ${VERILATOR_TARGET_DIR}/*.cpp)
+      file(GLOB VERILATOR_SRC ${VTA_HW_DPI_DIR}/tsim_device.cc)
+      add_library(tsim SHARED ${VERILATOR_LIB_SRC} ${VERILATOR_GEN_SRC} ${VERILATOR_SRC})
+
+      set(VERILATOR_DEF VL_TSIM_NAME=V${TSIM_TOP_NAME} VL_PRINTF=printf VM_COVERAGE=0 VM_SC=0)
+      if (NOT TSIM_USE_TRACE STREQUAL "OFF")
+        list(APPEND VERILATOR_DEF VM_TRACE=1 TSIM_TRACE_FILE=${TSIM_BUILD_DIR}/${TSIM_TRACE_NAME}.vcd)
+      else()
+        list(APPEND VERILATOR_DEF VM_TRACE=0)
+      endif()
+      target_compile_definitions(tsim PRIVATE ${VERILATOR_DEF})
+      target_compile_options(tsim PRIVATE -Wno-sign-compare -include V${TSIM_TOP_NAME}.h)
+      target_include_directories(tsim PRIVATE ${VERILATOR_TARGET_DIR} ${VERILATOR_INC_DIR} ${VERILATOR_INC_DIR}/vltstd ${VTA_DIR}/include)
+
+      if(APPLE)
+        set_target_properties(tsim PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+      endif(APPLE)
+
+    endif() # TSIM_TARGET STREQUAL "chisel" OR TSIM_TARGET STREQUAL "verilog"
+
+  else()
+    message(STATUS "[TSIM] could not find Python or Verilator, build is skipped...")
+  endif() # VERILATOR
+endif() # MSVC

vta/apps/tsim_example/hardware/chisel/Makefile

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+clean:
+	-rm -rf target project/target project/project