initial op test (OAID#641)

* update the readme doc * initial the op test from onnx * add protobuf install in ci * add op test in codecov
shaoeric · May 5, 2021 · 14f2250 · 14f2250
1 parent 8279b4e
commit 14f2250
Show file tree

Hide file tree

Showing 21 changed files with 2,340 additions and 150 deletions.
diff --git a/.github/workflows/linux-x64-cpu-gcc.yml b/.github/workflows/linux-x64-cpu-gcc.yml
@@ -10,11 +10,18 @@ jobs:
     - name: update
       run: sudo apt-get update
     - name: protobuf
-      run: sudo apt-get install libopencv-dev    
+      run: sudo apt-get install libprotobuf-dev protobuf-compiler libopencv-dev
     - name: configure
       run: mkdir build && cd build && cmake -DTENGINE_BUILD_TESTS=ON ..
     - name: build
       run: cmake --build build -j 2
+    - name: optest
+      run: |
+        cd build
+        git clone https://github.com/BUG1989/onnx_op_test_data.git
+        mv onnx_op_test_data onnx_node
+        export TG_DEBUG_REF=1
+        ctest --output-on-failure
     - name: models
       run: |
         cd build
@@ -25,8 +32,9 @@ jobs:
         mv x86_data data
         wget http://198.55.120.234/lite/images.zip
         unzip images.zip
-    - name: test
+    - name: model-test
       run: |
         chmod +x ./tests/test.sh
         cd build
+        unset TG_DEBUG_REF
         ../tests/test.sh
diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml
@@ -10,14 +10,14 @@ jobs:
     - name: update
       run: sudo apt-get update
     - name: protobuf
-      run: sudo apt-get install libopencv-dev
+      run: sudo apt-get install libprotobuf-dev protobuf-compiler libopencv-dev
     - name: lcov
       run: sudo apt-get install lcov
     - name: configure
       run: mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=debug -DTENGINE_BUILD_TESTS=ON -DTENGINE_COVERAGE=ON ..
     - name: build
       run: cmake --build build -j 2
-    - name: models
+    - name: test-data
       run: |
         cd build
         wget http://198.55.120.234/lite/models.zip
@@ -27,11 +27,18 @@ jobs:
         mv x86_data data
         wget http://198.55.120.234/lite/images.zip
         unzip images.zip
-    - name: test-cpu
+        git clone https://github.com/BUG1989/onnx_op_test_data.git
+        mv onnx_op_test_data onnx_node        
+    - name: test-models
       run: |
         chmod +x ./tests/test.sh
         cd build
         ../tests/test.sh
+    - name: test-op
+      run: |
+        cd build
+        export TG_DEBUG_REF=1
+        ctest --output-on-failure
     - name: lcov-collect
       run: |
         cd build

diff --git a/.travis.yml b/.travis.yml
diff --git a/source/device/cpu/cpu_dump.c b/source/device/cpu/cpu_dump.c
@@ -552,17 +552,17 @@ void extract_feature_from_tensor(const char* comment, const char* layer_name, co
     }
     else
     {
-        TLOG_WARNING("Tengine: Env var \"TENGINE_DUMP_DIR\" is too long(%d vs. 254). Using default path.\n", strlen(env_path));
+//        TLOG_WARNING("Tengine: Env var \"TENGINE_DUMP_DIR\" is too long(%d vs. 254). Using default path.\n", strlen(env_path));
         sprintf(save_dir, "./output/");
 #ifdef _MSC_VER
         CreateDirectoryA(save_dir, NULL);
 #else
         int ret = mkdir(save_dir, S_IRWXU | S_IRGRP | S_IWGRP | S_IROTH);
-        if (0 != ret)
-        {
-            TLOG_WARNING("Tengine: Create saving folder failed(%d), skip dump.\n", ret);
-            return;
-        }
+//        if (0 != ret)
+//        {
+//            TLOG_WARNING("Tengine: Create saving folder failed(%d), skip dump.\n", ret);
+//            return;
+//        }
 #endif
     }
 

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -4,7 +4,7 @@ FILE (COPY ${CMAKE_SOURCE_DIR}/source/api/c_api.h DESTINATION ${CMAKE_CURRENT_BI
 FILE (COPY ${CMAKE_SOURCE_DIR}/source/api/c_api_ex.h DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tengine)
 
 
-# macro for adding test
+# macro for adding original test
 function (tengine_test name file)
     add_executable (${name} ${CMAKE_CURRENT_SOURCE_DIR}/${file} "${PROJECT_SOURCE_DIR}/tests/common/tengine_operations.cpp")
 
@@ -20,100 +20,15 @@ function (tengine_test name file)
         target_compile_options (${name} PRIVATE "-mfp16-format=ieee")
     endif()
 
-    add_test (${name} ${name})
+    # add_test (${name} ${name})
 endfunction()
 
 # operator level test
+#[[
 tengine_test(test_op_conv               op/test_op_conv.c)
 tengine_test(test_op_prelu              op/test_op_prelu.c)
 tengine_test(test_op_relu               op/test_op_relu.c)
 tengine_test(test_op_relu6              op/test_op_relu6.c)
-
-if (TENGINE_ENABLE_TIM_VX)
-    tengine_test(test_op_prelu_timvx        op/test_op_prelu_timvx.c)
-endif()
-
-#[[ add op
-tengine_test(test_op_conv               op/test_op_conv.c)
-tengine_test(test_op_bias               op/test_op_bias.cpp)
-tengine_test(test_op_batchnorm          op/test_op_batchnorm.c)
-tengine_test(test_op_ceil               op/test_op_ceil.c)
-tengine_test(test_op_clip               op/test_op_clip.c)
-tengine_test(test_op_concat             op/test_op_concat.cpp)
-tengine_test(test_op_crop               op/test_op_crop.c)
-tengine_test(test_op_deconv             op/test_op_deconv.c)
-tengine_test(test_op_depthtospace       op/test_op_depth_to_space.c)
-tengine_test(test_op_dropout            op/test_op_dropout.c)
-tengine_test(test_op_eltwise            op/test_op_eltwise.c)
-tengine_test(test_op_expanddims         op/test_op_expanddims.c)
-tengine_test(test_op_flatten            op/test_op_flatten.c)
-tengine_test(test_op_gather             op/test_op_gather.c)
-tengine_test(test_op_lrn                op/test_op_lrn.c)
-tengine_test(test_op_permute            op/test_op_permute.cpp)
-tengine_test(test_op_pooling            op/test_op_pooling.c)
-tengine_test(test_op_relu               op/test_op_relu.c)
-tengine_test(test_op_relu6              op/test_op_relu6.c)
-tengine_test(test_op_prelu              op/test_op_prelu.c)
-tengine_test(test_op_reshape            op/test_op_reshape.c)
-tengine_test(test_op_resize             op/test_op_resize.c)
-tengine_test(test_op_reverse            op/test_op_reverse.c)
-tengine_test(test_op_round              op/test_op_round.c)
-tengine_test(test_op_scale              op/test_op_scale.c)
-tengine_test(test_op_spacetodepth       op/test_op_spacetodepth.c)
-tengine_test(test_op_softmax            op/test_op_softmax.c)
-tengine_test(test_op_squareddifference  op/test_op_squareddifference.c)
-tengine_test(test_op_squeeze            op/test_op_squeeze.c)
-tengine_test(test_op_unary              op/test_op_unary.cpp)
-tengine_test(test_op_zeroslike          op/test_op_zeroslike.c)
-tengine_test(test_op_tanh               op/test_op_tanh.c)
-tengine_test(test_op_sigmoid            op/test_op_sigmoid.c)
-tengine_test(test_op_lstm               op/test_op_lstm.cpp)
-tengine_test(test_op_gru                op/test_op_gru.cpp)
-tengine_test(test_op_reduction          op/test_op_reduction.cpp)
-tengine_test(test_op_shufflechannel     op/test_op_shufflechannel.cpp)
-tengine_test(test_op_threshold          op/test_op_threshold.cpp)
-tengine_test(test_op_hardsigmoid        op/test_op_hardsigmoid.cpp)
-tengine_test(test_op_embedding          op/test_op_embedding.cpp)
-tengine_test(test_op_instancenorm       op/test_op_instancenorm.cpp)
-tengine_test(test_op_mvn                op/test_op_mvn.cpp)
-tengine_test(test_op_absval             op/test_op_absval.cpp)
-tengine_test(test_op_cast               op/test_op_cast.cpp)
-tengine_test(test_op_hardswish          op/test_op_hardswish.cpp)
-tengine_test(test_op_selu               op/test_op_selu.cpp)
-tengine_test(test_op_elu                op/test_op_elu.cpp)
-tengine_test(test_op_broadmul           op/test_op_broadmul.cpp)
-tengine_test(test_op_interp             op/test_op_interp.cpp)
-tengine_test(test_op_argmax             op/test_op_argmax.cpp)
-tengine_test(test_op_argmin             op/test_op_argmin.cpp)
-tengine_test(test_op_maximum            op/test_op_maximum.cpp)
-tengine_test(test_op_minimum            op/test_op_minimum.cpp)
-tengine_test(test_op_mean               op/test_op_mean.cpp)
-tengine_test(test_op_add_n              op/test_op_add_n.cpp)
-]]
-
-#[[ new add
-tengine_test(test_op_gather_tflite              op/test_op_gather_tflite.cpp)
-tengine_test(test_op_transpose_tflite             op/test_op_transpose_tflite.cpp)
-tengine_test(test_op_psroipooling             op/test_op_psroipooling.cpp)
-tengine_test(test_op_roialign           op/test_op_roialign.cpp)
-tengine_test(test_op_leakyRelu_onnx           op/test_op_leakyRelu_onnx.cpp)
-tengine_test(test_op_clip_onnx           op/test_op_clip_onnx.cpp)
-tengine_test(test_op_div_onnx           op/test_op_div_onnx.cpp)
-tengine_test(test_op_elu_onnx           op/test_op_elu_onnx.cpp)
-tengine_test(test_op_mul_onnx           op/test_op_mul_onnx.cpp)
-tengine_test(test_op_transpose_onnx           op/test_op_transpose_onnx.cpp)
-]]
-
-# unit test
-#[[
-tengine_test(test_buddy_mem     utility/test_buddy_mem.c)
-tengine_test(test_graph         utility/test_graph.c)
-tengine_test(test_log           utility/test_log.cpp)
-tengine_test(test_model_load    utility/test_model_load.c)
-tengine_test(test_pack          utility/test_pack.cpp)
-tengine_test(test_pack_graph    utility/test_pack_graph.c)
-tengine_test(test_param         utility/test_param.c)
-tengine_test(test_plugin        utility/test_plugin.c)
 ]]
 
 # model level test
@@ -133,9 +48,67 @@ tengine_test(test_tm_landmark      models/test_tm_landmark.cpp)
 tengine_test(test_tm_yolact        models/test_tm_yolact.cpp)
 tengine_test(tm_mfn                models/test_tm_mfn.cpp)
 
-#
-#tengine_test(test_tiny.c tiny_graph_generated.c utility/tiny)
+if (TENGINE_ENABLE_TIM_VX)
+    tengine_test(test_op_prelu_timvx        op/test_op_prelu_timvx.c)
+endif()
+
+
+# operator level test using onnx test
+find_package(Protobuf)
+
+if(PROTOBUF_FOUND)
+    protobuf_generate_cpp(ONNX_PROTO_SRCS ONNX_PROTO_HDRS onnx.proto)
+
+    function (tengine_onnx_op_test name file)
+        add_executable (${name} ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${ONNX_PROTO_SRCS} ${ONNX_PROTO_HDRS})
+
+        target_link_libraries (${name} PRIVATE ${CMAKE_PROJECT_NAME} ${PROTOBUF_LIBRARIES})
+
+        target_include_directories (${name} PRIVATE "${PROTOBUF_INCLUDE_DIR}")
+        target_include_directories (${name} PRIVATE "${PROJECT_SOURCE_DIR}/source")
+        target_include_directories (${name} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+        target_include_directories (${name} PRIVATE "${PROJECT_BINARY_DIR}")
+        target_include_directories (${name} PRIVATE "${PROJECT_BINARY_DIR}/source")
+
+        add_test (${name} ${name})
+    endfunction()
+
+    tengine_onnx_op_test(test_onnx_op_add                           op/test_onnx_op_add.cpp)
+    tengine_onnx_op_test(test_onnx_op_averagepool_2d_default        op/test_onnx_op_averagepool_2d_default.cpp)
+    tengine_onnx_op_test(test_onnx_op_basic_conv_with_padding       op/test_onnx_op_basic_conv_with_padding.cpp)
+    tengine_onnx_op_test(test_onnx_op_elu                           op/test_onnx_op_elu.cpp)
+    tengine_onnx_op_test(test_onnx_op_exp                           op/test_onnx_op_exp.cpp)
+    tengine_onnx_op_test(test_onnx_op_floor                         op/test_onnx_op_floor.cpp)
+    tengine_onnx_op_test(test_onnx_op_globalaveragepool             op/test_onnx_op_globalaveragepool.cpp)
+    tengine_onnx_op_test(test_onnx_op_maxpool_2d_default            op/test_onnx_op_maxpool_2d_default.cpp)
+    tengine_onnx_op_test(test_onnx_op_relu                          op/test_onnx_op_relu.cpp)
+    # tengine_onnx_op_test(test_onnx_op_softmax_default_axis          op/test_onnx_op_softmax_default_axis.cpp)
+    tengine_onnx_op_test(test_onnx_op_squeeze                       op/test_onnx_op_squeeze.cpp)
+    tengine_onnx_op_test(test_onnx_op_tanh                          op/test_onnx_op_tanh.cpp)
+    tengine_onnx_op_test(test_onnx_op_unsqueeze_axis_1              op/test_onnx_op_unsqueeze_axis_1.cpp)
+else()
+    message(WARNING "Protobuf not found, onnx op test won't be built")
+endif()
+# macro for adding test
+function (tengine_onnx_op_test name file)
+
+    target_link_libraries (${name} PRIVATE ${CMAKE_PROJECT_NAME})
+
+    target_include_directories (${name} PRIVATE "${PROJECT_SOURCE_DIR}/source")
+    target_include_directories (${name} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+    target_include_directories (${name} PRIVATE "${PROJECT_BINARY_DIR}")
+    target_include_directories (${name} PRIVATE "${PROJECT_BINARY_DIR}/source")
+    target_include_directories (${name} PRIVATE "${PROJECT_SOURCE_DIR}/tests/common")
+
+    if (${TENGINE_TARGET_PROCESSOR} MATCHES "ARM" AND (NOT ANDROID AND NOT OHOS) AND TENGINE_TARGET_PROCESSOR_32Bit)
+        target_compile_options (${name} PRIVATE "-mfp16-format=ieee")
+    endif()
+
+    add_test (${name} ${name})
+endfunction()
+
 
+# operator level test using opencv
 find_package(OpenCV QUIET)
 
 if(OpenCV_FOUND)
@@ -158,6 +131,6 @@ if(OpenCV_FOUND)
 
     #tengine_test_cv(test_tm_yolact_cv      models/test_tm_yolact_cv.cpp)
 else()
-    message(WARNING "OpenCV not found, some examples won't be built")
+    message(WARNING "OpenCV not found, some model examples won't be built")
 endif()