Fix perf test executable.

cmake/onnxruntime_unittests.cmake

@@ -472,6 +472,9 @@ set(onnx_test_runner_common_srcs
   ${onnx_test_runner_src_dir}/onnxruntime_event.h
   ${onnx_test_runner_src_dir}/sync_api.h
   ${onnx_test_runner_src_dir}/sync_api.cc
+  ${onnx_test_runner_src_dir}/callback.h
+  ${onnx_test_runner_src_dir}/callback.cc
+  ${onnx_test_runner_src_dir}/mem_buffer.h
   ${onnx_test_runner_src_dir}/tensorprotoutils.h
   ${onnx_test_runner_src_dir}/tensorprotoutils.cc)
 
@@ -566,7 +569,7 @@ onnxruntime_add_include_to_target(onnxruntime_perf_test gsl)
 
 if (onnxruntime_BUILD_SHARED_LIB)
   set(onnxruntime_perf_test_libs onnxruntime_test_utils onnx_test_runner_common onnxruntime_common
-          onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime onnxruntime_framework onnx
+          onnx_test_data_proto onnx_proto libprotobuf ${GETOPT_LIB_WIDE} onnxruntime
           ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
   if(onnxruntime_USE_NSYNC)
     list(APPEND onnxruntime_perf_test_libs nsync_cpp)

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -150,7 +150,6 @@ ORT_RUNTIME_CLASS(RunOptions);
 ORT_RUNTIME_CLASS(TypeInfo);
 ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
 ORT_RUNTIME_CLASS(SessionOptions);
-ORT_RUNTIME_CLASS(Callback);
 ORT_RUNTIME_CLASS(CustomOpDomain);
 ORT_RUNTIME_CLASS(Allocator);
 

onnxruntime/test/onnx/TestCase.cc

@@ -13,6 +13,7 @@
 #include "core/platform/ort_mutex.h"
 #include "core/session/onnxruntime_cxx_api.h"
 #include "core/framework/path_lib.h"
+#include "core/framework/allocator.h"
 #include <sstream>
 #include <map>
 #include <regex>
@@ -265,18 +266,18 @@ static void SortTensorFileNames(std::vector<std::basic_string<PATH_CHAR_TYPE>>&
   }
 }
 
-OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b) {
+OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, onnxruntime::test::HeapBuffer& b) {
   size_t len = 0;
-  auto status = onnxruntime::test::utils::GetSizeInBytesFromTensorProto<0>(t, &len);
+  auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(t, &len);
   if (!status.IsOK()) {
     ORT_THROW(status.ToString());
   }
   void* p = len == 0 ? nullptr : b.AllocMemory(len);
-  auto d = std::make_unique<onnxruntime::OrtCallback>();
-  auto temp_value = std::make_unique<OrtValue>();
+  Ort::Value temp_value{nullptr};
+  auto d = std::make_unique<onnxruntime::test::OrtCallback>();
   OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
-  status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, t,
-                                                          MemBuffer(p, len, cpu_allocator_info), *temp_value, *d);
+  status = onnxruntime::test::TensorProtoToMLValue(t, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info),
+                                                   temp_value, *d);
   if (!status.IsOK()) {
     ORT_THROW(status.ToString());
   }
@@ -287,7 +288,8 @@ OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, HeapBuffer& b)
 }
 
 void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo* modelinfo,
-                  std::unordered_map<std::string, OrtValue*>& name_data_map, HeapBuffer& b, std::ostringstream& oss) {
+                  std::unordered_map<std::string, OrtValue*>& name_data_map, onnxruntime::test::HeapBuffer& b,
+                  std::ostringstream& oss) {
   google::protobuf::io::FileInputStream f(test_data_pb_fd);
   f.SetCloseOnDelete(true);
   google::protobuf::io::CodedInputStream coded_input(&f);
@@ -395,7 +397,8 @@ class OnnxTestCase : public ITestCase {
     return std::string();
   }
 
-  void ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, HeapBuffer& b, bool is_input,
+  void ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, onnxruntime::test::HeapBuffer& b,
+                       bool is_input,
                        std::unordered_map<std::string, OrtValue*>& out);
 
   std::once_flag model_parsed_;
@@ -430,7 +433,8 @@ class OnnxTestCase : public ITestCase {
   std::string GetTestCaseVersion() const override {
     return model_info_->GetModelVersion();
   }
-  void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>&, bool is_input) override;
+  void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map<std::string, OrtValue*>&,
+                    bool is_input) override;
 };
 
 ITestCase* CreateOnnxTestCase(const std::string& test_case_name, TestModelInfo* model,
@@ -503,7 +507,8 @@ static void LoadTensors(const std::vector<PATH_STRING_TYPE>& pb_files,
   }
 }
 
-void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
+void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b,
+                                std::unordered_map<std::string, OrtValue*>& name_data_map,
                                 bool is_input) {
   if (id >= test_data_dirs_.size()) {
     ORT_THROW("index out of bound");
@@ -556,7 +561,8 @@ void OnnxTestCase::LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std
   ConvertTestData(test_data_pbs, b, is_input, name_data_map);
 }
 
-void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs, HeapBuffer& b,
+void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto>& test_data_pbs,
+                                   onnxruntime::test::HeapBuffer& b,
                                    bool is_input, std::unordered_map<std::string, OrtValue*>& out) {
   bool has_valid_names = true;
   std::vector<std::string> var_names(test_data_pbs.size());
@@ -582,16 +588,16 @@ void OnnxTestCase::ConvertTestData(const std::vector<ONNX_NAMESPACE::TensorProto
     const ONNX_NAMESPACE::TensorProto& input = test_data_pbs[input_index];
     size_t len = 0;
 
-    auto status = onnxruntime::test::utils::GetSizeInBytesFromTensorProto<0>(input, &len);
+    auto status = onnxruntime::test::GetSizeInBytesFromTensorProto<0>(input, &len);
     if (!status.IsOK()) {
       ORT_THROW(status.ToString());
     }
     void* p = len == 0 ? nullptr : b.AllocMemory(len);
-    auto d = std::make_unique<onnxruntime::OrtCallback>();
+    Ort::Value v1{nullptr};
+    auto d = std::make_unique<onnxruntime::test::OrtCallback>();
     OrtAllocatorInfo cpu_allocator_info(onnxruntime::CPU, OrtDeviceAllocator, OrtDevice(), 0, OrtMemTypeDefault);
-    auto v1 = std::make_unique<OrtValue>();
-    status = onnxruntime::test::utils::TensorProtoToMLValue(Env::Default(), nullptr, input,
-                                                            MemBuffer(p, len, cpu_allocator_info), *v1, *d);
+    status = onnxruntime::test::TensorProtoToMLValue(input, onnxruntime::test::MemBuffer(p, len, cpu_allocator_info),
+                                                     v1, *d);
     if (!status.IsOK()) {
       ORT_THROW(status.ToString());
     }

onnxruntime/test/onnx/TestCase.h

@@ -19,7 +19,7 @@ class ValueInfoProto;
 //One test case can contain multiple test data(input/output pairs)
 class ITestCase {
  public:
-  virtual void LoadTestData(size_t id, HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
+  virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map<std::string, OrtValue*>& name_data_map,
                             bool is_input) = 0;
   virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0;
   virtual const std::string& GetNodeName() const = 0;

onnxruntime/test/onnx/callback.cc

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "callback.h"
+
+namespace onnxruntime {
+namespace test {
+void OrtRunCallback(OrtCallback* f) noexcept {
+  if (f == nullptr) return;
+  if (f->f != nullptr) {
+    f->f(f->param);
+    delete f;
+  }
+}
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/onnx/callback.h

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+#pragma once
+
+namespace onnxruntime {
+namespace test {
+struct OrtCallback {
+  void (*f)(void* param) noexcept;
+  void* param;
+};
+
+/**
+ *  f will be freed in this call
+ */
+void OrtRunCallback(OrtCallback* f) noexcept;
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/onnx/heap_buffer.cc

@@ -2,18 +2,22 @@
 // Licensed under the MIT License.
 
 #include "heap_buffer.h"
-#include "core/framework/callback.h"
 #include "core/session/onnxruntime_c_api.h"
+#include "callback.h"
 
-void HeapBuffer::AddDeleter(onnxruntime::OrtCallback* d) {
+namespace onnxruntime {
+namespace test {
+void HeapBuffer::AddDeleter(OrtCallback* d) {
   if (d != nullptr) deleters_.push_back(d);
 }
 
 HeapBuffer::~HeapBuffer() {
   for (auto d : deleters_) {
-    onnxruntime::OrtRunCallback(d);
+    OrtRunCallback(d);
   }
   for (void* p : buffers_) {
     free(p);
   }
-}
+}
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/onnx/heap_buffer.h

@@ -4,10 +4,10 @@
 #pragma once
 #include <vector>
 #include <memory>
+
 namespace onnxruntime {
+namespace test {
 struct OrtCallback;
-}
-
 /**
  * A holder for delay freed buffers
  */
@@ -23,9 +23,11 @@ class HeapBuffer {
     buffers_.push_back(p);
     return p;
   }
-  void AddDeleter(onnxruntime::OrtCallback* d);
+  void AddDeleter(OrtCallback* d);
 
  private:
-  std::vector<onnxruntime::OrtCallback*> deleters_;
+  std::vector<OrtCallback*> deleters_;
   std::vector<void*> buffers_;
-};
+};
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/onnx/mem_buffer.h

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "core/common/common.h"
+
+namespace onnxruntime {
+namespace test {
+/**
+ * A simple POD for using with tensor deserialization
+ */
+class MemBuffer {
+ public:
+  MemBuffer(void* buffer, size_t len, const OrtAllocatorInfo& alloc_info)
+      : buffer_(buffer), len_(len), alloc_info_(alloc_info) {}
+  void* GetBuffer() const { return buffer_; }
+
+  size_t GetLen() const { return len_; }
+  const OrtAllocatorInfo& GetAllocInfo() const { return alloc_info_; }
+
+ private:
+  void* const buffer_;
+  const size_t len_;
+  const OrtAllocatorInfo& alloc_info_;
+};
+};  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/onnx/runner.cc

@@ -27,45 +27,44 @@
 using namespace onnxruntime;
 using ::onnxruntime::common::Status;
 
-// Permanently exclude following tests because ORT support only opset staring from 7, 
+// Permanently exclude following tests because ORT support only opset staring from 7,
 // Please make no more changes to the list
-const std::set<std::string> immutable_broken_tests = 
-{
-    "AvgPool1d", 
-    "AvgPool1d_stride",
-    "AvgPool2d",
-    "AvgPool2d_stride",
-    "AvgPool3d",
-    "AvgPool3d_stride",
-    "AvgPool3d_stride1_pad0_gpu_input",
-    "BatchNorm1d_3d_input_eval",
-    "BatchNorm2d_eval",
-    "BatchNorm2d_momentum_eval",
-    "BatchNorm3d_eval",
-    "BatchNorm3d_momentum_eval",
-    "GLU",
-    "GLU_dim",
-    "Linear",
-    "PReLU_1d",
-    "PReLU_1d_multiparam",
-    "PReLU_2d",
-    "PReLU_2d_multiparam",
-    "PReLU_3d",
-    "PReLU_3d_multiparam",
-    "PoissonNLLLLoss_no_reduce",
-    "Softsign",
-    "operator_add_broadcast",
-    "operator_add_size1_broadcast",
-    "operator_add_size1_right_broadcast",
-    "operator_add_size1_singleton_broadcast",
-    "operator_addconstant",
-    "operator_addmm",
-    "operator_basic",
-    "operator_mm",
-    "operator_non_float_params",
-    "operator_params", 
-    "operator_pow"
-};
+const std::set<std::string> immutable_broken_tests =
+    {
+        "AvgPool1d",
+        "AvgPool1d_stride",
+        "AvgPool2d",
+        "AvgPool2d_stride",
+        "AvgPool3d",
+        "AvgPool3d_stride",
+        "AvgPool3d_stride1_pad0_gpu_input",
+        "BatchNorm1d_3d_input_eval",
+        "BatchNorm2d_eval",
+        "BatchNorm2d_momentum_eval",
+        "BatchNorm3d_eval",
+        "BatchNorm3d_momentum_eval",
+        "GLU",
+        "GLU_dim",
+        "Linear",
+        "PReLU_1d",
+        "PReLU_1d_multiparam",
+        "PReLU_2d",
+        "PReLU_2d_multiparam",
+        "PReLU_3d",
+        "PReLU_3d_multiparam",
+        "PoissonNLLLLoss_no_reduce",
+        "Softsign",
+        "operator_add_broadcast",
+        "operator_add_size1_broadcast",
+        "operator_add_size1_right_broadcast",
+        "operator_add_size1_singleton_broadcast",
+        "operator_addconstant",
+        "operator_addmm",
+        "operator_basic",
+        "operator_mm",
+        "operator_non_float_params",
+        "operator_params",
+        "operator_pow"};
 
 void ORT_CALLBACK RunTestCase(ORT_CALLBACK_INSTANCE pci, void* context, ORT_WORK work) {
   OnnxRuntimeCloseThreadpoolWork(work);
@@ -232,13 +231,13 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c
   }
   for (size_t i = 0; i != env.tests.size(); ++i) {
     if (!results[i]) {
-      stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
+      stat.AddFailedTest(std::pair<std::string, std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
       continue;
     }
     const TestCaseResult& r = *results[i];
     for (const EXECUTE_RESULT res : r.GetExcutionResult()) {
       if (res != EXECUTE_RESULT::SUCCESS && res != EXECUTE_RESULT::NOT_SUPPORT) {
-        stat.AddFailedTest(std::pair<std::string,std::string>(env.tests[i]->GetTestCaseName(),env.tests[i]->GetTestCaseVersion()));
+        stat.AddFailedTest(std::pair<std::string, std::string>(env.tests[i]->GetTestCaseName(), env.tests[i]->GetTestCaseVersion()));
       }
       switch (res) {
         case EXECUTE_RESULT::SUCCESS:
@@ -347,7 +346,7 @@ void DataRunner::RunTask(size_t task_id, ORT_CALLBACK_INSTANCE pci, bool store_r
 }
 
 EXECUTE_RESULT DataRunner::RunTaskImpl(size_t task_id) {
-  HeapBuffer holder;
+  onnxruntime::test::HeapBuffer holder;
   std::unordered_map<std::string, OrtValue*> feeds;
   c_->LoadTestData(task_id, holder, feeds, true);
 
@@ -499,7 +498,6 @@ void SeqTestRunner::Start(ORT_CALLBACK_INSTANCE pci, size_t) {
 }
 
 void RunSingleTestCase(ITestCase* info, Ort::Env& env, const Ort::SessionOptions& sf, size_t concurrent_runs, size_t repeat_count, PThreadPool tpool, ORT_CALLBACK_INSTANCE pci, TestCaseCallBack on_finished) {
-
   //for test in immutable list, do not even run it
   if (immutable_broken_tests.find(info->GetTestCaseName()) != immutable_broken_tests.end()) {
     on_finished(std::make_shared<TestCaseResult>(0, EXECUTE_RESULT::NOT_SUPPORT, info->GetNodeName()), pci);