Property to terminate tbb threads

src/bindings/python/src/pyopenvino/core/properties/properties.cpp

@@ -28,6 +28,7 @@ void regmodule_properties(py::module m) {
     wrap_property_RW(m_properties, ov::inference_num_threads, "inference_num_threads");
     wrap_property_RW(m_properties, ov::compilation_num_threads, "compilation_num_threads");
     wrap_property_RW(m_properties, ov::affinity, "affinity");
+    wrap_property_RW(m_properties, ov::force_tbb_terminate, "force_tbb_terminate");
 
     wrap_property_RO(m_properties, ov::supported_properties, "supported_properties");
     wrap_property_RO(m_properties, ov::available_devices, "available_devices");

src/cmake/ie_parallel.cmake

@@ -72,17 +72,21 @@ function(set_ie_threading_interface_for TARGET_NAME)
 
     if(target_type STREQUAL "INTERFACE_LIBRARY")
         set(LINK_TYPE "INTERFACE")
+        set(COMPILE_DEF_TYPE "INTERFACE")
     elseif(target_type STREQUAL "EXECUTABLE" OR target_type STREQUAL "OBJECT_LIBRARY" OR
            target_type STREQUAL "MODULE_LIBRARY")
         set(LINK_TYPE "PRIVATE")
+        set(COMPILE_DEF_TYPE "PUBLIC")
     elseif(target_type STREQUAL "STATIC_LIBRARY")
         # Affected libraries: inference_engine_s, openvino_gapi_preproc_s
         # they don't have TBB in public headers => PRIVATE
         set(LINK_TYPE "PRIVATE")
+        set(COMPILE_DEF_TYPE "PUBLIC")
     elseif(target_type STREQUAL "SHARED_LIBRARY")
         # Affected libraries: inference_engine only
         # TODO: why TBB propogates its headers to inference_engine?
         set(LINK_TYPE "PRIVATE")
+        set(COMPILE_DEF_TYPE "PUBLIC")
     else()
         message(WARNING "Unknown target type")
     endif()
@@ -113,6 +117,7 @@ function(set_ie_threading_interface_for TARGET_NAME)
         if (TBB_FOUND)
             set(IE_THREAD_DEFINE "IE_THREAD_TBB")
             ie_target_link_libraries(${TARGET_NAME} ${LINK_TYPE} ${TBB_IMPORTED_TARGETS})
+            target_compile_definitions(${TARGET_NAME} ${COMPILE_DEF_TYPE} TBB_PREVIEW_WAITING_FOR_WORKERS=1)
         else ()
             set(THREADING "SEQ" PARENT_SCOPE)
             message(WARNING "TBB was not found by the configured TBB_DIR path.\

src/core/CMakeLists.txt

@@ -113,7 +113,7 @@ set_source_files_properties("${CMAKE_CURRENT_SOURCE_DIR}/src/pass/convert_precis
 
 # Defines macro in C++ to load backend plugin
 target_include_directories(ngraph_obj PUBLIC $<BUILD_INTERFACE:${OV_CORE_INCLUDE_PATH}>
-                                      PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
+                                    PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
 add_library(ngraph INTERFACE)
 target_link_libraries(ngraph INTERFACE openvino::runtime)

src/core/include/openvino/core/model.hpp

@@ -114,6 +114,9 @@ class OPENVINO_API Model : public std::enable_shared_from_this<Model> {
     /// \brief Clones the original model
     std::shared_ptr<ov::Model> clone() const;
 
+    /// \brief Reference to frontend maanager
+    std::shared_ptr<void> m_femgr;
+
     /// Model outputs
     std::vector<ov::Output<ov::Node>> outputs();
     ov::Output<ov::Node> output();

src/core/include/openvino/op/constant.hpp

@@ -716,6 +716,7 @@ class OPENVINO_API Constant : public Op {
     element::Type m_element_type;
     Shape m_shape{};
     std::shared_ptr<ngraph::runtime::AlignedBuffer> m_data;
+    std::shared_ptr<void> m_femgr;
     mutable std::atomic_bool m_all_elements_bitwise_identical{false};
     mutable std::atomic_bool m_all_elements_bitwise_identical_checked{false};
     bool m_alloc_buffer_on_visit_attributes = true;

src/core/src/model.cpp

@@ -34,6 +34,13 @@ BWDCMP_RTTI_DEFINITION(ov::AttributeAdapter<std::shared_ptr<ov::Model>>);
 
 atomic<size_t> ov::Model::m_next_instance_id(0);
 
+namespace ov {
+namespace frontend {
+class FrontEndManager;
+std::shared_ptr<FrontEndManager> get_frontend_manager();
+}  // namespace frontend
+}  // namespace ov
+
 namespace {
 
 void check_all_variables_registered(const std::vector<shared_ptr<ov::Node>>& ordered_ops,
@@ -192,6 +199,7 @@ void ov::Model::prerequirements(bool detect_variables, bool detect_parameters) {
     OV_ITT_SCOPED_TASK(ov::itt::domains::nGraph, "Model::prerequirements");
 
     m_shared_rt_info = std::make_shared<SharedRTInfo>();
+    m_femgr = ov::frontend::get_frontend_manager();
 
     const auto& ordered_ops = get_ordered_ops();
     if (detect_parameters)

src/core/src/op/constant.cpp

@@ -17,6 +17,13 @@
 
 using namespace std;
 
+namespace ov {
+namespace frontend {
+class FrontEndManager;
+std::shared_ptr<FrontEndManager> get_frontend_manager();
+}  // namespace frontend
+}  // namespace ov
+
 template <typename T>
 static inline string to_cpp_string(T value) {
     string rc;
@@ -50,6 +57,7 @@ ov::op::v0::Constant::Constant(const shared_ptr<ngraph::runtime::Tensor>& tensor
         tensor->read(get_data_ptr_nc(), tensor->get_size_in_bytes());
     }
     constructor_validate_and_infer_types();
+    m_femgr = ov::frontend::get_frontend_manager();
 }
 
 ov::op::v0::Constant::Constant(const element::Type& type,
@@ -193,6 +201,7 @@ ov::op::v0::Constant::Constant(bool memset_allocation, const element::Type& type
       m_shape(shape) {
     allocate_buffer(memset_allocation);
     constructor_validate_and_infer_types();
+    m_femgr = ov::frontend::get_frontend_manager();
 }
 
 void ov::op::v0::Constant::allocate_buffer(bool memset_allocation) {
@@ -214,6 +223,7 @@ ov::op::v0::Constant::Constant(const Constant& other) {
     m_data = other.m_data;
     update_identical_flags(other.m_all_elements_bitwise_identical_checked, other.m_all_elements_bitwise_identical);
     constructor_validate_and_infer_types();
+    m_femgr = ov::frontend::get_frontend_manager();
 }
 
 ov::op::v0::Constant::Constant(const Constant& other, const ov::Shape& new_shape) {
@@ -225,9 +235,14 @@ ov::op::v0::Constant::Constant(const Constant& other, const ov::Shape& new_shape
     m_data = other.m_data;
     update_identical_flags(other.m_all_elements_bitwise_identical_checked, other.m_all_elements_bitwise_identical);
     constructor_validate_and_infer_types();
+    m_femgr = ov::frontend::get_frontend_manager();
 }
 
-ov::op::v0::Constant::~Constant() = default;
+ov::op::v0::Constant::~Constant() {
+    // guarantee m_data is released before femgr
+    m_data = nullptr;
+    m_femgr = nullptr;
+}
 
 string ov::op::v0::Constant::convert_value_to_string(size_t index) const {
     string rc;

src/frontends/common/include/openvino/frontend/manager.hpp

@@ -22,6 +22,8 @@ using FrontEndFactory = std::function<FrontEnd::Ptr()>;
 /// frontends This is a main frontend entry point for client applications
 class FRONTEND_API FrontEndManager final {
 public:
+    using Ptr = std::shared_ptr<FrontEndManager>;
+
     /// \brief Default constructor. Searches and loads of available frontends
     FrontEndManager();
 
@@ -79,6 +81,7 @@ class FRONTEND_API FrontEndManager final {
 
 template <>
 FRONTEND_API FrontEnd::Ptr FrontEndManager::load_by_model(const std::vector<ov::Any>& variants);
+FRONTEND_API FrontEndManager::Ptr get_frontend_manager();
 
 // --------- Plugin exporting information --------------
 

src/frontends/common/src/manager.cpp

@@ -242,3 +242,40 @@ template <>
 FrontEnd::Ptr FrontEndManager::load_by_model(const std::vector<ov::Any>& variants) {
     return load_by_model_impl(variants);
 }
+namespace ov {
+namespace frontend {
+
+namespace {
+
+class FrontEndManagerHolder {
+    std::mutex m_mutex;
+    std::weak_ptr<FrontEndManager> m_manager;
+
+public:
+    FrontEndManagerHolder(const FrontEndManagerHolder&) = delete;
+    FrontEndManagerHolder& operator=(const FrontEndManagerHolder&) = delete;
+
+    FrontEndManagerHolder() = default;
+
+    FrontEndManager::Ptr get() {
+        std::lock_guard<std::mutex> lock(m_mutex);
+        auto manager = m_manager.lock();
+        if (!manager)
+            m_manager = manager = std::make_shared<FrontEndManager>();
+        return manager;
+    }
+};
+
+FrontEndManagerHolder& get_frontend_manager_holder() {
+    static FrontEndManagerHolder frontend_manager_holder;
+    return frontend_manager_holder;
+}
+
+}  // namespace
+
+FrontEndManager::Ptr get_frontend_manager() {
+    return get_frontend_manager_holder().get();
+}
+
+}  // namespace frontend
+}  // namespace ov

src/inference/dev_api/threading/ie_executor_manager.hpp

@@ -66,6 +66,16 @@ class INFERENCE_ENGINE_API_CLASS(ExecutorManager) {
      */
     INFERENCE_ENGINE_DEPRECATED("Use IInferencePlugin::executorManager() instead")
     static ExecutorManager* getInstance();
+
+    /**
+     * @brief Set TBB terminate flag
+     * @param flag A boolean value:
+     * True to terminate tbb during destruction
+     * False to not terminate tbb during destruction
+     * @return void
+     */
+    virtual void setTbbFlag(bool flag) = 0;
+    virtual bool getTbbFlag() = 0;
 };
 
 INFERENCE_ENGINE_API_CPP(ExecutorManager::Ptr) executorManager();

src/inference/include/openvino/runtime/properties.hpp

@@ -509,6 +509,15 @@ static constexpr Property<uint32_t, PropertyMutability::RW> auto_batch_timeout{"
 static constexpr Property<std::tuple<unsigned int, unsigned int, unsigned int>, PropertyMutability::RO>
     range_for_async_infer_requests{"RANGE_FOR_ASYNC_INFER_REQUESTS"};
 
+/**
+ * @brief Read-write property to set whether force terminate tbb when ov core destruction
+ * value type: boolean
+ *   - True explicitly terminate tbb when ov core destruction
+ *   - False will not involve additional tbb operations when core destruction
+ * @ingroup ov_runtime_cpp_prop_api
+ */
+static constexpr Property<bool, PropertyMutability::RW> force_tbb_terminate{"FORCE_TBB_TERMINATE"};
+
 /**
  * @brief Namespace with device properties
  */

src/inference/src/ie_core.cpp

@@ -10,6 +10,7 @@
 #include <memory>
 #include <mutex>
 #include <string>
+#include <threading/ie_executor_manager.hpp>
 #include <vector>
 
 #include "any_copy.hpp"
@@ -54,6 +55,11 @@ using namespace std::placeholders;
 
 namespace ov {
 
+namespace frontend {
+class FrontEndManager;
+std::shared_ptr<FrontEndManager> get_frontend_manager();
+}  // namespace frontend
+
 // Specify the default device when no device name is provided.
 const std::string DEFAULT_DEVICE_NAME = "DEFAULT_DEVICE";
 
@@ -258,6 +264,8 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
         }
     };
 
+    std::shared_ptr<void> frontEndManagerPtr;
+    ExecutorManager::Ptr executorManagerPtr;
     mutable std::unordered_set<std::string> opsetNames;
     // TODO: make extensions to be optional with conditional compilation
     mutable std::vector<ie::IExtensionPtr> extensions;
@@ -424,6 +432,8 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
 
 public:
     CoreImpl(bool _newAPI) : newAPI(_newAPI) {
+        executorManagerPtr = executorManager();
+        frontEndManagerPtr = ov::frontend::get_frontend_manager();
         opsetNames.insert("opset1");
         opsetNames.insert("opset2");
         opsetNames.insert("opset3");
@@ -860,6 +870,10 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
                         "You can only get_config of the AUTO itself (without devices). "
                         "get_config is also possible for the individual devices before creating the AUTO on top.");
 
+        if (name == ov::force_tbb_terminate.name()) {
+            const auto flag = executorManager()->getTbbFlag();
+            return decltype(ov::force_tbb_terminate)::value_type(flag);
+        }
         auto parsed = parseDeviceNameIntoConfig(deviceName, arguments);
         return GetCPPPluginByName(parsed._deviceName).get_property(name, parsed._config);
     }
@@ -1113,6 +1127,18 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
     void SetConfigForPlugins(const std::map<std::string, std::string>& configMap, const std::string& deviceName) {
         auto config = configMap;
 
+        for (auto& item : config) {
+            if (item.first == ov::force_tbb_terminate.name()) {
+                auto flag = item.second == CONFIG_VALUE(YES) ? true : false;
+                executorManager()->setTbbFlag(flag);
+                config.erase(item.first);
+                break;
+            }
+        }
+        if (config.empty()) {
+            return;
+        }
+
         InferenceEngine::DeviceIDParser parser(deviceName);
         std::string clearDeviceName = parser.getDeviceName();
 

src/inference/src/ie_network_reader.cpp

@@ -440,11 +440,6 @@ CNNNetwork convert_to_cnnnetwork(std::shared_ptr<ngraph::Function>& function,
     OPENVINO_SUPPRESS_DEPRECATED_END
 }
 
-ov::frontend::FrontEndManager& get_frontend_manager() {
-    static ov::frontend::FrontEndManager manager;
-    return manager;
-}
-
 std::vector<ov::Extension::Ptr> wrap_old_extensions(const std::vector<InferenceEngine::IExtensionPtr>& exts) {
     std::vector<ov::Extension::Ptr> extensions;
     for (const auto& ext : exts) {
@@ -488,7 +483,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
 #endif
 
     // Try to load with FrontEndManager
-    auto& manager = get_frontend_manager();
+    auto manager = ov::frontend::get_frontend_manager();
     ov::frontend::FrontEnd::Ptr FE;
     ov::frontend::InputModel::Ptr inputModel;
 
@@ -503,7 +498,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
         params.emplace_back(weights_path);
     }
 
-    FE = manager.load_by_model(params);
+    FE = manager->load_by_model(params);
     if (FE) {
         FE->add_extension(ov_exts);
         if (!exts.empty())
@@ -518,7 +513,7 @@ CNNNetwork details::ReadNetwork(const std::string& modelPath,
 
     const auto fileExt = modelPath.substr(modelPath.find_last_of(".") + 1);
     std::string FEs;
-    for (const auto& fe_name : manager.get_available_front_ends())
+    for (const auto& fe_name : manager->get_available_front_ends())
         FEs += fe_name + " ";
     IE_THROW(NetworkNotRead) << "Unable to read the model: " << modelPath
                              << " Please check that model format: " << fileExt
@@ -555,7 +550,7 @@ CNNNetwork details::ReadNetwork(const std::string& model,
 #endif  // ENABLE_IR_V7_READER
 
     // Try to load with FrontEndManager
-    auto& manager = get_frontend_manager();
+    auto manager = ov::frontend::get_frontend_manager();
     ov::frontend::FrontEnd::Ptr FE;
     ov::frontend::InputModel::Ptr inputModel;
 
@@ -567,7 +562,7 @@ CNNNetwork details::ReadNetwork(const std::string& model,
         params.emplace_back(weights_buffer);
     }
 
-    FE = manager.load_by_model(params);
+    FE = manager->load_by_model(params);
     if (FE) {
         FE->add_extension(ov_exts);
         if (!exts.empty())

src/inference/src/threading/ie_cpu_streams_executor.cpp

@@ -19,6 +19,7 @@
 
 #include "ie_parallel_custom_arena.hpp"
 #include "ie_system_conf.h"
+#include "threading/ie_executor_manager.hpp"
 #include "threading/ie_thread_affinity.hpp"
 #include "threading/ie_thread_local.hpp"
 
@@ -185,6 +186,7 @@ struct CPUStreamsExecutor::Impl {
           _streams([this] {
               return std::make_shared<Impl::Stream>(this);
           }) {
+        _exectorMgr = executorManager();
         auto numaNodes = getAvailableNUMANodes();
         if (_config._streams != 0) {
             std::copy_n(std::begin(numaNodes),
@@ -294,6 +296,7 @@ struct CPUStreamsExecutor::Impl {
     using StreamIdToCoreTypes = std::vector<std::pair<custom::core_type_id, int>>;
     StreamIdToCoreTypes total_streams_on_core_types;
 #endif
+    ExecutorManager::Ptr _exectorMgr;
 };
 
 int CPUStreamsExecutor::GetStreamId() {