[GPU] Global Buffer manager and optimization

nntrainer/cl_buffer_manager.cpp

@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 Debadri Samaddar <[email protected]>
+ *
+ * @file    cl_buffer_manager.cpp
+ * @date    01 Dec 2024
+ * @see     https://github.com/nnstreamer/nntrainer
+ * @author  Debadri Samaddar <[email protected]>
+ * @bug     No known bugs except for NYI items
+ * @brief   This file contains global Buffer objects and manages them
+ */
+
+#include <cl_buffer_manager.h>
+
+namespace nntrainer {
+
+ClBufferManager &ClBufferManager::getInstance() {
+  static ClBufferManager instance;
+  return instance;
+}
+
+// to-do: Implementation to be updated with array of Buffer objects if required
+// fp16 Buffer objects to be added in future
+void ClBufferManager::initBuffers() {
+  inBufferA = new opencl::Buffer(context_inst_, buffer_size_bytes, true);
+  inBufferB = new opencl::Buffer(context_inst_, buffer_size_bytes, true);
+  inBufferC = new opencl::Buffer(context_inst_, buffer_size_bytes, true);
+  outBufferA = new opencl::Buffer(context_inst_, buffer_size_bytes, false);
+  outBufferB = new opencl::Buffer(context_inst_, buffer_size_bytes, false);
+  ml_logi("ClBufferManager: Buffers initialized");
+}
+
+ClBufferManager::~ClBufferManager() {
+  delete inBufferA;
+  delete inBufferB;
+  delete inBufferC;
+  delete outBufferA;
+  delete outBufferB;
+  ml_logi("ClBufferManager: Buffers destroyed");
+}
+
+} // namespace nntrainer

nntrainer/cl_buffer_manager.h

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 Debadri Samaddar <[email protected]>
+ *
+ * @file    cl_buffer_manager.h
+ * @date    01 Dec 2024
+ * @see     https://github.com/nnstreamer/nntrainer
+ * @author  Debadri Samaddar <[email protected]>
+ * @bug     No known bugs except for NYI items
+ * @brief   This file contains global Buffer objects and manages them
+ */
+
+#ifndef __CL_BUFFER_MANAGER_H__
+#define __CL_BUFFER_MANAGER_H__
+
+#include <string>
+
+#include <opencl_buffer.h>
+#include <opencl_context_manager.h>
+
+#include <nntrainer_log.h>
+
+namespace nntrainer {
+
+/**
+ * @class ClBufferManager contains Buffer object management
+ * @brief Support for Buffer management
+ */
+
+class ClBufferManager {
+
+private:
+  /**
+   * @brief Private constructor to prevent object creation
+   *
+   */
+  ClBufferManager() :
+    inBufferA(nullptr),
+    inBufferB(nullptr),
+    inBufferC(nullptr),
+    outBufferA(nullptr),
+    outBufferB(nullptr){};
+
+  /**
+   * @brief OpenCl context global instance
+   *
+   */
+  opencl::ContextManager &context_inst_ = opencl::ContextManager::GetInstance();
+
+  /**
+   * @brief Buffer size in bytes preset (256 mebibytes)
+   */
+  const size_t buffer_size_bytes = 8192 * 8192 * sizeof(float);
+
+  opencl::Buffer *inBufferA;
+  opencl::Buffer *inBufferB;
+  opencl::Buffer *inBufferC;
+  opencl::Buffer *outBufferA;
+  opencl::Buffer *outBufferB;
+
+public:
+  /**
+   * @brief Get Global ClBufferManager.
+   *
+   * @return ClBufferManager&
+   */
+  static ClBufferManager &getInstance();
+
+  /**
+   * @brief Initialize Buffer objects.
+   */
+  void initBuffers();
+
+  /**
+   * @brief Get read only inBufferA.
+   * @return opencl::Buffer* or nullptr if initBuffers() is not called
+   */
+  opencl::Buffer *getInBufferA() { return inBufferA; }
+
+  /**
+   * @brief Get read only inBufferB.
+   * @return opencl::Buffer* or nullptr if initBuffers() is not called
+   */
+  opencl::Buffer *getInBufferB() { return inBufferB; }
+
+  /**
+   * @brief Get read only inBufferC.
+   * @return opencl::Buffer* or nullptr if initBuffers() is not called
+   */
+  opencl::Buffer *getInBufferC() { return inBufferC; }
+
+  /**
+   * @brief Get read-write outBufferA.
+   * @return opencl::Buffer* or nullptr if initBuffers() is not called
+   */
+  opencl::Buffer *getOutBufferA() { return outBufferA; }
+
+  /**
+   * @brief Get read-write outBufferB.
+   * @return opencl::Buffer* or nullptr if initBuffers() is not called
+   */
+  opencl::Buffer *getOutBufferB() { return outBufferB; }
+
+  /**
+   * @brief Destroy Buffer pointers.
+   *
+   */
+  ~ClBufferManager();
+};
+} // namespace nntrainer
+
+#endif /* __CL_BUFFER_MANAGER_H__ */

nntrainer/cl_context.h

@@ -29,6 +29,7 @@
 #include <layer.h>
 #include <layer_devel.h>
 
+#include <cl_buffer_manager.h>
 #include <opencl_command_queue_manager.h>
 #include <opencl_context_manager.h>
 #include <opencl_kernel.h>
@@ -79,12 +80,14 @@ class ClContext {
 
   template <typename... Ts> using FactoryMap = std::tuple<IndexType<Ts>...>;
 
-  // getting static instance of commandqueue and opencl context
+  // getting static instance of commandqueue, opencl context and buffermanager
   opencl::CommandQueueManager &command_queue_inst_ =
     opencl::CommandQueueManager::GetInstance();
 
   opencl::ContextManager &context_inst_ = opencl::ContextManager::GetInstance();
 
+  ClBufferManager &clbuffInstance = ClBufferManager::getInstance();
+
   /**
    * @brief   Default constructor
    */
@@ -272,6 +275,9 @@ class ClContext {
 
     // getContext() called inside createCommandQueue which creates clContext
     bool result = command_queue_inst_.CreateCommandQueue();
+    // initialize device buffers
+    clbuffInstance.initBuffers();
+
     cl_initialized = result;
     return cl_initialized;
   };

nntrainer/meson.build

@@ -64,7 +64,9 @@ nntrainer_common_sources = [
 
 if get_option('enable-opencl')
   nntrainer_headers += meson.current_source_dir() / 'cl_context.h'
+  nntrainer_headers += meson.current_source_dir() / 'cl_buffer_manager.h'
   nntrainer_common_sources += 'cl_context.cpp'
+  nntrainer_common_sources += 'cl_buffer_manager.cpp'
 endif
 
 foreach s : nntrainer_common_sources

nntrainer/opencl/opencl_buffer.cpp

@@ -27,7 +27,7 @@ namespace nntrainer::opencl {
  * @param read_only flag
  * @param data data for the buffer
  */
-Buffer::Buffer(ContextManager &context_manager, int size_in_bytes,
+Buffer::Buffer(ContextManager &context_manager, size_t size_in_bytes,
                bool read_only, void *data) {
   cl_context context = context_manager.GetContext();
   cl_mem_flags flags = read_only ? CL_MEM_READ_ONLY : CL_MEM_READ_WRITE;
@@ -94,6 +94,20 @@ bool Buffer::WriteData(CommandQueueManager &command_queue_inst,
   return command_queue_inst.EnqueueWriteBuffer(mem_buf_, size_, data);
 }
 
+bool Buffer::WriteDataRegion(CommandQueueManager &command_queue_inst,
+                             size_t size_in_bytes, const void *data,
+                             size_t host_origin_offset,
+                             size_t buffer_origin_offset) {
+  if (size_in_bytes > size_) {
+    ml_loge("Failed to write buffer region. Region size(%lu bytes) greater "
+            "than buffer size(%lu bytes).",
+            size_in_bytes, size_);
+    return false;
+  }
+  return command_queue_inst.EnqueueWriteBufferRegion(
+    mem_buf_, size_in_bytes, data, host_origin_offset, buffer_origin_offset);
+}
+
 /**
  * @brief reading data from the buffer
  *
@@ -105,6 +119,20 @@ bool Buffer::ReadData(CommandQueueManager &command_queue_inst, void *data) {
   return command_queue_inst.EnqueueReadBuffer(mem_buf_, size_, data);
 }
 
+bool Buffer::ReadDataRegion(CommandQueueManager &command_queue_inst,
+                            size_t size_in_bytes, void *data,
+                            size_t host_origin_offset,
+                            size_t buffer_origin_offset) {
+  if (size_in_bytes > size_) {
+    ml_loge("Failed to read from buffer region. Region size(%lu bytes) greater "
+            "than buffer size(%lu bytes).",
+            size_in_bytes, size_);
+    return false;
+  }
+  return command_queue_inst.EnqueueReadBufferRegion(
+    mem_buf_, size_in_bytes, data, host_origin_offset, buffer_origin_offset);
+}
+
 void *Buffer::MapBuffer(CommandQueueManager &command_queue_inst,
                         size_t offset_in_bytes, size_t size_in_bytes,
                         bool read_only, bool async) {

nntrainer/opencl/opencl_buffer.h

@@ -54,8 +54,8 @@ class Buffer {
    * @param read_only flag
    * @param data data for the buffer
    */
-  Buffer(ContextManager &context_manager, int size_in_bytes, bool read_only,
-         void *data);
+  Buffer(ContextManager &context_manager, size_t size_in_bytes, bool read_only,
+         void *data = nullptr);
 
   /**
    * @brief Move constructor for buffer by deleting the previous buffer
@@ -106,6 +106,21 @@ class Buffer {
    */
   bool WriteData(CommandQueueManager &command_queue_inst, const void *data);
 
+  /**
+   * @brief writing data to a buffer region
+   *
+   * @param command_queue_inst reference of command queue instance
+   * @param size_in_bytes size of region
+   * @param data pointer of region
+   * @param host_origin_offset offset in the host memory region
+   * @param buffer_origin_offset offset in the buffer memory region
+   * @return true if successful write or false otherwise
+   */
+  bool WriteDataRegion(CommandQueueManager &command_queue_inst,
+                       size_t size_in_bytes, const void *data,
+                       size_t host_origin_offset = 0,
+                       size_t buffer_origin_offset = 0);
+
   /**
    * @brief reading data from the buffer
    *
@@ -115,6 +130,21 @@ class Buffer {
    */
   bool ReadData(CommandQueueManager &command_queue_inst, void *data);
 
+  /**
+   * @brief Reading data from a buffer region
+   *
+   * @param command_queue_inst reference of command queue instance
+   * @param size_in_bytes size of region
+   * @param data pointer of region
+   * @param host_origin_offset offset in the host memory region
+   * @param buffer_origin_offset offset in the buffer memory region
+   * @return true if successful write or false otherwise
+   */
+  bool ReadDataRegion(CommandQueueManager &command_queue_inst,
+                      size_t size_in_bytes, void *data,
+                      size_t host_origin_offset = 0,
+                      size_t buffer_origin_offset = 0);
+
   /**
    * @brief Mapping buffer to host memory
    *

nntrainer/opencl/opencl_command_queue_manager.cpp

@@ -131,6 +131,41 @@ bool CommandQueueManager::EnqueueReadBuffer(cl_mem buffer, size_t size_in_bytes,
   return true;
 }
 
+bool CommandQueueManager::EnqueueReadBufferRegion(
+  cl_mem buffer, size_t size_in_bytes, void *data, size_t host_origin_offset,
+  size_t buffer_origin_offset, bool async) {
+
+  // managing synchronization
+  const cl_bool blocking = async ? CL_FALSE : CL_TRUE;
+
+  // (x, y, z) offset in the memory region associated with buffer
+  const size_t buffer_origin[] = {buffer_origin_offset, 0, 0};
+  // (x, y, z) offset in the memory region associated with host
+  const size_t host_origin[] = {host_origin_offset, 0, 0};
+  // region defines the (width in bytes, height in rows, depth in slices)
+  const size_t region[] = {size_in_bytes, 1, 1};
+  // length of each row in bytes
+  size_t row_pitch = region[0];
+  // length of each 2D slice in bytes
+  size_t slice_pitch = region[0] * region[1];
+
+  // Buffer and host data are interpreted as 1D in this case
+  // hence row and slice pitch are same for both
+  cl_int error_code = clEnqueueReadBufferRect(
+    command_queue_, buffer, blocking, buffer_origin, host_origin, region,
+    row_pitch, slice_pitch, row_pitch, slice_pitch, data, 0, nullptr, nullptr);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to write data region to GPU (clEnqueueWriteBufferRect). "
+            "OpenCL error "
+            "code: %d",
+            error_code);
+    return false;
+  }
+
+  return true;
+}
+
 /**
  * @brief Writing buffer object. Used from Buffer class
  *
@@ -150,6 +185,7 @@ bool CommandQueueManager::EnqueueWriteBuffer(cl_mem buffer,
   auto error_code =
     clEnqueueWriteBuffer(command_queue_, buffer, blocking, 0, size_in_bytes,
                          data, 0, nullptr, nullptr);
+
   if (error_code != CL_SUCCESS) {
     ml_loge("Failed to upload data to GPU (clEnqueueWriteBuffer). OpenCL error "
             "code: %d",
@@ -160,6 +196,41 @@ bool CommandQueueManager::EnqueueWriteBuffer(cl_mem buffer,
   return true;
 }
 
+bool CommandQueueManager::EnqueueWriteBufferRegion(
+  cl_mem buffer, size_t size_in_bytes, const void *data,
+  size_t host_origin_offset, size_t buffer_origin_offset, bool async) {
+
+  // managing synchronization
+  const cl_bool blocking = async ? CL_FALSE : CL_TRUE;
+
+  // (x, y, z) offset in the memory region associated with buffer
+  const size_t buffer_origin[] = {buffer_origin_offset, 0, 0};
+  // (x, y, z) offset in the memory region associated with host
+  const size_t host_origin[] = {host_origin_offset, 0, 0};
+  // region defines the (width in bytes, height in rows, depth in slices)
+  const size_t region[] = {size_in_bytes, 1, 1};
+  // length of each row in bytes
+  size_t row_pitch = region[0];
+  // length of each 2D slice in bytes
+  size_t slice_pitch = region[0] * region[1];
+
+  // Buffer and host data are interpreted as 1D in this case
+  // hence row and slice pitch are same for both
+  cl_int error_code = clEnqueueWriteBufferRect(
+    command_queue_, buffer, blocking, buffer_origin, host_origin, region,
+    row_pitch, slice_pitch, row_pitch, slice_pitch, data, 0, nullptr, nullptr);
+
+  if (error_code != CL_SUCCESS) {
+    ml_loge("Failed to write data region to GPU (clEnqueueWriteBufferRect). "
+            "OpenCL error "
+            "code: %d",
+            error_code);
+    return false;
+  }
+
+  return true;
+}
+
 /**
  * @brief Mapping a region of a buffer object into the host address space
  *