feat: Implement Indexed builder (#2883)

* feat: create exact copy of IndexedOption

* feat: remove handling of invalid values from copy of IndexedOption

* doc: support also uint32

* feat: implement is_valid checking index and content lengths

* feat: renamed append and extend to append_index and extend_index

* test: implemented test_Indexed, should fail

* test: fix unit test, check CI passes

* fix: remove leftover cout

* test: removed the test_Indexed_as_IndexedOption test

header-only/layout-builder/awkward/LayoutBuilder.h

@@ -1085,6 +1085,193 @@ namespace awkward {
     };
 
 
+    /// @class Indexed
+    ///
+    /// @brief Builds an IndexedArray which consists of an `index` buffer.
+    ///
+    /// The index values can be 64-bit signed integers `int64`, 32-bit signed
+    /// integers `int32` or 32-bit unsigned integers `uint32`.
+    ///
+    /// @tparam PRIMITIVE The type of `index` buffer.
+    /// @tparam BUILDER The type of builder content.
+    template <typename PRIMITIVE, typename BUILDER>
+    class Indexed {
+    public:
+      /// @brief Creates a new Indexed layout builder by allocating a new `index`
+      /// buffer, using `AWKWARD_LAYOUTBUILDER_DEFAULT_OPTIONS` for initializing the buffer.
+      Indexed()
+          : index_(
+                awkward::GrowableBuffer<PRIMITIVE>(AWKWARD_LAYOUTBUILDER_DEFAULT_OPTIONS)) {
+        size_t id = 0;
+        set_id(id);
+      }
+
+      /// @brief Creates a new Indexed layout builder by allocating a new `index`
+      /// buffer, taking `options` from {@link BuilderOptions BuilderOptions}
+      /// for initializing the buffer.
+      ///
+      /// @param options Initial size configuration of a buffer.
+      Indexed(const awkward::BuilderOptions& options)
+          : index_(awkward::GrowableBuffer<PRIMITIVE>(options)) {
+        size_t id = 0;
+        set_id(id);
+      }
+
+      /// @brief Returns the reference to the builder content.
+      BUILDER&
+      content() noexcept {
+        return content_;
+      }
+
+      /// @brief Inserts the last valid index in the `index` buffer and
+      /// returns the reference to the builder content.
+      BUILDER&
+      append_index() noexcept {
+        index_.append(content_.length());
+        return content_;
+      }
+
+      /// @brief Inserts `size` number indices in the `index` buffer
+      /// and returns the reference to the builder content.
+      ///
+      /// Just an interface; not actually faster than calling append many times.
+      BUILDER&
+      extend_index(size_t size) noexcept {
+        size_t start = content_.length();
+        size_t stop = start + size;
+        for (size_t i = start; i < stop; i++) {
+          index_.append(i);
+        }
+        return content_;
+      }
+
+      /// @brief Parameters for the builder form.
+      const std::string&
+      parameters() const noexcept {
+        return parameters_;
+      }
+
+      /// @brief Sets the form parameters.
+      void
+      set_parameters(std::string parameter) noexcept {
+        parameters_ = parameter;
+      }
+
+      /// @brief Assigns a unique ID to each node.
+      void
+      set_id(size_t& id) noexcept {
+        id_ = id;
+        id++;
+        content_.set_id(id);
+      }
+
+      /// @brief Discards the accumulated index and clears the content
+      /// of the builder.
+      void
+      clear() noexcept {
+        index_.clear();
+        content_.clear();
+      }
+
+      /// @brief Current length of the `index` buffer.
+      size_t
+      length() const noexcept {
+        return index_.length();
+      }
+
+      /// @brief Retrieves the names and sizes (in bytes) of the buffers used
+      /// in the builder and its contents.
+      void
+      buffer_nbytes(std::map<std::string, size_t>& names_nbytes) const
+          noexcept {
+        names_nbytes["node" + std::to_string(id_) + "-index"] = index_.nbytes();
+        content_.buffer_nbytes(names_nbytes);
+      }
+
+      /// @brief Checks for validity and consistency.
+      bool
+      is_valid(std::string& error) const noexcept {
+        if (content_.length() != index_.length()) {
+          std::stringstream out;
+          out << "Indexed node" << id_ << " has content length "
+              << content_.length() << " but index has length " << index_.length()
+              << "\n";
+          error.append(out.str());
+
+          return false;
+        } else {
+          return content_.is_valid(error);
+        }
+      }
+
+      /// @brief Copies and concatenates all the accumulated data in each of the
+      /// buffers of the builder and its contents to user-defined pointers.
+      ///
+      /// Used to fill the buffers map by allocating it with user-defined pointers
+      /// using the same names and sizes (in bytes) obtained from #buffer_nbytes.
+      void
+      to_buffers(std::map<std::string, void*>& buffers) const noexcept {
+        index_.concatenate(reinterpret_cast<PRIMITIVE*>(
+            buffers["node" + std::to_string(id_) + "-index"]));
+        content_.to_buffers(buffers);
+      }
+
+      /// @brief Copies and concatenates the accumulated data in the builder buffer to
+      /// a user-defined pointer if the given node name matches with the node associated
+      /// with the builder; otherwise, it searches the builder contents to locate a
+      /// matching node.
+      void
+      to_buffer(void* buffer, const char* name) const noexcept {
+        if (std::string(name) == std::string("node" + std::to_string(id_) + "-index")) {
+          index_.concatenate(reinterpret_cast<PRIMITIVE*>(buffer));
+        }
+        content_.to_buffer(buffer, name);
+      }
+
+      /// @brief Copies and concatenates all the accumulated data in the builder
+      /// to a map of user-allocated buffers.
+      ///
+      /// The map keys and the buffer sizes are obtained from #buffer_nbytes
+      void
+      to_char_buffers(std::map<std::string, uint8_t*>& buffers) const noexcept {
+        index_.concatenate(reinterpret_cast<PRIMITIVE*>(
+            buffers["node" + std::to_string(id_) + "-index"]));
+        content_.to_char_buffers(buffers);
+      }
+
+      /// @brief Generates a unique description of the builder and its
+      /// contents in the form of a JSON-like string.
+      std::string
+      form() const noexcept {
+        std::stringstream form_key;
+        form_key << "node" << id_;
+        std::string params("");
+        if (parameters_ == "") {
+        } else {
+          params = std::string(", \"parameters\": { " + parameters_ + " }");
+        }
+        return "{ \"class\": \"IndexedArray\", \"index\": \"" +
+               type_to_numpy_like<PRIMITIVE>() +
+               "\", \"content\": " + content_.form() + params +
+               ", \"form_key\": \"" + form_key.str() + "\" }";
+      }
+
+    private:
+      /// @brief Buffer of `PRIMITIVE` type.
+      ///
+      /// It specifies the index of each element.
+      GrowableBuffer<PRIMITIVE> index_;
+
+      /// @brief The content of the IndexedOptionArray.
+      BUILDER content_;
+
+      /// @brief Form parameters.
+      std::string parameters_;
+
+      /// @brief Unique form ID.
+      size_t id_;
+    };
+
     /// @class IndexedOption
     ///
     /// @brief Builds an IndexedOptionArray which consists of an `index` buffer.

header-only/tests/test_1494-layout-builder.cpp

@@ -58,6 +58,9 @@ using TupleBuilder = awkward::LayoutBuilder::Tuple<BUILDERS...>;
 template <unsigned SIZE, class BUILDER>
 using RegularBuilder = awkward::LayoutBuilder::Regular<SIZE, BUILDER>;
 
+template<class PRIMITIVE, class BUILDER>
+using IndexedBuilder = awkward::LayoutBuilder::Indexed<PRIMITIVE, BUILDER>;
+
 template<class PRIMITIVE, class BUILDER>
 using IndexedOptionBuilder = awkward::LayoutBuilder::IndexedOption<PRIMITIVE, BUILDER>;
 
@@ -1210,19 +1213,19 @@ test_Regular_size0() {
 }
 
 void
-test_Indexed_as_IndexedOption() {
-  IndexedOptionBuilder<uint32_t, NumpyBuilder<double>> builder;
+test_Indexed() {
+  IndexedBuilder<uint32_t, NumpyBuilder<double>> builder;
   assert(builder.length() == 0);
 
-  auto& subbuilder = builder.append_valid();
+  auto& subbuilder = builder.append_index();
   subbuilder.append(1.1);
 
-  builder.append_valid();
+  builder.append_index();
   subbuilder.append(2.2);
 
   double data[3] = {3.3, 4.4, 5.5};
 
-  builder.extend_valid(3);
+  builder.extend_index(3);
   subbuilder.extend(data, 3);
 
   // [1.1, 2.2, 3.3, 4.4, 5.5]
@@ -1248,7 +1251,7 @@ test_Indexed_as_IndexedOption() {
 
   assert(builder.form() ==
   "{ "
-      "\"class\": \"IndexedOptionArray\", "
+      "\"class\": \"IndexedArray\", "
       "\"index\": \"u32\", "
       "\"content\": { "
           "\"class\": \"NumpyArray\", "
@@ -1874,7 +1877,7 @@ int main(int /* argc */, char ** /* argv */) {
   test_Tuple_Numpy_ListOffset();
   test_Regular();
   test_Regular_size0();
-  test_Indexed_as_IndexedOption();
+  test_Indexed();
   test_IndexedOption();
   test_IndexedOption_Record();
   test_Unmasked();