Static methods to make empty arrays of a given type. (#38)

In the end, they weren't static methods on `Content` objects, but normal methods on `Type` objects. * [WIP] Static methods to make empty arrays of a given type. * Added Type::empty to all types. * Using it in EmptyArray::astype. We're done.
scikit-hep · Dec 28, 2019 · fbec76e · fbec76e
1 parent 5f73d23
commit fbec76e
Show file tree

Hide file tree

Showing 21 changed files with 141 additions and 14 deletions.
diff --git a/VERSION_INFO b/VERSION_INFO
@@ -1 +1 @@
-0.1.37
+0.1.38
diff --git a/include/awkward/type/ArrayType.h b/include/awkward/type/ArrayType.h
@@ -18,6 +18,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
     const std::shared_ptr<Type> type() const;
     int64_t length() const;

diff --git a/include/awkward/type/ListType.h b/include/awkward/type/ListType.h
@@ -18,6 +18,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
   const std::shared_ptr<Type> type() const;
 

diff --git a/include/awkward/type/OptionType.h b/include/awkward/type/OptionType.h
@@ -18,6 +18,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
   const std::shared_ptr<Type> type() const;
 

diff --git a/include/awkward/type/PrimitiveType.h b/include/awkward/type/PrimitiveType.h
@@ -33,6 +33,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
   const DType dtype() const;
 

diff --git a/include/awkward/type/RecordType.h b/include/awkward/type/RecordType.h
@@ -29,6 +29,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
     const std::shared_ptr<Type> field(int64_t fieldindex) const;
     const std::shared_ptr<Type> field(const std::string& key) const;

diff --git a/include/awkward/type/RegularType.h b/include/awkward/type/RegularType.h
@@ -20,6 +20,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
     const std::shared_ptr<Type> type() const;
     int64_t size() const;

diff --git a/include/awkward/type/Type.h b/include/awkward/type/Type.h
@@ -10,6 +10,8 @@
 #include "awkward/cpu-kernels/util.h"
 
 namespace awkward {
+  class Content;
+
   class Type {
   public:
     typedef std::map<std::string, std::string> Parameters;
@@ -27,6 +29,7 @@ namespace awkward {
     virtual const std::string key(int64_t fieldindex) const = 0;
     virtual bool haskey(const std::string& key) const = 0;
     virtual const std::vector<std::string> keys() const = 0;
+    virtual const std::shared_ptr<Content> empty() const = 0;
 
     const Parameters parameters() const;
     void setparameters(const Parameters& parameters);

diff --git a/include/awkward/type/UnionType.h b/include/awkward/type/UnionType.h
@@ -20,6 +20,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
     int64_t numtypes() const;
     const std::vector<std::shared_ptr<Type>> types() const;

diff --git a/include/awkward/type/UnknownType.h b/include/awkward/type/UnknownType.h
@@ -18,6 +18,7 @@ namespace awkward {
     const std::string key(int64_t fieldindex) const override;
     bool haskey(const std::string& key) const override;
     const std::vector<std::string> keys() const override;
+    const std::shared_ptr<Content> empty() const override;
 
   private:
   };

diff --git a/src/libawkward/array/EmptyArray.cpp b/src/libawkward/array/EmptyArray.cpp
@@ -40,7 +40,7 @@ namespace awkward {
   }
 
   const std::shared_ptr<Content> EmptyArray::astype(const std::shared_ptr<Type>& type) const {
-    return std::make_shared<EmptyArray>(id_, type);
+    return type.get()->empty();
   }
 
   const std::string EmptyArray::tostring_part(const std::string& indent, const std::string& pre, const std::string& post) const {

diff --git a/src/libawkward/type/ArrayType.cpp b/src/libawkward/type/ArrayType.cpp
@@ -55,6 +55,13 @@ namespace awkward {
     return type_.get()->keys();
   }
 
+  const std::shared_ptr<Content> ArrayType::empty() const {
+    if (length_ != 0) {
+      throw std::invalid_argument(std::string("ArrayType with length ") + std::to_string(length_) + std::string(" does not describe an empty array"));
+    }
+    return type_.get()->empty();
+  }
+
   int64_t ArrayType::length() const {
     return length_;
   }

diff --git a/src/libawkward/type/ListType.cpp b/src/libawkward/type/ListType.cpp
@@ -3,6 +3,7 @@
 #include <string>
 #include <sstream>
 
+#include "awkward/array/ListOffsetArray.h"
 #include "awkward/type/UnknownType.h"
 #include "awkward/type/OptionType.h"
 
@@ -65,6 +66,13 @@ namespace awkward {
     return type_.get()->keys();
   }
 
+  const std::shared_ptr<Content> ListType::empty() const {
+    Index64 offsets(1);
+    offsets.ptr().get()[0] = 0;
+    std::shared_ptr<Content> content = type_.get()->empty();
+    return std::make_shared<ListOffsetArray64>(Identity::none(), Type::none(), offsets, content);
+  }
+
   const std::shared_ptr<Type> ListType::type() const {
     return type_;
   }

diff --git a/src/libawkward/type/OptionType.cpp b/src/libawkward/type/OptionType.cpp
@@ -72,6 +72,11 @@ namespace awkward {
     return type_.get()->keys();
   }
 
+  const std::shared_ptr<Content> OptionType::empty() const {
+    std::shared_ptr<Content> content = type_.get()->empty();
+    throw std::runtime_error("OptionType::empty() needs OptionArray");
+  }
+
   const std::shared_ptr<Type> OptionType::type() const {
     std::shared_ptr<Type> out = type_;
     while (OptionType* t = dynamic_cast<OptionType*>(out.get())) {

diff --git a/src/libawkward/type/PrimitiveType.cpp b/src/libawkward/type/PrimitiveType.cpp
@@ -3,6 +3,7 @@
 #include <cassert>
 #include <sstream>
 
+#include "awkward/array/NumpyArray.h"
 #include "awkward/type/UnknownType.h"
 #include "awkward/type/OptionType.h"
 
@@ -80,6 +81,36 @@ namespace awkward {
     throw std::invalid_argument("type contains no Records");
   }
 
+  const std::shared_ptr<Content> PrimitiveType::empty() const {
+    std::shared_ptr<void> ptr(new uint8_t[0], awkward::util::array_deleter<uint8_t>());
+    std::vector<ssize_t> shape({ 0 });
+    std::vector<ssize_t> strides({ 0 });
+    ssize_t itemsize;
+    std::string format;
+    switch (dtype_) {
+      case boolean: itemsize = 1; format = "?"; break;
+      case int8:    itemsize = 1; format = "b"; break;
+      case uint8:   itemsize = 1; format = "B"; break;
+      case int16:   itemsize = 2; format = "h"; break;
+      case uint16:  itemsize = 2; format = "H"; break;
+#ifdef _MSC_VER
+      case int32:   itemsize = 4; format = "l"; break;
+      case uint32:  itemsize = 4; format = "L"; break;
+      case int64:   itemsize = 8; format = "q"; break;
+      case uint64:  itemsize = 8; format = "Q"; break;
+#else
+      case int32:   itemsize = 4; format = "i"; break;
+      case uint32:  itemsize = 4; format = "I"; break;
+      case int64:   itemsize = 8; format = "l"; break;
+      case uint64:  itemsize = 8; format = "L"; break;
+#endif
+      case float32: itemsize = 4; format = "f"; break;
+      case float64: itemsize = 8; format = "d"; break;
+      default: throw std::runtime_error(std::string("unexpected dtype: ") + std::to_string(dtype_));
+    }
+    return std::make_shared<NumpyArray>(Identity::none(), Type::none(), ptr, shape, strides, 0, itemsize, format);
+  }
+
   const PrimitiveType::DType PrimitiveType::dtype() const {
     return dtype_;
   }

diff --git a/src/libawkward/type/RecordType.cpp b/src/libawkward/type/RecordType.cpp
@@ -3,6 +3,7 @@
 #include <string>
 #include <sstream>
 
+#include "awkward/array/RecordArray.h"
 #include "awkward/type/UnknownType.h"
 #include "awkward/type/OptionType.h"
 #include "awkward/util.h"
@@ -160,6 +161,14 @@ namespace awkward {
     return util::keys(recordlookup_, numfields());
   }
 
+  const std::shared_ptr<Content> RecordType::empty() const {
+    std::vector<std::shared_ptr<Content>> contents;
+    for (auto type : types_) {
+      contents.push_back(type.get()->empty());
+    }
+    return std::make_shared<RecordArray>(Identity::none(), Type::none(), contents, recordlookup_);
+  }
+
   const std::shared_ptr<Type> RecordType::field(int64_t fieldindex) const {
     if (fieldindex >= numfields()) {
       throw std::invalid_argument(std::string("fieldindex ") + std::to_string(fieldindex) + std::string(" for record with only " + std::to_string(numfields()) + std::string(" fields")));

diff --git a/src/libawkward/type/RegularType.cpp b/src/libawkward/type/RegularType.cpp
@@ -3,6 +3,7 @@
 #include <string>
 #include <sstream>
 
+#include "awkward/array/RegularArray.h"
 #include "awkward/type/UnknownType.h"
 #include "awkward/type/OptionType.h"
 
@@ -66,6 +67,11 @@ namespace awkward {
     return type_.get()->keys();
   }
 
+  const std::shared_ptr<Content> RegularType::empty() const {
+    std::shared_ptr<Content> content = type_.get()->empty();
+    return std::make_shared<RegularArray>(Identity::none(), Type::none(), content, size_);
+  }
+
   const std::shared_ptr<Type> RegularType::type() const {
     return type_;
   }

diff --git a/src/libawkward/type/UnionType.cpp b/src/libawkward/type/UnionType.cpp
@@ -86,6 +86,14 @@ namespace awkward {
     return types_;
   }
 
+  const std::shared_ptr<Content> UnionType::empty() const {
+    std::vector<std::shared_ptr<Content>> contents;
+    for (auto type : types_) {
+      contents.push_back(type.get()->empty());
+    }
+    throw std::runtime_error("UnionType::empty() needs UnionArray");
+  }
+
   const std::shared_ptr<Type> UnionType::type(int64_t index) const {
     return types_[(size_t)index];
   }

diff --git a/src/libawkward/type/UnknownType.cpp b/src/libawkward/type/UnknownType.cpp
@@ -3,6 +3,7 @@
 #include <string>
 #include <sstream>
 
+#include "awkward/array/EmptyArray.h"
 #include "awkward/type/UnknownType.h"
 
 namespace awkward {
@@ -60,4 +61,8 @@ namespace awkward {
   const std::vector<std::string> UnknownType::keys() const {
     throw std::invalid_argument("type contains no Records");
   }
+
+  const std::shared_ptr<Content> UnknownType::empty() const {
+    return std::make_shared<EmptyArray>(Identity::none(), Type::none());
+  }
 }
diff --git a/src/pyawkward.cpp b/src/pyawkward.cpp
@@ -49,18 +49,6 @@ class pyobject_deleter {
   PyObject* pyobj_;
 };
 
-py::class_<ak::Type, std::shared_ptr<ak::Type>> make_Type(py::handle m, std::string name) {
-  return (py::class_<ak::Type, std::shared_ptr<ak::Type>>(m, name.c_str())
-      .def("__eq__", [](std::shared_ptr<ak::Type> self, std::shared_ptr<ak::Type> other) -> bool {
-        return self.get()->equal(other, true);
-      })
-      .def("__ne__", [](std::shared_ptr<ak::Type> self, std::shared_ptr<ak::Type> other) -> bool {
-        return !self.get()->equal(other, true);
-      })
-  );
-}
-
-
 py::object box(std::shared_ptr<ak::Type> t) {
   if (ak::ArrayType* raw = dynamic_cast<ak::ArrayType*>(t.get())) {
     return py::cast(*raw);
@@ -799,6 +787,17 @@ void setparameters(T& self, py::object parameters) {
   self.setparameters(dict2parameters(parameters));
 }
 
+py::class_<ak::Type, std::shared_ptr<ak::Type>> make_Type(py::handle m, std::string name) {
+  return (py::class_<ak::Type, std::shared_ptr<ak::Type>>(m, name.c_str())
+      .def("__eq__", [](std::shared_ptr<ak::Type> self, std::shared_ptr<ak::Type> other) -> bool {
+        return self.get()->equal(other, true);
+      })
+      .def("__ne__", [](std::shared_ptr<ak::Type> self, std::shared_ptr<ak::Type> other) -> bool {
+        return !self.get()->equal(other, true);
+      })
+  );
+}
+
 template <typename T>
 py::class_<T, ak::Type> type_methods(py::class_<T, std::shared_ptr<T>, ak::Type>& x) {
   return x.def("__repr__", &T::tostring)
@@ -816,6 +815,7 @@ py::class_<T, ak::Type> type_methods(py::class_<T, std::shared_ptr<T>, ak::Type>
           .def("key", &T::key)
           .def("haskey", &T::haskey)
           .def("keys", &T::keys)
+          .def("empty", &T::empty)
   ;
 }
 

diff --git a/tests/test_PR038_emptyarray_astype.py b/tests/test_PR038_emptyarray_astype.py
@@ -0,0 +1,37 @@
+# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE
+
+import sys
+
+import pytest
+import numpy
+
+import awkward1
+
+def test_typeempty():
+    assert numpy.asarray(awkward1.layout.PrimitiveType("bool").empty()).dtype == numpy.dtype("bool")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("int8").empty()).dtype == numpy.dtype("int8")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("int16").empty()).dtype == numpy.dtype("int16")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("int32").empty()).dtype == numpy.dtype("int32")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("int64").empty()).dtype == numpy.dtype("int64")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("uint8").empty()).dtype == numpy.dtype("uint8")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("uint16").empty()).dtype == numpy.dtype("uint16")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("uint32").empty()).dtype == numpy.dtype("uint32")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("uint64").empty()).dtype == numpy.dtype("uint64")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("float32").empty()).dtype == numpy.dtype("float32")
+    assert numpy.asarray(awkward1.layout.PrimitiveType("float64").empty()).dtype == numpy.dtype("float64")
+    assert type(awkward1.layout.UnknownType().empty()) is awkward1.layout.EmptyArray
+    assert type(awkward1.layout.ArrayType(awkward1.layout.UnknownType(), 0).empty()) is awkward1.layout.EmptyArray
+    assert type(awkward1.layout.RegularType(awkward1.layout.UnknownType(), 5).empty()) is awkward1.layout.RegularArray
+    assert type(awkward1.layout.ListType(awkward1.layout.UnknownType()).empty()) is awkward1.layout.ListOffsetArray64
+    array = awkward1.layout.RecordType({"one": awkward1.layout.PrimitiveType("float64"), "two": awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64"))}).empty()
+    assert type(array) is awkward1.layout.RecordArray
+    assert type(array["one"]) is awkward1.layout.NumpyArray
+    assert numpy.asarray(array["one"]).dtype == numpy.dtype("float64")
+    assert type(array["two"]) is awkward1.layout.ListOffsetArray64
+
+def test_astype():
+    empty = awkward1.layout.EmptyArray()
+    assert numpy.asarray(empty.astype(awkward1.layout.PrimitiveType("bool"))).dtype == numpy.dtype("bool")
+    assert numpy.asarray(empty.astype(awkward1.layout.PrimitiveType("uint8"))).dtype == numpy.dtype("uint8")
+    assert numpy.asarray(empty.astype(awkward1.layout.PrimitiveType("float64"))).dtype == numpy.dtype("float64")
+    assert type(empty.astype(awkward1.layout.ListType(awkward1.layout.PrimitiveType("float64")))) is awkward1.layout.ListOffsetArray64