diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..c4d549adce --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,57 @@ +version: 2 + +jobs: + linux-aarch64: + working_directory: ~/linux-aarch64-wheels + machine: + image: ubuntu-2204:current + # resource_class is what tells CircleCI to use an ARM worker for native arm builds + # https://circleci.com/product/features/resource-classes/ + # https://circleci.com/docs/using-arm/ + resource_class: arm.large + steps: + - checkout + - run: + name: Install build dependencies + command: | + sudo apt update + sudo apt install cmake g++ gfortran libfabric-dev libopenmpi-dev libhdf5-openmpi-dev hdf5-tools pkgconf python3 python3-setuptools + sudo .github/workflows/dependencies/install_spack + python3 -m pip install -U pip + python3 -m pip install -U packaging setuptools wheel + python3 -m pip install -U numpy + python3 -m pip install -U mpi4py + python3 -m pip install -U pandas + python3 -m pip install -U dask + python3 -m pip install -U pyarrow + + eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/) + spack install + + share/openPMD/download_samples.sh build + - run: + name: Build openPMD-api + command: | + eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/) + + export CXXFLAGS="-DPYBIND11_DETAILED_ERROR_MESSAGES=1" + + cmake -S . -B build \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=ON \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_ADIOS2=ON \ + -DopenPMD_USE_INVASIVE_TESTS=ON \ + -DPython_EXECUTABLE=$(which python3) + cmake --build build --parallel 4 + - run: + name: Test openPMD-api + command: | + eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/) + ctest --test-dir build --output-on-failure + +workflows: + version: 2 + all-tests: + jobs: + - linux-aarch64 diff --git a/.github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/spack.yaml b/.github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/spack.yaml new file mode 100644 index 0000000000..7f051faa79 --- /dev/null +++ b/.github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/spack.yaml @@ -0,0 +1,79 @@ +# This is a Spack environment file. +# +# Activating and installing this environment will provide all dependencies +# that are needed for full-feature development. +# https//spack.readthedocs.io/en/latest/environments.html#anonymous-environments +# +spack: + specs: + - adios2 + - hdf5 + - openmpi + + packages: + adios2: + variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 + cmake: + externals: + - spec: cmake@3.22.1 + prefix: /usr + buildable: False + libfabric: + externals: + - spec: libfabric@1.11.0 + prefix: /usr + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.2 + prefix: /usr + buildable: False + perl: + externals: + - spec: perl@5.34.0 + prefix: /usr + buildable: False + pkgconf: + externals: + - spec: pkgconf@1.8.0 + prefix: /usr + buildable: False + python: + externals: + - spec: python@3.11.5 + prefix: /usr + buildable: False + hdf5: + externals: + - spec: hdf5@1.10.7 + prefix: /usr + buildable: False + all: + target: [aarch64] + variants: ~fortran + compiler: [gcc@11.4.0] + + compilers: + - compiler: + environment: {} + extra_rpaths: [] + flags: {} + modules: [] + operating_system: ubuntu22.04 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + spec: gcc@11.4.0 + target: aarch64 + + # arm.large with 4 vCPU cores + # https://circleci.com/product/features/resource-classes/ + # https://circleci.com/docs/using-arm/ + config: + build_jobs: 4 + + # https://cache.spack.io + mirrors: + E4S: https://cache.e4s.io diff --git a/include/openPMD/binding/python/Numpy.hpp b/include/openPMD/binding/python/Numpy.hpp index 971aa8613f..49af1fca05 100644 --- a/include/openPMD/binding/python/Numpy.hpp +++ b/include/openPMD/binding/python/Numpy.hpp @@ -105,7 +105,14 @@ inline Datatype dtype_from_bufferformat(std::string const &fmt) if (fmt.find("?") != std::string::npos) return DT::BOOL; else if (fmt.find("b") != std::string::npos) - return DT::CHAR; + if constexpr (std::is_signed_v) + { + return Datatype::CHAR; + } + else + { + return Datatype::SCHAR; + } else if (fmt.find("h") != std::string::npos) return DT::SHORT; else if (fmt.find("i") != std::string::npos) @@ -115,7 +122,14 @@ inline Datatype dtype_from_bufferformat(std::string const &fmt) else if (fmt.find("q") != std::string::npos) return DT::LONGLONG; else if (fmt.find("B") != std::string::npos) - return DT::UCHAR; + if constexpr (std::is_unsigned_v) + { + return Datatype::CHAR; + } + else + { + return Datatype::UCHAR; + } else if (fmt.find("H") != std::string::npos) return DT::USHORT; else if (fmt.find("I") != std::string::npos) @@ -150,10 +164,18 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt) { case DT::CHAR: case DT::VEC_CHAR: - case DT::SCHAR: - case DT::VEC_SCHAR: case DT::STRING: case DT::VEC_STRING: + if constexpr (std::is_signed_v) + { + return pybind11::dtype("b"); + } + else + { + return pybind11::dtype("B"); + } + case DT::SCHAR: + case DT::VEC_SCHAR: return pybind11::dtype("b"); break; case DT::UCHAR: diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index a3fa63cdca..b3ce7885ea 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -26,11 +26,17 @@ #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" +#include + #include #include #include +#include +#include #include +#include #include +#include #include using PyAttributeKeys = std::vector; @@ -262,29 +268,24 @@ bool setAttributeFromBufferInfo( } } -struct SetAttributeFromObject +namespace detail { - static constexpr char const *errorMsg = "Attributable.set_attribute()"; - - template - static bool - call(Attributable &attr, std::string const &key, py::object &obj) +template +bool setAttributeFromObject_default( + Attributable &attr, std::string const &key, py::object &obj) +{ + if (std::string(py::str(obj.get_type())) == "") { - if (std::string(py::str(obj.get_type())) == "") - { - using ListType = std::vector; - return attr.setAttribute(key, obj.cast()); - } - else - { - return attr.setAttribute( - key, obj.cast()); - } + using ListType = std::vector; + return attr.setAttribute(key, obj.cast()); } -}; + else + { + return attr.setAttribute(key, obj.cast()); + } +} -template <> -bool SetAttributeFromObject::call( +bool setAttributeFromObject_double( Attributable &attr, std::string const &key, py::object &obj) { if (std::string(py::str(obj.get_type())) == "") @@ -309,39 +310,149 @@ bool SetAttributeFromObject::call( } } -template <> -bool SetAttributeFromObject::call( +bool setAttributeFromObject_bool( Attributable &attr, std::string const &key, py::object &obj) { return attr.setAttribute(key, obj.cast()); } +template > +struct char_to_explicit_char; + +template <> +struct char_to_explicit_char +{ + using type = signed char; + using opposite_type = unsigned char; +}; + template <> -bool SetAttributeFromObject::call( +struct char_to_explicit_char +{ + using type = unsigned char; + using opposite_type = signed char; +}; + +template +std::optional tryCast(py::object const &obj) +{ + try + { + return obj.cast(); + } + catch (py::cast_error const &) + { + return std::nullopt; + } + catch (py::value_error const &err) + { + return std::nullopt; + } +} + +template +bool setAttributeFromObject_char( Attributable &attr, std::string const &key, py::object &obj) { - if (std::string(py::str(obj.get_type())) == "") + using explicit_char_type = std::conditional_t< + std::is_same_v, + typename char_to_explicit_char<>::type, + Char_t>; + using ListChar = std::vector; + using ListString = std::vector; + + if (auto casted_char = tryCast(obj); casted_char.has_value()) { - using ListChar = std::vector; - using ListString = std::vector; - try - { - return attr.setAttribute(key, obj.cast()); - } - catch (const py::cast_error &) - { - return attr.setAttribute(key, obj.cast()); - } + return attr.setAttribute(key, *casted_char); } - else if (std::string(py::str(obj.get_type())) == "") + // This must come after tryCast + // because tryCast implicitly covers chars as well. + else if (auto casted_string = tryCast(obj); + casted_string.has_value()) { - return attr.setAttribute(key, obj.cast()); + return attr.setAttribute(key, std::move(*casted_string)); + } + // Assuming `char` is signed on the current platform, + // then this cast will cover `signed char`. + // It's a bit weird: The Numpy datatype will be the same as `char` + // (.ie. 'b'), but the `py::object` will contain an integer. + // Similar for `unsigned char` if `char`s are unsigned. + else if (auto casted_int = tryCast(obj); casted_int.has_value()) + { + return attr.setAttribute( + key, explicit_char_type(*casted_int)); + } + + // NOW: List casts. + // All list casts must come after all scalar casts, + // because list casts implicitly cover scalars too. + else if (auto list_of_char = tryCast(obj); + list_of_char.has_value()) + { + return attr.setAttribute(key, std::move(*list_of_char)); + } + // this must come after tryCast>, + // because tryCast> implicitly covers chars as well + else if (auto list_of_string = tryCast(obj); + list_of_string.has_value()) + { + return attr.setAttribute(key, std::move(*list_of_string)); + } + // Again: `char` vs. `signed char`, resp. `char` vs. `unsigned char` + // depending on `char`'s signedness. + else if (auto list_of_int = tryCast>(obj); + list_of_int.has_value()) + { + std::vector casted; + casted.reserve(list_of_int->size()); + std::transform( + list_of_int->begin(), + list_of_int->end(), + std::back_inserter(casted), + [](int const val) { return explicit_char_type(val); }); + return attr.setAttribute>( + key, std::move(casted)); } else { - return attr.setAttribute(key, obj.cast()); + throw std::runtime_error( + "[Python SetAttributeFromObject] Was not able to use passed " + "object as any char-based type."); } } +} // namespace detail + +struct SetAttributeFromObject +{ + static constexpr char const *errorMsg = "Attributable.set_attribute()"; + + template + static bool + call(Attributable &attr, std::string const &key, py::object &obj) + { + if constexpr (std::is_same_v) + { + return ::detail::setAttributeFromObject_double(attr, key, obj); + } + else if constexpr (std::is_same_v) + { + return ::detail::setAttributeFromObject_bool(attr, key, obj); + } + else if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + return ::detail::setAttributeFromObject_char( + attr, key, obj); + } + else + { + return ::detail::setAttributeFromObject_default( + attr, key, obj); + } + } +}; bool setAttributeFromObject( Attributable &attr, @@ -410,7 +521,10 @@ void init_Attributable(py::module &m) // fundamental Python types .def("set_attribute", &Attributable::setAttribute) - .def("set_attribute", &Attributable::setAttribute) + .def( + "set_attribute", + &Attributable::setAttribute< + typename ::detail::char_to_explicit_char<>::opposite_type>) // -> handle all native python integers as long // .def("set_attribute", &Attributable::setAttribute< short >) // .def("set_attribute", &Attributable::setAttribute< int >) diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index a510098c8d..6ff987f657 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -264,8 +264,25 @@ def attributeRoundTrip(self, file_ending): self.assertEqual(series.get_attribute("char"), "c") self.assertEqual(series.get_attribute("pystring"), "howdy!") self.assertEqual(series.get_attribute("pystring2"), "howdy, too!") - self.assertEqual(bytes(series.get_attribute("pystring3")), - b"howdy, again!") + if file_ending == 'h5': + # A byte string b"hello" is always (really?) a vector of unsigned + # chars. + # HDF5 does not distinguish a platform char type, only explicitly + # signed or unsigned chars. Depending on the signed-ness of char + # on the current platform, the unsigned char from the byte string + # might then be interpreted as a char, not as an unsigned char. + # This means that the roundtrip might not work on platforms with + # unsigned char. + try: + as_bytes = bytes(series.get_attribute("pystring3")) + self.assertEqual(as_bytes, b"howdy, again!") + except TypeError: + self.assertEqual( + series.get_attribute("pystring3"), + [c for c in "howdy, again!"]) + else: + self.assertEqual(bytes(series.get_attribute("pystring3")), + b"howdy, again!") self.assertEqual(series.get_attribute("pyint"), 13) self.assertAlmostEqual(series.get_attribute("pyfloat"), 3.1416) self.assertEqual(series.get_attribute("pybool"), False) @@ -361,7 +378,11 @@ def attributeRoundTrip(self, file_ending): self.assertEqual(series.get_attribute("ubyte_c"), 50) # TODO: returns [100] instead of 100 in json/toml if file_ending != "json" and file_ending != "toml": - self.assertEqual(chr(series.get_attribute("char_c")), 'd') + try: + c = chr(series.get_attribute("char_c")) + self.assertEqual(c, 'd') + except TypeError: + self.assertEqual(series.get_attribute("char_c"), 'd') self.assertEqual(series.get_attribute("int16_c"), 2) self.assertEqual(series.get_attribute("int32_c"), 3) self.assertEqual(series.get_attribute("int64_c"), 4)