CI: Linux aarch64/arm64 (#1517)

* CI: Linux aarch64/arm64

Add native Linux aarch64/arm64 runners with CircleCI.

* OS: Ubuntu 20.04 -> 22.04

* Select Python3 Executable

* Update Python Version in Spack to 3.11

* [Debug] Set `-DPYBIND11_DETAILED_ERROR_MESSAGES=1`

* Fix handling of set_attribute() with explicit type

* Exclude HDF5 from unsigned char roundtrip on ARM64

* Explicit set_attribute() overload for char type of the other platform

* A further fix in the test

* Other fixes

---------

Co-authored-by: Franz Pöschel <[email protected]>

.circleci/config.yml

@@ -0,0 +1,57 @@
+version: 2
+
+jobs:
+  linux-aarch64:
+    working_directory: ~/linux-aarch64-wheels
+    machine:
+      image: ubuntu-2204:current
+    # resource_class is what tells CircleCI to use an ARM worker for native arm builds
+    #   https://circleci.com/product/features/resource-classes/
+    #   https://circleci.com/docs/using-arm/
+    resource_class: arm.large
+    steps:
+      - checkout
+      - run:
+          name: Install build dependencies
+          command: |
+            sudo apt update
+            sudo apt install cmake g++ gfortran libfabric-dev libopenmpi-dev libhdf5-openmpi-dev hdf5-tools pkgconf python3 python3-setuptools
+            sudo .github/workflows/dependencies/install_spack
+            python3 -m pip install -U pip
+            python3 -m pip install -U packaging setuptools wheel
+            python3 -m pip install -U numpy
+            python3 -m pip install -U mpi4py
+            python3 -m pip install -U pandas
+            python3 -m pip install -U dask
+            python3 -m pip install -U pyarrow
+
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+            spack install
+
+            share/openPMD/download_samples.sh build
+      - run:
+          name: Build openPMD-api
+          command: |
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+
+            export CXXFLAGS="-DPYBIND11_DETAILED_ERROR_MESSAGES=1"
+
+            cmake -S . -B build \
+              -DopenPMD_USE_PYTHON=ON \
+              -DopenPMD_USE_MPI=ON    \
+              -DopenPMD_USE_HDF5=ON   \
+              -DopenPMD_USE_ADIOS2=ON \
+              -DopenPMD_USE_INVASIVE_TESTS=ON \
+              -DPython_EXECUTABLE=$(which python3)
+            cmake --build build --parallel 4
+      - run:
+          name: Test openPMD-api
+          command: |
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+            ctest --test-dir build --output-on-failure
+
+workflows:
+  version: 2
+  all-tests:
+    jobs:
+      - linux-aarch64

.github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/spack.yaml

@@ -0,0 +1,79 @@
+# This is a Spack environment file.
+#
+# Activating and installing this environment will provide all dependencies
+# that are needed for full-feature development.
+#   https//spack.readthedocs.io/en/latest/environments.html#anonymous-environments
+#
+spack:
+  specs:
+  - adios2
+  - hdf5
+  - openmpi
+
+  packages:
+    adios2:
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    libfabric:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    openmpi:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    perl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    pkgconf:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    python:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    hdf5:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: False
+    all:
+      target: [aarch64]
+      variants: ~fortran
+      compiler: [[email protected]]
+
+  compilers:
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags: {}
+      modules: []
+      operating_system: ubuntu22.04
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      spec: [email protected]
+      target: aarch64
+
+  # arm.large with 4 vCPU cores
+  #   https://circleci.com/product/features/resource-classes/
+  #   https://circleci.com/docs/using-arm/
+  config:
+    build_jobs: 4
+
+  # https://cache.spack.io
+  mirrors:
+    E4S: https://cache.e4s.io

include/openPMD/binding/python/Numpy.hpp

@@ -105,7 +105,14 @@ inline Datatype dtype_from_bufferformat(std::string const &fmt)
     if (fmt.find("?") != std::string::npos)
         return DT::BOOL;
     else if (fmt.find("b") != std::string::npos)
-        return DT::CHAR;
+        if constexpr (std::is_signed_v<char>)
+        {
+            return Datatype::CHAR;
+        }
+        else
+        {
+            return Datatype::SCHAR;
+        }
     else if (fmt.find("h") != std::string::npos)
         return DT::SHORT;
     else if (fmt.find("i") != std::string::npos)
@@ -115,7 +122,14 @@ inline Datatype dtype_from_bufferformat(std::string const &fmt)
     else if (fmt.find("q") != std::string::npos)
         return DT::LONGLONG;
     else if (fmt.find("B") != std::string::npos)
-        return DT::UCHAR;
+        if constexpr (std::is_unsigned_v<char>)
+        {
+            return Datatype::CHAR;
+        }
+        else
+        {
+            return Datatype::UCHAR;
+        }
     else if (fmt.find("H") != std::string::npos)
         return DT::USHORT;
     else if (fmt.find("I") != std::string::npos)
@@ -150,10 +164,18 @@ inline pybind11::dtype dtype_to_numpy(Datatype const dt)
     {
     case DT::CHAR:
     case DT::VEC_CHAR:
-    case DT::SCHAR:
-    case DT::VEC_SCHAR:
     case DT::STRING:
     case DT::VEC_STRING:
+        if constexpr (std::is_signed_v<char>)
+        {
+            return pybind11::dtype("b");
+        }
+        else
+        {
+            return pybind11::dtype("B");
+        }
+    case DT::SCHAR:
+    case DT::VEC_SCHAR:
         return pybind11::dtype("b");
         break;
     case DT::UCHAR: