Merge branch 'main' into features/#1117-array-copy-None

.gitlab-ci.yml

@@ -5,28 +5,37 @@ test:
     - x86_64
   script:
     - apt update
-    - apt -y install build-essential python3-pip curl git
-    - "curl -X POST -H \"Accept: application/vnd.github+json\" -H \"Authorization: Bearer $GITHUB_TOKEN\" -H \"X-GitHub-Api-Version: 2022-11-28\" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d '{\"state\":\"pending\",\"target_url\":\"https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID\",\"description\":\"The build runs!\",\"context\":\"continuous-integration/codebase\"}'\n"
+    - apt -y install build-essential python3-pip curl git jo
+    - json=$(jo state="pending" target_url="https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID" description="The build runs!" context="continuous-integration/codebase")
+    - "curl -o /dev/null -L -X POST -H \"Accept: application/vnd.github+json\" -H \"Authorization: Bearer $GITHUB_TOKEN\" -H \"X-GitHub-Api-Version: 2022-11-28\" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d \"$json\" \n"
     - pip install pytest coverage
     - pip install .[hdf5,netcdf]
     - COVERAGE_FILE=report/cov/coverage1 HEAT_TEST_USE_DEVICE=cpu mpirun --allow-run-as-root -n 1 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report1.xml heat/
     - COVERAGE_FILE=report/cov/coverage2 HEAT_TEST_USE_DEVICE=cpu mpirun --allow-run-as-root -n 2 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report2.xml heat/
     - COVERAGE_FILE=report/cov/coverage3 HEAT_TEST_USE_DEVICE=gpu mpirun --allow-run-as-root -n 3 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report3.xml heat/
     - COVERAGE_FILE=report/cov/coverage5 HEAT_TEST_USE_DEVICE=cpu mpirun --allow-run-as-root -n 5 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report5.xml heat/
-    - COVERAGE_FILE=report/cov/coverage8 HEAT_TEST_USE_DEVICE=gpu mpirun --allow-run-as-root -n 6 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report6.xml heat/
+    - COVERAGE_FILE=report/cov/coverage6 HEAT_TEST_USE_DEVICE=gpu mpirun --allow-run-as-root -n 6 coverage run --source=heat --parallel-mode -m pytest --junitxml=report/test/report6.xml heat/
     - coverage combine report/cov/*
     - coverage report
     - coverage xml
     - curl -Os https://uploader.codecov.io/latest/linux/codecov
     - chmod +x codecov
     - ./codecov -F unit -f ./coverage.xml -t $CODECOV_TOKEN -Z
   after_script:
+    - apt -y install jo
     - |
       case $CI_JOB_STATUS in
-        'success') curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d '{"state":"success","target_url":"https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID","description":"The build succeeded!","context":"continuous-integration/codebase"}';;
-        'failed') curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d '{"state":"failure","target_url":"https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID","description":"The build failed!","context":"continuous-integration/codebase"}';;
-        *) curl -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d '{"state":"error","target_url":"https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID","description":"The build errored!","context":"continuous-integration/codebase"}';;
+        'success')
+          json=$(jo state="success" target_url="https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID" description="The build succeeded!" context="continuous-integration/codebase")
+          ;;
+        'failed')
+          json=$(jo state="failure" target_url="https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID" description="The build failed!" context="continuous-integration/codebase")
+          ;;
+        *)
+          json=$(jo state="error" target_url="https://codebase.helmholtz.cloud/haf/heat/-/jobs/$CI_JOB_ID" description="The build errored!" context="continuous-integration/codebase")
+          ;;
       esac
+      curl -o /dev/null -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/helmholtz-analytics/heat/statuses/$CI_COMMIT_SHA -d "$json"
   artifacts:
     when: always
     paths:

heat/cluster/_kcluster.py

@@ -204,7 +204,7 @@ def _assign_to_cluster(self, x: DNDarray):
         """
         # calculate the distance matrix and determine the closest centroid
         distances = self._metric(x, self._cluster_centers)
-        matching_centroids = distances.argmin(axis=1, keepdim=True)
+        matching_centroids = distances.argmin(axis=1, keepdims=True)
 
         return matching_centroids
 

heat/cluster/kmeans.py

@@ -88,13 +88,13 @@ def _update_centroids(self, x: DNDarray, matching_centroids: DNDarray):
             selection = (matching_centroids == i).astype(ht.int64)
             # accumulate points and total number of points in cluster
             assigned_points = x * selection
-            points_in_cluster = selection.sum(axis=0, keepdim=True).clip(
+            points_in_cluster = selection.sum(axis=0, keepdims=True).clip(
                 1.0, ht.iinfo(ht.int64).max
             )
 
             # compute the new centroids
             new_cluster_centers[i : i + 1, :] = (assigned_points / points_in_cluster).sum(
-                axis=0, keepdim=True
+                axis=0, keepdims=True
             )
 
         return new_cluster_centers

heat/cluster/kmedians.py

@@ -95,7 +95,7 @@ def _update_centroids(self, x: DNDarray, matching_centroids: DNDarray):
             else:
                 if clean.shape[0] <= ht.MPI_WORLD.size:
                     clean.resplit_(axis=None)
-                median = ht.median(clean, axis=0, keepdim=True)
+                median = ht.median(clean, axis=0, keepdims=True)
                 new_cluster_centers[i : i + 1, :] = median
 
         return new_cluster_centers

heat/cluster/kmedoids.py

@@ -94,11 +94,11 @@ def _update_centroids(self, x: DNDarray, matching_centroids: DNDarray):
             else:
                 if clean.shape[0] <= ht.MPI_WORLD.size:
                     clean.resplit_(axis=None)
-                median = ht.median(clean, axis=0, keepdim=True)
+                median = ht.median(clean, axis=0, keepdims=True)
 
                 dist = self._metric(x, median)
                 _, displ, _ = x.comm.counts_displs_shape(shape=x.shape, axis=0)
-                idx = dist.argmin(axis=0, keepdim=False).item()
+                idx = dist.argmin(axis=0, keepdims=False).item()
                 proc = 0
                 for p in range(x.comm.size):
                     if displ[p] > idx:

heat/core/_operations.py

@@ -414,7 +414,7 @@ def __reduce_op(
     axis = stride_tricks.sanitize_axis(x.shape, kwargs.get("axis"))
     if isinstance(axis, int):
         axis = (axis,)
-    keepdim = kwargs.get("keepdim")
+    keepdims = kwargs.get("keepdims")
     out = kwargs.get("out")
     split = x.split
     balanced = x.balanced
@@ -446,7 +446,7 @@ def __reduce_op(
             ):  # no neutral element for max/min
                 partial = partial_op(partial, dim=dim, keepdim=True)
             output_shape = output_shape[:dim] + (1,) + output_shape[dim + 1 :]
-        if not keepdim and not len(partial.shape) == 1:
+        if not keepdims and not len(partial.shape) == 1:
             gshape_losedim = tuple(x.gshape[dim] for dim in range(len(x.gshape)) if dim not in axis)
             lshape_losedim = tuple(x.lshape[dim] for dim in range(len(x.lshape)) if dim not in axis)
             output_shape = gshape_losedim
@@ -464,7 +464,7 @@ def __reduce_op(
             balanced = True
             if x.comm.is_distributed():
                 x.comm.Allreduce(MPI.IN_PLACE, partial, reduction_op)
-        elif axis is not None and not keepdim:
+        elif axis is not None and not keepdims:
             down_dims = len(tuple(dim for dim in axis if dim < x.split))
             split -= down_dims
             balanced = x.balanced

heat/core/arithmetics.py

@@ -793,6 +793,37 @@ def pow(t1: Union[DNDarray, float], t2: Union[DNDarray, float]) -> DNDarray:
     DNDarray([[ 1.,  8.],
             [27., 64.]], dtype=ht.float32, device=cpu:0, split=None)
     """
+    # early exit for integer scalars
+    if isinstance(t2, int):
+        try:
+            result = torch.pow(t1.larray, t2)
+            return DNDarray(
+                result,
+                gshape=t1.gshape,
+                dtype=t1.dtype,
+                device=t1.device,
+                split=t1.split,
+                comm=t1.comm,
+                balanced=t1.balanced,
+            )
+        except AttributeError:
+            # t1 is no DNDarray
+            pass
+    elif isinstance(t1, int):
+        try:
+            result = torch.pow(t1, t2.larray)
+            return DNDarray(
+                result,
+                gshape=t2.gshape,
+                dtype=t2.dtype,
+                device=t2.device,
+                split=t2.split,
+                comm=t2.comm,
+                balanced=t2.balanced,
+            )
+        except AttributeError:
+            # t2 is no DNDarray
+            pass
     return _operations.__binary_op(torch.pow, t1, t2)
 
 
@@ -871,7 +902,7 @@ def prod(
     a: DNDarray,
     axis: Union[int, Tuple[int, ...]] = None,
     out: DNDarray = None,
-    keepdim: bool = None,
+    keepdims: bool = None,
 ) -> DNDarray:
     """
     Return the product of array elements over a given axis in form of a DNDarray shaped as a but with the specified axis removed.
@@ -888,7 +919,7 @@ def prod(
     out : DNDarray, optional
         Alternative output array in which to place the result. It must have the same shape as the expected output, but
         the datatype of the output values will be cast if necessary.
-    keepdim : bool, optional
+    keepdims : bool, optional
         If this is set to ``True``, the axes which are reduced are left in the result as dimensions with size one. With this
         option, the result will broadcast correctly against the input array.
 
@@ -907,11 +938,11 @@ def prod(
     DNDarray([ 2., 12.], dtype=ht.float32, device=cpu:0, split=None)
     """
     return _operations.__reduce_op(
-        a, torch.prod, MPI.PROD, axis=axis, out=out, neutral=1, keepdim=keepdim
+        a, torch.prod, MPI.PROD, axis=axis, out=out, neutral=1, keepdims=keepdims
     )
 
 
-DNDarray.prod = lambda self, axis=None, out=None, keepdim=None: prod(self, axis, out, keepdim)
+DNDarray.prod = lambda self, axis=None, out=None, keepdims=None: prod(self, axis, out, keepdims)
 DNDarray.prod.__doc__ = prod.__doc__
 
 
@@ -961,7 +992,7 @@ def sum(
     a: DNDarray,
     axis: Union[int, Tuple[int, ...]] = None,
     out: DNDarray = None,
-    keepdim: bool = None,
+    keepdims: bool = None,
 ) -> DNDarray:
     """
     Sum of array elements over a given axis. An array with the same shape as ``self.__array`` except for the specified
@@ -978,7 +1009,7 @@ def sum(
     out : DNDarray, optional
         Alternative output array in which to place the result. It must have the same shape as the expected output, but
         the datatype of the output values will be cast if necessary.
-    keepdim : bool, optional
+    keepdims : bool, optional
         If this is set to ``True``, the axes which are reduced are left in the result as dimensions with size one. With this
         option, the result will broadcast correctly against the input array.
 
@@ -996,8 +1027,8 @@ def sum(
     """
     # TODO: make me more numpy API complete Issue #101
     return _operations.__reduce_op(
-        a, torch.sum, MPI.SUM, axis=axis, out=out, neutral=0, keepdim=keepdim
+        a, torch.sum, MPI.SUM, axis=axis, out=out, neutral=0, keepdims=keepdims
     )
 
 
-DNDarray.sum = lambda self, axis=None, out=None, keepdim=None: sum(self, axis, out, keepdim)
+DNDarray.sum = lambda self, axis=None, out=None, keepdims=None: sum(self, axis, out, keepdims)

heat/core/linalg/basics.py

@@ -1185,17 +1185,17 @@ def matrix_norm(
         if col_axis > row_axis and not keepdims:
             col_axis -= 1
         return statistics.max(
-            arithmetics.sum(rounding.abs(x), axis=row_axis, keepdim=keepdims),
+            arithmetics.sum(rounding.abs(x), axis=row_axis, keepdims=keepdims),
             axis=col_axis,
-            keepdim=keepdims,
+            keepdims=keepdims,
         )
     elif ord == -1:
         if col_axis > row_axis and not keepdims:
             col_axis -= 1
         return statistics.min(
-            arithmetics.sum(rounding.abs(x), axis=row_axis, keepdim=keepdims),
+            arithmetics.sum(rounding.abs(x), axis=row_axis, keepdims=keepdims),
             axis=col_axis,
-            keepdim=keepdims,
+            keepdims=keepdims,
         )
     elif ord == 2:
         raise NotImplementedError("The largest singular value can't be computed yet.")
@@ -1205,21 +1205,21 @@ def matrix_norm(
         if row_axis > col_axis and not keepdims:
             row_axis -= 1
         return statistics.max(
-            arithmetics.sum(rounding.abs(x), axis=col_axis, keepdim=keepdims),
+            arithmetics.sum(rounding.abs(x), axis=col_axis, keepdims=keepdims),
             axis=row_axis,
-            keepdim=keepdims,
+            keepdims=keepdims,
         )
     elif ord == -constants.inf:
         if row_axis > col_axis and not keepdims:
             row_axis -= 1
         return statistics.min(
-            arithmetics.sum(rounding.abs(x), axis=col_axis, keepdim=keepdims),
+            arithmetics.sum(rounding.abs(x), axis=col_axis, keepdims=keepdims),
             axis=row_axis,
-            keepdim=keepdims,
+            keepdims=keepdims,
         )
     elif ord in [None, "fro"]:
         return exponential.sqrt(
-            arithmetics.sum((complex_math.conj(x) * x).real, axis=axis, keepdim=keepdims)
+            arithmetics.sum((complex_math.conj(x) * x).real, axis=axis, keepdims=keepdims)
         )
     elif ord == "nuc":
         raise NotImplementedError("The nuclear norm can't be computed yet.")
@@ -2277,7 +2277,7 @@ def vdot(x1: DNDarray, x2: DNDarray) -> DNDarray:
 
 
 def vecdot(
-    x1: DNDarray, x2: DNDarray, axis: Optional[int] = None, keepdim: Optional[bool] = None
+    x1: DNDarray, x2: DNDarray, axis: Optional[int] = None, keepdims: Optional[bool] = None
 ) -> DNDarray:
     """
     Computes the (vector) dot product of two DNDarrays.
@@ -2290,7 +2290,7 @@ def vecdot(
         second input array. Must be compatible with x1.
     axis : int, optional
         axis over which to compute the dot product. The last dimension is used if 'None'.
-    keepdim : bool, optional
+    keepdims : bool, optional
         If this is set to 'True', the axes which are reduced are left in the result as dimensions with size one.
 
     See Also
@@ -2310,7 +2310,7 @@ def vecdot(
     if axis is None:
         axis = m.ndim - 1
 
-    return arithmetics.sum(m, axis=axis, keepdim=keepdim)
+    return arithmetics.sum(m, axis=axis, keepdims=keepdims)
 
 
 def vector_norm(
@@ -2386,20 +2386,20 @@ def vector_norm(
             raise TypeError("'axis' must be an integer or 1-tuple for vectors.")
 
     if ord == constants.INF:
-        return statistics.max(rounding.abs(x), axis=axis, keepdim=keepdims)
+        return statistics.max(rounding.abs(x), axis=axis, keepdims=keepdims)
     elif ord == -constants.INF:
-        return statistics.min(rounding.abs(x), axis=axis, keepdim=keepdims)
+        return statistics.min(rounding.abs(x), axis=axis, keepdims=keepdims)
     elif ord == 0:
-        return arithmetics.sum(x != 0, axis=axis, keepdim=keepdims).astype(types.float)
+        return arithmetics.sum(x != 0, axis=axis, keepdims=keepdims).astype(types.float)
     elif ord == 1:
-        return arithmetics.sum(rounding.abs(x), axis=axis, keepdim=keepdims)
+        return arithmetics.sum(rounding.abs(x), axis=axis, keepdims=keepdims)
     elif ord is None or ord == 2:
         s = (complex_math.conj(x) * x).real
-        return exponential.sqrt(arithmetics.sum(s, axis=axis, keepdim=keepdims))
+        return exponential.sqrt(arithmetics.sum(s, axis=axis, keepdims=keepdims))
     elif isinstance(ord, str):
         raise ValueError("Norm order {} is invalid for vectors".format(ord))
     else:
         ret = arithmetics.pow(rounding.abs(x), ord)
-        ret = arithmetics.sum(ret, axis=axis, keepdim=keepdims)
+        ret = arithmetics.sum(ret, axis=axis, keepdims=keepdims)
         ret = arithmetics.pow(ret, 1.0 / ord)
         return ret

heat/core/linalg/tests/test_basics.py

@@ -2071,7 +2071,7 @@ def test_vecdot(self):
         a = ht.full((4, 4), 2, split=0)
         b = ht.ones(4)
 
-        c = ht.linalg.vecdot(a, b, axis=0, keepdim=True)
+        c = ht.linalg.vecdot(a, b, axis=0, keepdims=True)
         self.assertEqual(c.dtype, ht.float32)
         self.assertEqual(c.device, a.device)
         self.assertTrue(ht.equal(c, ht.array([[8, 8, 8, 8]])))
@@ -2118,7 +2118,7 @@ def test_vector_norm(self):
         self.assertEqual(vn.device, b0.device)
         self.assertTrue(ht.allclose(vn, ht.array([5.38516481, 1.41421356, 5.38516481], split=0)))
 
-        # split matrix axis keepdim norm 3
+        # split matrix axis keepdims norm 3
         vn = ht.vector_norm(b1, axis=1, keepdims=True, ord=3)
         self.assertEqual(vn.split, None)
         self.assertEqual(vn.dtype, b1.dtype)