From 130590f524c37cba488cbfffc134483e43b6cb89 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 4 Jul 2021 15:01:54 +0200 Subject: [PATCH 01/19] [dimension] add RangeMode enum --- nixio/dimensions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index ac1c5672..08d4cde4 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -37,6 +37,16 @@ class IndexMode(Enum): GEQ = "geq" +class RangeMode(Enum): + """ + RangeMode used for slicing along dimensions. + *Inclusive* ranges are defines as [start, end], i.e. start and end are included + *Exclusive* ranges are defined as [start, end), i.e. start is included, end is not + """ + Inclusive = "inclusive" + Exclusive = "exclusive" + + class DimensionContainer(Container): """ DimensionContainer extends Container to support returning different types From c2e2d676c53914ba4781edb4b3ad01b7441ad5a1 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 4 Jul 2021 16:40:57 +0200 Subject: [PATCH 02/19] [SampledDimension] method for range indices --- nixio/dimensions.py | 32 +++++++++++++++++++++++++++++++- nixio/test/test_dimensions.py | 11 +++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 08d4cde4..3aaa2da8 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -391,7 +391,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): )) if np.isclose(position, 0) and mode == IndexMode.Less: - raise IndexError("Position {} is out of bounds for SetDimension with mode {}".format(position, mode.name)) + raise IndexError("Position {} is out of bounds for SampledDimension with mode {}".format(position, mode.name)) index = int(np.floor(scaled_position)) if np.isclose(scaled_position, index): @@ -411,6 +411,36 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): raise ValueError("Unknown IndexMode: {}".format(mode)) + def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + """ + Returns the start and end indices in this dimension that are matching to the given start and end position. + + :param start_position: the start position of the range. + :type start_position: float + :param end_position: the end position of the range. + :type end_position: fload + :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.RangeMode + + :returns: The respective start and end indices. + :rtype: tuple of int + + :raises: ValueError if invalid mode is given + :raises: Index Error if start position is greater than end position. + """ + if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: + raise ValueError("Unknown RangeMode: {}".format(mode)) + + end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + try: + start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual) + end_index = self.index_of(end_position, mode=end_mode) + except IndexError as e: + raise e + if start_index > end_index: + raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) + return (start_index, end_index) + def axis(self, count, start=None, start_position=None): """ Get an axis as defined by this nixio.SampledDimension. It either starts at the offset of the dimension, diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index ed1b69e1..c10fc157 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -6,6 +6,7 @@ # Redistribution and use in source and binary forms, with or without # modification, are permitted under the terms of the BSD License. See # LICENSE file in the root of the Project. +from nixio.dimensions import RangeMode from nixio.exceptions.exceptions import IncompatibleDimensions import os import unittest @@ -108,6 +109,16 @@ def test_sample_dimension(self): assert self.sample_dim.axis(10, start_position=5.0)[0] == 5.0 assert self.sample_dim.axis(10, start_position=5.0)[-1] == 5.0 + 9 * 2 + with self.assertRaises(ValueError): + self.sample_dim.range_indices(0, 1, mode="invalid") + with self.assertRaises(IndexError): + self.sample_dim.range_indices(10, -10, mode=RangeMode.Inclusive) + assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive)[0] == 0 + assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive)[-1] == 4 + + assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive)[0] == 0 + assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive)[-1] == 3 + def test_range_dimension(self): assert self.range_dim.index == 3 assert self.range_dim.dimension_type == nix.DimensionType.Range From f31f8aea4f68615e7650b3e3e55bd3004d12c2cb Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Thu, 8 Jul 2021 17:31:21 +0200 Subject: [PATCH 03/19] [setdimension] add range_indices method --- nixio/dimensions.py | 53 +++++++++++++++++++++++++++++------ nixio/test/test_dimensions.py | 11 +++++++- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 3aaa2da8..a81819c6 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -418,13 +418,13 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :param start_position: the start position of the range. :type start_position: float :param end_position: the end position of the range. - :type end_position: fload + :type end_position: float :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. :type mode: nixio.RangeMode - + :returns: The respective start and end indices. :rtype: tuple of int - + :raises: ValueError if invalid mode is given :raises: Index Error if start position is greater than end position. """ @@ -436,7 +436,7 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual) end_index = self.index_of(end_position, mode=end_mode) except IndexError as e: - raise e + raise IndexError("Error using SampledDimension.range_indices: {}".format(e)) if start_index > end_index: raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) return (start_index, end_index) @@ -739,7 +739,7 @@ def labels(self, labels): labels = list(labels) self._h5group.write_data("labels", labels, dtype=dt) - def index_of(self, position, mode=IndexMode.LessOrEqual): + def index_of(self, position, mode=IndexMode.LessOrEqual, dim_labels=None): """ Returns the index of a certain position in the dimension. Raises IndexError if the position is out of bounds (depending on mode and number of labels). @@ -751,6 +751,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): If the position is not an integer (or is not equal to the nearest integer), then the value is rounded down (for LessOrEqual) or rounded up (for GreaterOrEqual). If the mode is Less, the previous integer is always returned. + :param dim_labels: The labels of this dimension, if None (default) the labels will be read from file. :returns: The matching index :rtype: int @@ -764,12 +765,13 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): if position == 0 and mode == IndexMode.Less: raise IndexError("Position {} is out of bounds for SetDimension with mode {}".format(position, mode.name)) - labels = self.labels - if labels and len(labels) and position > len(labels)-1: + if dim_labels is None: + dim_labels = self.labels + if dim_labels and len(dim_labels) and position > len(dim_labels) - 1: if mode in (IndexMode.Less, IndexMode.LessOrEqual): - return len(labels) - 1 + return len(dim_labels) - 1 raise IndexError("Position {} is out of bounds for SetDimension with length {} and mode {}".format( - position, len(labels), mode.name + position, len(dim_labels), mode.name )) index = int(np.floor(position)) @@ -789,3 +791,36 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): return index raise ValueError("Unknown IndexMode: {}".format(mode)) + + def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + """ + Returns the start and end indices in this dimension that are matching to the given start and end position. + + :param start_position: the start position of the range. + :type start_position: float + :param end_position: the end position of the range. + :type end_position: float + :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.RangeMode + + :returns: The respective start and end indices. + :rtype: tuple of int + + :raises: ValueError if invalid mode is given + :raises: Index Error if start position is greater than end position. + """ + + dim_labels = self.labels + if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: + raise ValueError("Unknown RangeMode: {}".format(mode)) + + end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + + if start_position > end_position: + raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) + try: + start = self.index_of(start_position, mode=IndexMode.GreaterOrEqual, dim_labels=dim_labels) + end = self.index_of(end_position, mode=end_mode, dim_labels=dim_labels) + except IndexError as e: + raise IndexError("Error using SetDimension.range_indices: {}".format(e)) + return (start, end) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index c10fc157..21da9ecc 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -260,7 +260,16 @@ def test_set_dimension_modes(self): with self.assertRaises(IndexError): self.set_dim.index_of(12398, nix.IndexMode.GreaterOrEqual) with self.assertRaises(IndexError): - self.set_dim.index_of(len(test_labels)-0.5, nix.IndexMode.GreaterOrEqual) + self.set_dim.index_of(len(test_labels) - 0.5, nix.IndexMode.GreaterOrEqual) + + with self.assertRaises(ValueError): + self.set_dim.range_indices(0, 10, mode="invalid") + with self.assertRaises(IndexError): + self.set_dim.range_indices(10, -10) + assert self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive)[0] == 0 + assert self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive)[-1] == 8 + assert self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive)[0] == 0 + assert self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive)[-1] == 9 def test_sampled_dimension_modes(self): # exact From 2c7507cfd9d46c6c0e32b50f3866f503afe614fd Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Thu, 8 Jul 2021 18:25:24 +0200 Subject: [PATCH 04/19] [sampleDim] return None for empty ranges --- nixio/dimensions.py | 4 ++-- nixio/test/test_dimensions.py | 29 ++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index a81819c6..9a320eb5 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -422,7 +422,7 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. :type mode: nixio.RangeMode - :returns: The respective start and end indices. + :returns: The respective start and end indices. None, if the range is empty! :rtype: tuple of int :raises: ValueError if invalid mode is given @@ -438,7 +438,7 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): except IndexError as e: raise IndexError("Error using SampledDimension.range_indices: {}".format(e)) if start_index > end_index: - raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) + return None return (start_index, end_index) def axis(self, count, start=None, start_position=None): diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 21da9ecc..c01d10d4 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -113,11 +113,30 @@ def test_sample_dimension(self): self.sample_dim.range_indices(0, 1, mode="invalid") with self.assertRaises(IndexError): self.sample_dim.range_indices(10, -10, mode=RangeMode.Inclusive) - assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive)[0] == 0 - assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive)[-1] == 4 - - assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive)[0] == 0 - assert self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive)[-1] == 3 + range_indices = self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 4 + + range_indices = self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 3 + + range_indices = self.sample_dim.range_indices(3., 3.1, mode=RangeMode.Inclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 0 + range_indices = self.sample_dim.range_indices(3., 3.1, mode=RangeMode.Exclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 0 + + range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=RangeMode.Inclusive) + self.assertIsNone(range_indices) + range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=RangeMode.Exclusive) + self.assertIsNone(range_indices) + + range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=RangeMode.Inclusive) + self.assertIsNotNone(range_indices) + range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=RangeMode.Exclusive) + self.assertIsNone(range_indices) def test_range_dimension(self): assert self.range_dim.index == 3 From ce015086fcea8889dc3afe6e886c773d7e9617b2 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Thu, 8 Jul 2021 18:26:12 +0200 Subject: [PATCH 05/19] [setdimension] return None for empty ranges --- nixio/dimensions.py | 8 ++++---- nixio/test/test_dimensions.py | 26 ++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 9a320eb5..0915bb50 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -803,19 +803,17 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. :type mode: nixio.RangeMode - :returns: The respective start and end indices. + :returns: The respective start and end indices. None, if the range is empty :rtype: tuple of int :raises: ValueError if invalid mode is given :raises: Index Error if start position is greater than end position. """ - - dim_labels = self.labels if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: raise ValueError("Unknown RangeMode: {}".format(mode)) + dim_labels = self.labels end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual - if start_position > end_position: raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) try: @@ -823,4 +821,6 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): end = self.index_of(end_position, mode=end_mode, dim_labels=dim_labels) except IndexError as e: raise IndexError("Error using SetDimension.range_indices: {}".format(e)) + if start > end: + return None return (start, end) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index c01d10d4..9a4278ba 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -285,10 +285,28 @@ def test_set_dimension_modes(self): self.set_dim.range_indices(0, 10, mode="invalid") with self.assertRaises(IndexError): self.set_dim.range_indices(10, -10) - assert self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive)[0] == 0 - assert self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive)[-1] == 8 - assert self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive)[0] == 0 - assert self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive)[-1] == 9 + + range_indices = self.set_dim.range_indices(0.1, 0.4, mode=RangeMode.Inclusive) + self.assertIsNone(range_indices) + range_indices = self.set_dim.range_indices(0.1, 0.4, mode=RangeMode.Exclusive) + self.assertIsNone(range_indices) + + range_indices = self.set_dim.range_indices(0.1, 1.0, mode=RangeMode.Inclusive) + self.assertIsNotNone(range_indices) + assert range_indices[0] == 1 + assert range_indices[1] == 1 + range_indices = self.set_dim.range_indices(0.1, 1.0, mode=RangeMode.Exclusive) + self.assertIsNone(range_indices) + range_indices = self.set_dim.range_indices(0.1, 1.1, mode=RangeMode.Exclusive) + self.assertIsNotNone(range_indices) + assert range_indices[0] == 1 + assert range_indices[1] == 1 + range_indices = self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 8 + range_indices = self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive) + assert range_indices[0] == 0 + assert range_indices[-1] == 9 def test_sampled_dimension_modes(self): # exact From 8ea61ddd622756ec2c4145fc10fd1b8fcf33a0b0 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Thu, 8 Jul 2021 18:26:40 +0200 Subject: [PATCH 06/19] [rangedimension] add range_indices method --- nixio/dimensions.py | 37 +++++++++++++++++++++++++++++++++-- nixio/test/test_dimensions.py | 35 +++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 0915bb50..cd804cb8 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -625,7 +625,7 @@ def unit(self, unit): else: self._h5group.set_attr("unit", unit) - def index_of(self, position, mode=IndexMode.LessOrEqual): + def index_of(self, position, mode=IndexMode.LessOrEqual, ticks=None): """ Returns the index of a certain position in the dimension. Raises IndexError if the position is out of bounds (depending on mode). @@ -641,7 +641,8 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): :returns: The matching index :rtype: int """ - ticks = self.ticks + if ticks is None: + ticks = self.ticks if position < ticks[0]: if mode == IndexMode.GreaterOrEqual: return 0 @@ -666,6 +667,38 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): raise ValueError("Unknown IndexMode: {}".format(mode)) + def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + """ + Returns the start and end indices in this dimension that are matching to the given start and end position. + + :param start_position: the start position of the range. + :type start_position: float + :param end_position: the end position of the range. + :type end_position: float + :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.RangeMode + + :returns: The respective start and end indices. None, if range is empty + :rtype: tuple of int + + :raises: ValueError if invalid mode is given + :raises: Index Error if start position is greater than end position. + """ + if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: + raise ValueError("Unknown RangeMode: {}".format(mode)) + if start_position > end_position: + raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) + ticks = self.ticks + end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + try: + start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual, ticks=ticks) + end_index = self.index_of(end_position, mode=end_mode, ticks=ticks) + except IndexError as e: + raise IndexError("Error using SampledDimension.range_indices: {}".format(e)) + if start_index > end_index: + return None + return (start_index, end_index) + def tick_at(self, index): """ Returns the tick at the given index. Will throw an Exception if the diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 9a4278ba..7190c21a 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -156,12 +156,12 @@ def test_range_dimension(self): assert self.range_dim.unit is None assert self.range_dim.ticks == test_range - other = tuple([i*3.14 for i in range(10)]) + other = tuple([i * 3.14 for i in range(10)]) self.range_dim.ticks = other assert self.range_dim.ticks == other assert self.range_dim.index_of(0.) == 0 - assert self.range_dim.index_of(10.) == (np.floor(10./3.14)) + assert self.range_dim.index_of(10.) == (np.floor(10. / 3.14)) assert self.range_dim.index_of(18.84) == 6 assert self.range_dim.index_of(28.26) == 9 assert self.range_dim.index_of(100.) == 9 @@ -179,6 +179,29 @@ def test_range_dimension(self): self.range_dim.axis(10, 2) self.range_dim.axis(100) + # ticks = (0.0, 3.14, 6.28, 9.42, 12.56, 15.7, 18.84, 21.98, 25.12, 28.26) + with self.assertRaises(ValueError): + self.range_dim.range_indices(0, 10, mode="invalid") + with self.assertRaises(IndexError): + self.range_dim.range_indices(10, -10) + + self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=RangeMode.Inclusive)) + self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=RangeMode.Exclusive)) + + range_indices = self.range_dim.range_indices(3.14, 4.0, mode=RangeMode.Inclusive) + assert range_indices[0] == 1 + assert range_indices[1] == 1 + range_indices = self.range_dim.range_indices(3.14, 4.0, mode=RangeMode.Exclusive) + assert range_indices[0] == 1 + assert range_indices[1] == 1 + + range_indices = self.range_dim.range_indices(6.2, 25.12, mode=RangeMode.Inclusive) + assert range_indices[0] == 2 + assert range_indices[1] == 8 + range_indices = self.range_dim.range_indices(6.2, 25.12, mode=RangeMode.Exclusive) + assert range_indices[0] == 2 + assert range_indices[1] == 7 + def test_set_dim_label_resize(self): setdim = self.array.append_set_dimension() labels = ["A", "B"] @@ -347,17 +370,17 @@ def test_sampled_dimension_modes(self): # valid oob below assert self.sample_dim.index_of(-30, nix.IndexMode.GreaterOrEqual) == 0 - assert self.sample_dim.index_of(offset-0.3, nix.IndexMode.GreaterOrEqual) == 0 + assert self.sample_dim.index_of(offset - 0.3, nix.IndexMode.GreaterOrEqual) == 0 # invalid oob with self.assertRaises(IndexError): - self.sample_dim.index_of(offset-0.2, nix.IndexMode.Less) + self.sample_dim.index_of(offset - 0.2, nix.IndexMode.Less) with self.assertRaises(IndexError): self.sample_dim.index_of(0, nix.IndexMode.Less) with self.assertRaises(IndexError): - self.sample_dim.index_of(offset-0.01, nix.IndexMode.LessOrEqual) + self.sample_dim.index_of(offset - 0.01, nix.IndexMode.LessOrEqual) with self.assertRaises(IndexError): - self.sample_dim.index_of(offset-0.5) + self.sample_dim.index_of(offset - 0.5) def test_range_dimension_modes(self): # exact From 8899530266bddc29814e0fcac8e6ddf095cf99d9 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sat, 10 Jul 2021 14:37:58 +0200 Subject: [PATCH 07/19] [dimension/tag] move sliceMode to Dimension --- nixio/dimensions.py | 51 ++++++++++++++++++----------------- nixio/tag.py | 14 +--------- nixio/test/test_dimensions.py | 44 +++++++++++++++--------------- 3 files changed, 50 insertions(+), 59 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index cd804cb8..13fde0e5 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -37,14 +37,15 @@ class IndexMode(Enum): GEQ = "geq" -class RangeMode(Enum): - """ - RangeMode used for slicing along dimensions. - *Inclusive* ranges are defines as [start, end], i.e. start and end are included - *Exclusive* ranges are defined as [start, end), i.e. start is included, end is not - """ - Inclusive = "inclusive" +class SliceMode(Enum): Exclusive = "exclusive" + Inclusive = "inclusive" + + def to_index_mode(self): + if self == self.Exclusive: + return IndexMode.Less + if self == self.Inclusive: + return IndexMode.LessOrEqual class DimensionContainer(Container): @@ -411,7 +412,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): raise ValueError("Unknown IndexMode: {}".format(mode)) - def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): """ Returns the start and end indices in this dimension that are matching to the given start and end position. @@ -419,8 +420,8 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :type start_position: float :param end_position: the end position of the range. :type end_position: float - :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. - :type mode: nixio.RangeMode + :param mode: The nixio.SliceMode. Defaults to nixio.SliceMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.SliceMode :returns: The respective start and end indices. None, if the range is empty! :rtype: tuple of int @@ -428,10 +429,10 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :raises: ValueError if invalid mode is given :raises: Index Error if start position is greater than end position. """ - if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: - raise ValueError("Unknown RangeMode: {}".format(mode)) + if mode is not SliceMode.Exclusive and mode is not SliceMode.Inclusive: + raise ValueError("Unknown SliceMode: {}".format(mode)) - end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + end_mode = IndexMode.Less if mode == SliceMode.Exclusive else IndexMode.LessOrEqual try: start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual) end_index = self.index_of(end_position, mode=end_mode) @@ -667,7 +668,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual, ticks=None): raise ValueError("Unknown IndexMode: {}".format(mode)) - def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): """ Returns the start and end indices in this dimension that are matching to the given start and end position. @@ -675,8 +676,8 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :type start_position: float :param end_position: the end position of the range. :type end_position: float - :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. - :type mode: nixio.RangeMode + :param mode: The nixio.SliceMode. Defaults to nixio.SliceMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.SliceMode :returns: The respective start and end indices. None, if range is empty :rtype: tuple of int @@ -684,12 +685,12 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :raises: ValueError if invalid mode is given :raises: Index Error if start position is greater than end position. """ - if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: - raise ValueError("Unknown RangeMode: {}".format(mode)) + if mode is not SliceMode.Exclusive and mode is not SliceMode.Inclusive: + raise ValueError("Unknown SliceMode: {}".format(mode)) if start_position > end_position: raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) ticks = self.ticks - end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + end_mode = IndexMode.Less if mode == SliceMode.Exclusive else IndexMode.LessOrEqual try: start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual, ticks=ticks) end_index = self.index_of(end_position, mode=end_mode, ticks=ticks) @@ -825,7 +826,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual, dim_labels=None): raise ValueError("Unknown IndexMode: {}".format(mode)) - def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): + def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): """ Returns the start and end indices in this dimension that are matching to the given start and end position. @@ -833,8 +834,8 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :type start_position: float :param end_position: the end position of the range. :type end_position: float - :param mode: The nixio.RangeMode. Defaults to nixio.RangeMode.Exclusive, i.e. the end position is not part of the range. - :type mode: nixio.RangeMode + :param mode: The nixio.SliceMode. Defaults to nixio.SliceMode.Exclusive, i.e. the end position is not part of the range. + :type mode: nixio.SliceMode :returns: The respective start and end indices. None, if the range is empty :rtype: tuple of int @@ -842,11 +843,11 @@ def range_indices(self, start_position, end_position, mode=RangeMode.Exclusive): :raises: ValueError if invalid mode is given :raises: Index Error if start position is greater than end position. """ - if mode is not RangeMode.Exclusive and mode is not RangeMode.Inclusive: - raise ValueError("Unknown RangeMode: {}".format(mode)) + if mode is not SliceMode.Exclusive and mode is not SliceMode.Inclusive: + raise ValueError("Unknown SliceMode: {}".format(mode)) dim_labels = self.labels - end_mode = IndexMode.Less if mode == RangeMode.Exclusive else IndexMode.LessOrEqual + end_mode = IndexMode.Less if mode == SliceMode.Exclusive else IndexMode.LessOrEqual if start_position > end_position: raise IndexError("Start position {} is greater than end position {}.".format(start_position, end_position)) try: diff --git a/nixio/tag.py b/nixio/tag.py index d59b8ba7..d01ae7f0 100644 --- a/nixio/tag.py +++ b/nixio/tag.py @@ -25,19 +25,7 @@ from .link_type import LinkType from . import util from .section import Section -from enum import Enum -from .dimensions import IndexMode - - -class SliceMode(Enum): - Exclusive = "exclusive" - Inclusive = "inclusive" - - def to_index_mode(self): - if self == self.Exclusive: - return IndexMode.Less - if self == self.Inclusive: - return IndexMode.LessOrEqual +from .dimensions import SliceMode class FeatureContainer(Container): diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 7190c21a..285e285c 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -6,7 +6,7 @@ # Redistribution and use in source and binary forms, with or without # modification, are permitted under the terms of the BSD License. See # LICENSE file in the root of the Project. -from nixio.dimensions import RangeMode +from nixio.dimensions import SliceMode from nixio.exceptions.exceptions import IncompatibleDimensions import os import unittest @@ -114,28 +114,30 @@ def test_sample_dimension(self): with self.assertRaises(IndexError): self.sample_dim.range_indices(10, -10, mode=RangeMode.Inclusive) range_indices = self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive) + self.assertIsNone(self.sample_dim.range_indices(10, -10, mode=SliceMode.Inclusive)) + range_indices = self.sample_dim.range_indices(2, 11, mode=SliceMode.Inclusive) assert range_indices[0] == 0 assert range_indices[-1] == 4 - range_indices = self.sample_dim.range_indices(2, 11, mode=RangeMode.Exclusive) + range_indices = self.sample_dim.range_indices(2, 11, mode=SliceMode.Exclusive) assert range_indices[0] == 0 assert range_indices[-1] == 3 - range_indices = self.sample_dim.range_indices(3., 3.1, mode=RangeMode.Inclusive) + range_indices = self.sample_dim.range_indices(3., 3.1, mode=SliceMode.Inclusive) assert range_indices[0] == 0 assert range_indices[-1] == 0 - range_indices = self.sample_dim.range_indices(3., 3.1, mode=RangeMode.Exclusive) + range_indices = self.sample_dim.range_indices(3., 3.1, mode=SliceMode.Exclusive) assert range_indices[0] == 0 assert range_indices[-1] == 0 - range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=RangeMode.Inclusive) + range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=SliceMode.Inclusive) self.assertIsNone(range_indices) - range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=RangeMode.Exclusive) + range_indices = self.sample_dim.range_indices(3.1, 3.2, mode=SliceMode.Exclusive) self.assertIsNone(range_indices) - range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=RangeMode.Inclusive) + range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=SliceMode.Inclusive) self.assertIsNotNone(range_indices) - range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=RangeMode.Exclusive) + range_indices = self.sample_dim.range_indices(3.1, 5.0, mode=SliceMode.Exclusive) self.assertIsNone(range_indices) def test_range_dimension(self): @@ -185,20 +187,20 @@ def test_range_dimension(self): with self.assertRaises(IndexError): self.range_dim.range_indices(10, -10) - self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=RangeMode.Inclusive)) - self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=RangeMode.Exclusive)) + self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=SliceMode.Inclusive)) + self.assertIsNone(self.range_dim.range_indices(3.15, 3.16, mode=SliceMode.Exclusive)) - range_indices = self.range_dim.range_indices(3.14, 4.0, mode=RangeMode.Inclusive) + range_indices = self.range_dim.range_indices(3.14, 4.0, mode=SliceMode.Inclusive) assert range_indices[0] == 1 assert range_indices[1] == 1 - range_indices = self.range_dim.range_indices(3.14, 4.0, mode=RangeMode.Exclusive) + range_indices = self.range_dim.range_indices(3.14, 4.0, mode=SliceMode.Exclusive) assert range_indices[0] == 1 assert range_indices[1] == 1 - range_indices = self.range_dim.range_indices(6.2, 25.12, mode=RangeMode.Inclusive) + range_indices = self.range_dim.range_indices(6.2, 25.12, mode=SliceMode.Inclusive) assert range_indices[0] == 2 assert range_indices[1] == 8 - range_indices = self.range_dim.range_indices(6.2, 25.12, mode=RangeMode.Exclusive) + range_indices = self.range_dim.range_indices(6.2, 25.12, mode=SliceMode.Exclusive) assert range_indices[0] == 2 assert range_indices[1] == 7 @@ -309,25 +311,25 @@ def test_set_dimension_modes(self): with self.assertRaises(IndexError): self.set_dim.range_indices(10, -10) - range_indices = self.set_dim.range_indices(0.1, 0.4, mode=RangeMode.Inclusive) + range_indices = self.set_dim.range_indices(0.1, 0.4, mode=SliceMode.Inclusive) self.assertIsNone(range_indices) - range_indices = self.set_dim.range_indices(0.1, 0.4, mode=RangeMode.Exclusive) + range_indices = self.set_dim.range_indices(0.1, 0.4, mode=SliceMode.Exclusive) self.assertIsNone(range_indices) - range_indices = self.set_dim.range_indices(0.1, 1.0, mode=RangeMode.Inclusive) + range_indices = self.set_dim.range_indices(0.1, 1.0, mode=SliceMode.Inclusive) self.assertIsNotNone(range_indices) assert range_indices[0] == 1 assert range_indices[1] == 1 - range_indices = self.set_dim.range_indices(0.1, 1.0, mode=RangeMode.Exclusive) + range_indices = self.set_dim.range_indices(0.1, 1.0, mode=SliceMode.Exclusive) self.assertIsNone(range_indices) - range_indices = self.set_dim.range_indices(0.1, 1.1, mode=RangeMode.Exclusive) + range_indices = self.set_dim.range_indices(0.1, 1.1, mode=SliceMode.Exclusive) self.assertIsNotNone(range_indices) assert range_indices[0] == 1 assert range_indices[1] == 1 - range_indices = self.set_dim.range_indices(0, 9, mode=RangeMode.Exclusive) + range_indices = self.set_dim.range_indices(0, 9, mode=SliceMode.Exclusive) assert range_indices[0] == 0 assert range_indices[-1] == 8 - range_indices = self.set_dim.range_indices(0, 9, mode=RangeMode.Inclusive) + range_indices = self.set_dim.range_indices(0, 9, mode=SliceMode.Inclusive) assert range_indices[0] == 0 assert range_indices[-1] == 9 From 0132fce29b78cd7df99c72a2d29e577c653f8add Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sat, 10 Jul 2021 14:47:50 +0200 Subject: [PATCH 08/19] [dimension/range_indices] return invalid range instead of raising exeption --- nixio/dimensions.py | 12 ++++++------ nixio/test/test_dimensions.py | 3 --- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 13fde0e5..1e25fb0c 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -436,8 +436,8 @@ def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): try: start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual) end_index = self.index_of(end_position, mode=end_mode) - except IndexError as e: - raise IndexError("Error using SampledDimension.range_indices: {}".format(e)) + except IndexError: + return None if start_index > end_index: return None return (start_index, end_index) @@ -694,8 +694,8 @@ def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): try: start_index = self.index_of(start_position, mode=IndexMode.GreaterOrEqual, ticks=ticks) end_index = self.index_of(end_position, mode=end_mode, ticks=ticks) - except IndexError as e: - raise IndexError("Error using SampledDimension.range_indices: {}".format(e)) + except IndexError: + return None if start_index > end_index: return None return (start_index, end_index) @@ -853,8 +853,8 @@ def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): try: start = self.index_of(start_position, mode=IndexMode.GreaterOrEqual, dim_labels=dim_labels) end = self.index_of(end_position, mode=end_mode, dim_labels=dim_labels) - except IndexError as e: - raise IndexError("Error using SetDimension.range_indices: {}".format(e)) + except IndexError: + return None if start > end: return None return (start, end) diff --git a/nixio/test/test_dimensions.py b/nixio/test/test_dimensions.py index 285e285c..6d3a74cb 100644 --- a/nixio/test/test_dimensions.py +++ b/nixio/test/test_dimensions.py @@ -111,9 +111,6 @@ def test_sample_dimension(self): with self.assertRaises(ValueError): self.sample_dim.range_indices(0, 1, mode="invalid") - with self.assertRaises(IndexError): - self.sample_dim.range_indices(10, -10, mode=RangeMode.Inclusive) - range_indices = self.sample_dim.range_indices(2, 11, mode=RangeMode.Inclusive) self.assertIsNone(self.sample_dim.range_indices(10, -10, mode=SliceMode.Inclusive)) range_indices = self.sample_dim.range_indices(2, 11, mode=SliceMode.Inclusive) assert range_indices[0] == 0 From e80984f9a8711326af5417eaead06d579a7711aa Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:33:13 +0200 Subject: [PATCH 09/19] [dimension] docstring fixes --- nixio/dimensions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 1e25fb0c..8ad0357c 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -557,7 +557,7 @@ def is_alias(self): elif self.has_link and self.dimension_link._data_object_type == "DataArray": return True return False - + @property def _redirgrp(self): """ @@ -569,7 +569,7 @@ def _redirgrp(self): gname = self._h5group.get_by_pos(0).name return self._h5group.open_group(gname) return self._h5group - + @property def ticks(self): if self.is_alias and not self.has_link: @@ -638,6 +638,8 @@ def index_of(self, position, mode=IndexMode.LessOrEqual, ticks=None): If the mode is Less, the previous index of the matching tick is always returned. If the mode is GreaterOrEqual and the position does not match a tick exactly, the next index is returned. + :param ticks: Optional, the ticks stored in this dimension. If not passed as argument, they are (re)read from file. + :type ticks: iterable :returns: The matching index :rtype: int @@ -786,6 +788,7 @@ def index_of(self, position, mode=IndexMode.LessOrEqual, dim_labels=None): rounded down (for LessOrEqual) or rounded up (for GreaterOrEqual). If the mode is Less, the previous integer is always returned. :param dim_labels: The labels of this dimension, if None (default) the labels will be read from file. + :type dim_labels: iterable :returns: The matching index :rtype: int From 1d9bf80fddcd7f6d2bfcae76efb6770376746c43 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:33:34 +0200 Subject: [PATCH 10/19] [exception] add InvalidSlice exception --- nixio/exceptions/__init__.py | 2 +- nixio/exceptions/exceptions.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/nixio/exceptions/__init__.py b/nixio/exceptions/__init__.py index 9c895a63..a9dea68b 100644 --- a/nixio/exceptions/__init__.py +++ b/nixio/exceptions/__init__.py @@ -1,6 +1,6 @@ from .exceptions import (DuplicateName, UninitializedEntity, InvalidUnit, InvalidAttrType, InvalidEntity, OutOfBounds, - IncompatibleDimensions, InvalidFile, + IncompatibleDimensions, InvalidFile, InvalidSlice, DuplicateColumnName, UnsupportedLinkType) __all__ = ( diff --git a/nixio/exceptions/exceptions.py b/nixio/exceptions/exceptions.py index 3eb8c988..39aa9c3e 100644 --- a/nixio/exceptions/exceptions.py +++ b/nixio/exceptions/exceptions.py @@ -47,6 +47,13 @@ def __init__(self, *args, **kwargs): super(InvalidEntity, self).__init__(self.message, *args, **kwargs) +class InvalidSlice(Exception): + + def __init__(self, *args, **kwargs): + self.message = "Trying to access data with an invalid slice." + super(InvalidSlice, self).__init__(self.message, *args, **kwargs) + + class OutOfBounds(IndexError): def __init__(self, message, index=None): From 1f0a70ce8502d53b36cc2db5f026036f1dbd66eb Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:36:42 +0200 Subject: [PATCH 11/19] [DataView] allows to be created as invalid, ... no exceptions are thrown upon creation, DataView is invalid though and reading the data will return an empty array Adds a "debug_message" to troubleshoot why the DV might be empty Adapts the tests --- nixio/data_view.py | 62 ++++++++++++++++++++++++++++-------- nixio/test/test_data_view.py | 12 ++++--- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/nixio/data_view.py b/nixio/data_view.py index 34bd686b..82e6f67c 100644 --- a/nixio/data_view.py +++ b/nixio/data_view.py @@ -11,39 +11,66 @@ from collections.abc import Iterable except ImportError: from collections import Iterable +import numpy as np from .data_set import DataSet -from .exceptions import OutOfBounds, IncompatibleDimensions +from .exceptions import OutOfBounds, InvalidSlice class DataView(DataSet): def __init__(self, da, slices): - if len(slices) != len(da.shape): + self._valid = slices is not None and all(slices) + self._slices = slices + if not self.valid: + self._error_message = ( + "InvalidSlice error!" + "Given slice {} is invalid! At least one slice along one dimension" + "does not contain data.".format(slices) + ) + else: + self._error_message = "" + + if self.valid and len(slices) != len(da.shape): # This is always checked by the calling function, but we repeat # the check here for future bug catching - raise IncompatibleDimensions( + self._valid = False + self._error_message = ( + "IncompatibleDimensions error." "Number of dimensions for DataView does not match underlying " "data object: {} != {}".format(len(slices), len(da.shape)), - "DataView" ) - if any(s.stop > e for s, e in zip(slices, da.data_extent)): - raise OutOfBounds( - "Trying to create DataView which is out of bounds of the " - "underlying DataArray" + if self.valid and any(s.stop > e for s, e in zip(slices, da.data_extent)): + self._valid = False + self._error_message = ( + "OutOfBounds error!" + "Trying to create DataView with slices {} which are out of bounds of the " + "underlying DataArray {}".format(self._slices, da.shape) ) # Simplify all slices - slices = tuple(slice(*sl.indices(dimlen)) - for sl, dimlen in zip(slices, da.shape)) + if self.valid: + slices = tuple(slice(*sl.indices(dimlen)) + for sl, dimlen in zip(slices, da.shape)) + self._slices = slices self.array = da self._h5group = self.array._h5group - self._slices = slices + + @property + def valid(self): + return self._valid + + @property + def debug_message(self): + return self._error_message @property def data_extent(self): - return tuple(s.stop - s.start for s in self._slices) + if self.valid: + return tuple(s.stop - s.start for s in self._slices) + else: + return None @data_extent.setter def data_extent(self, v): @@ -54,12 +81,19 @@ def data_type(self): return self.array.data_type def _write_data(self, data, sl=None): + if not self.valid: + raise InvalidSlice( + "Write Data failed due to an invalid slice." + "Reason is: {}".format(self._error_message) + ) tsl = self._slices if sl: tsl = self._transform_coordinates(sl) super(DataView, self)._write_data(data, tsl) def _read_data(self, sl=None): + if not self.valid: + return np.array([]) tsl = self._slices if sl is not None: tsl = self._transform_coordinates(sl) @@ -90,7 +124,7 @@ def transform_slice(uslice, dvslice): ustart, ustop, ustep = uslice.indices(dimlen) if ustop < 0: # special case for None stop ustop = dimlen + ustop - tslice = slice(dvslice.start+ustart, dvslice.start+ustop, ustep) + tslice = slice(dvslice.start + ustart, dvslice.start + ustop, ustep) if tslice.stop > dvslice.stop: raise oob @@ -141,7 +175,7 @@ def _expand_user_slices(self, user_slices): expidx = user_slices.index(Ellipsis) npad = len(self.data_extent) - len(user_slices) + 1 padding = (slice(None),) * npad - return user_slices[:expidx] + padding + user_slices[expidx+1:] + return user_slices[:expidx] + padding + user_slices[expidx + 1:] # expand slices at the end npad = len(self.data_extent) - len(user_slices) diff --git a/nixio/test/test_data_view.py b/nixio/test/test_data_view.py index 226fcf85..d89e62a8 100644 --- a/nixio/test/test_data_view.py +++ b/nixio/test/test_data_view.py @@ -123,17 +123,21 @@ def test_data_view_write_index(self): def test_data_view_oob(self): da = self.file.blocks[0].data_arrays[0] + from IPython import embed - with self.assertRaises(nix.exceptions.OutOfBounds): - da.get_slice((41, 81), extents=(1, 1)) + dv = da.get_slice((41, 81), extents=(1, 1)) + assert not dv.valid + assert "OutOfBounds error" in dv.debug_message - with self.assertRaises(nix.exceptions.OutOfBounds): - da.get_slice((0, 0), extents=(100, 5)) + dv = da.get_slice((0, 0), extents=(100, 5)) + assert not dv.valid + assert "OutOfBounds error" in dv.debug_message with self.assertRaises(nix.exceptions.IncompatibleDimensions): da.get_slice((0, 0, 0), extents=(5, 5, 5)) dv = da.get_slice((5, 8), extents=(10, 20)) + assert dv.valid with self.assertRaises(nix.exceptions.OutOfBounds): _ = dv[12, :] From dd14fe10945cc5c3dfb3a36a65bd3d9ebe073bea Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:40:03 +0200 Subject: [PATCH 12/19] [BaseTag] adds new method that only does a position scaling ... and returns the new position as well as the scaling factor, this is done to separate scaling and getting the index for ranges/slices where we need more control --- nixio/tag.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nixio/tag.py b/nixio/tag.py index d01ae7f0..902d6d81 100644 --- a/nixio/tag.py +++ b/nixio/tag.py @@ -150,6 +150,37 @@ def _slices_in_data(data, slices): stops = tuple(sl.stop for sl in slices) return np.all(np.less_equal(stops, dasize)) + @staticmethod + def _scale_position(pos, unit, dim): + dimtype = dim.dimension_type + if dimtype == DimensionType.Set: + dimunit = None + else: + dimunit = dim.unit + scaling = 1.0 + if dimtype == DimensionType.Set: + if unit and unit != "none": + raise IncompatibleDimensions( + "Cannot apply a position with unit to a SetDimension", + "Tag._pos_to_idx" + ) + else: + if dimunit is None and unit is not None: + raise IncompatibleDimensions( + "Units of position and SampledDimension " + "must both be given!", + "Tag._pos_to_idx" + ) + elif dimunit is not None and unit is not None: + try: + scaling = util.units.scaling(unit, dimunit) + except InvalidUnit: + raise IncompatibleDimensions( + "Cannot scale Tag unit {} to match dimension unit {}".format(unit, dimunit), + "Tag._pos_to_idx" + ) + return pos * scaling, scaling + @staticmethod def _pos_to_idx(pos, unit, dim, mode): dimtype = dim.dimension_type From 446f0316d2dd443ff150b578147f9b99b3465f9d Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:41:17 +0200 Subject: [PATCH 13/19] [BaseTag] delegate figuring out the range indices to the methods on the respective dimension --- nixio/tag.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/nixio/tag.py b/nixio/tag.py index 902d6d81..3e3dd29f 100644 --- a/nixio/tag.py +++ b/nixio/tag.py @@ -122,30 +122,31 @@ def _calc_data_slices(self, data, position, extent, stop_rule): units = self.units for idx, dim in enumerate(data.dimensions): + scaling = 1.0 if idx < len(position): - pos = position[idx] - start = self._pos_to_idx(pos, units[idx], dim, IndexMode.GreaterOrEqual) - else: - # Tag doesn't specify (pos, ext) for all dimensions: will return entire remaining dimensions (0:len) - start = 0 - if extent is None or len(extent) == 0: - # no extents: return one element - stop = start + 1 - elif idx < len(extent): - ext = extent[idx] - stop = self._pos_to_idx(pos + ext, units[idx], dim, stop_rule.to_index_mode()) + 1 - else: - # Tag doesn't specify (pos, ext) for all dimensions: will return entire remaining dimensions (0:len) - stop = data.shape[idx] - - if stop <= start: - # always return at least one element per dimension - stop = start + 1 - refslice.append(slice(start, stop)) + start_pos = position[idx] + start_pos, scaling = self._scale_position(start_pos, units[idx], dim) + if extent is not None and idx < len(extent): + stop_pos = extent[idx] + stop_pos *= scaling + stop_pos += start_pos + slice_mode = stop_rule if extent[idx] > 0.0 else SliceMode.Inclusive + else: + stop_pos = start_pos + slice_mode = SliceMode.Inclusive + range_indices = dim.range_indices(start_pos, stop_pos, slice_mode) + refslice.append(range_indices if range_indices is None else slice(range_indices[0], range_indices[1] + 1)) + else: # no position, we take the whole slice for this dimension + start_index = 0 + stop_index = data.shape[idx] + refslice.append(slice(start_index, stop_index)) + return tuple(refslice) @staticmethod def _slices_in_data(data, slices): + if slices is None or not all(slices): + return False dasize = data.data_extent stops = tuple(sl.stop for sl in slices) return np.all(np.less_equal(stops, dasize)) From 0649ac38f267140b4b56798624e356b02baa7f8d Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:42:59 +0200 Subject: [PATCH 14/19] [MTag] adapt tests and do not throw an exception in tagged_data ... the returned DataView will be invalid, though --- nixio/multi_tag.py | 2 -- nixio/test/test_multi_tag.py | 25 ++++++++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/nixio/multi_tag.py b/nixio/multi_tag.py index 218d4833..675bd3be 100644 --- a/nixio/multi_tag.py +++ b/nixio/multi_tag.py @@ -139,8 +139,6 @@ def tagged_data(self, posidx, refidx, stop_rule=SliceMode.Exclusive): ref = references[refidx] slices = self._calc_data_slices_mtag(ref, posidx, stop_rule) - if not self._slices_in_data(ref, slices): - raise OutOfBounds("References data slice out of the extent of the DataArray!") return DataView(ref, slices) def retrieve_feature_data(self, posidx, featidx): diff --git a/nixio/test/test_multi_tag.py b/nixio/test/test_multi_tag.py index c846cdfb..03120fd5 100644 --- a/nixio/test/test_multi_tag.py +++ b/nixio/test/test_multi_tag.py @@ -6,6 +6,7 @@ # Redistribution and use in section and binary forms, with or without # modification, are permitted under the terms of the BSD License. See # LICENSE file in the root of the Project. +from nixio.exceptions.exceptions import InvalidSlice import os import time import unittest @@ -385,13 +386,16 @@ def test_multi_tag_tagged_data(self): assert np.array_equal(y_data[:2000], data[:]) # multi dimensional data + # position 1 should fail since the position in the third dimension does not point to a valid point + # positon 2 and 3 should deliver valid DataViews + # same for segment 0 should again return an invalid DataView because of dimension 3 sample_iv = 1.0 ticks = [1.2, 2.3, 3.4, 4.5, 6.7] unit = "ms" - pos = self.block.create_data_array("pos", "test", data=[[1, 1, 1], [1, 1, 1]]) + pos = self.block.create_data_array("pos", "test", data=[[1, 1, 1], [1, 1, 1.2], [1, 1, 1.2]]) pos.append_set_dimension() pos.append_set_dimension() - ext = self.block.create_data_array("ext", "test", data=[[1, 5, 2], [0, 4, 1]]) + ext = self.block.create_data_array("ext", "test", data=[[1, 5, 2], [1, 5, 2], [0, 4, 1]]) ext.append_set_dimension() ext.append_set_dimension() units = ["none", "ms", "ms"] @@ -414,20 +418,31 @@ def test_multi_tag_tagged_data(self): segtag.units = units posdata = postag.tagged_data(0, 0) + assert not posdata.valid + assert "InvalidSlice error" in posdata.debug_message + assert posdata.data_extent is None + assert posdata.shape is None + with self.assertRaises(InvalidSlice): + posdata._write_data(np.random.randn(1)) + assert sum(posdata[:].shape) == 0 + + posdata = postag.tagged_data(1, 0) + assert posdata.valid + assert posdata.debug_message == "" assert len(posdata.shape) == 3 assert posdata.shape == (1, 1, 1) assert np.isclose(posdata[0, 0, 0], data[1, 1, 0]) - posdata = postag.tagged_data(1, 0) + posdata = postag.tagged_data(2, 0) assert len(posdata.shape) == 3 assert posdata.shape == (1, 1, 1) assert np.isclose(posdata[0, 0, 0], data[1, 1, 0]) - segdata = segtag.tagged_data(0, 0) + segdata = segtag.tagged_data(1, 0) assert len(segdata.shape) == 3 assert segdata.shape == (1, 5, 2) - segdata = segtag.tagged_data(1, 0) + segdata = segtag.tagged_data(2, 0) assert len(segdata.shape) == 3 assert segdata.shape == (1, 4, 1) From c35eeb89666cad6c229834c5408366880ec9788f Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 16:43:38 +0200 Subject: [PATCH 15/19] [setup] remove py2.7 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a202e3b5..d69eaf0e 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,6 @@ def get_wheel_data(): classifiers = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Scientific/Engineering' From c6330e8558552241536ab3023138f914a1ed3117 Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 11 Jul 2021 17:06:31 +0200 Subject: [PATCH 16/19] remove unused import --- nixio/test/test_data_view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nixio/test/test_data_view.py b/nixio/test/test_data_view.py index d89e62a8..1f695fc9 100644 --- a/nixio/test/test_data_view.py +++ b/nixio/test/test_data_view.py @@ -123,7 +123,6 @@ def test_data_view_write_index(self): def test_data_view_oob(self): da = self.file.blocks[0].data_arrays[0] - from IPython import embed dv = da.get_slice((41, 81), extents=(1, 1)) assert not dv.valid From 2bdfdd30f7dbe478df9ce81b1f49d06ff33a50aa Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 18 Jul 2021 12:09:56 +0200 Subject: [PATCH 17/19] [tag] fix double creation of scaling variable --- nixio/tag.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nixio/tag.py b/nixio/tag.py index 3e3dd29f..81c55d66 100644 --- a/nixio/tag.py +++ b/nixio/tag.py @@ -122,7 +122,6 @@ def _calc_data_slices(self, data, position, extent, stop_rule): units = self.units for idx, dim in enumerate(data.dimensions): - scaling = 1.0 if idx < len(position): start_pos = position[idx] start_pos, scaling = self._scale_position(start_pos, units[idx], dim) From 526f53052b3ebe0e45a9fcd0921324f61507d49b Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sat, 7 Aug 2021 12:30:37 +0200 Subject: [PATCH 18/19] [data_view, tag] tiny fixes in class init and exception message --- nixio/data_view.py | 3 +-- nixio/tag.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/nixio/data_view.py b/nixio/data_view.py index 82e6f67c..cbae0de1 100644 --- a/nixio/data_view.py +++ b/nixio/data_view.py @@ -21,14 +21,13 @@ class DataView(DataSet): def __init__(self, da, slices): self._valid = slices is not None and all(slices) self._slices = slices + self._error_message = "" if not self.valid: self._error_message = ( "InvalidSlice error!" "Given slice {} is invalid! At least one slice along one dimension" "does not contain data.".format(slices) ) - else: - self._error_message = "" if self.valid and len(slices) != len(da.shape): # This is always checked by the calling function, but we repeat diff --git a/nixio/tag.py b/nixio/tag.py index 81c55d66..acaf1a34 100644 --- a/nixio/tag.py +++ b/nixio/tag.py @@ -167,8 +167,7 @@ def _scale_position(pos, unit, dim): else: if dimunit is None and unit is not None: raise IncompatibleDimensions( - "Units of position and SampledDimension " - "must both be given!", + "If a unit if given for the position the dimension must not be without a unit!", "Tag._pos_to_idx" ) elif dimunit is not None and unit is not None: From 430a6eec51c42d208df959b1be74b064c68cf74d Mon Sep 17 00:00:00 2001 From: Jan Grewe Date: Sun, 8 Aug 2021 13:35:12 +0200 Subject: [PATCH 19/19] [sampledDim.index_of] use round instead of floor --- nixio/dimensions.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/nixio/dimensions.py b/nixio/dimensions.py index 8ad0357c..4b63773e 100644 --- a/nixio/dimensions.py +++ b/nixio/dimensions.py @@ -386,15 +386,14 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): if scaled_position < 0: if mode == IndexMode.GreaterOrEqual: return 0 - # position is OOB (left side) but can't round up + # position is OOB (left side) but can't round up because LessOrEqual or Less raise IndexError("Position {} is out of bounds for SampledDimension with offset {} and mode {}".format( position, offset, mode.name )) if np.isclose(position, 0) and mode == IndexMode.Less: raise IndexError("Position {} is out of bounds for SampledDimension with mode {}".format(position, mode.name)) - - index = int(np.floor(scaled_position)) + index = int(np.round(scaled_position)) if np.isclose(scaled_position, index): # exact position if mode in (IndexMode.GreaterOrEqual, IndexMode.LessOrEqual): @@ -404,13 +403,20 @@ def index_of(self, position, mode=IndexMode.LessOrEqual): # exact position and Less mode return index - 1 raise ValueError("Unknown IndexMode: {}".format(mode)) - - if mode == IndexMode.GreaterOrEqual: # and inexact position - return index + 1 - if mode in (IndexMode.LessOrEqual, IndexMode.Less): # and inexact position - return index - - raise ValueError("Unknown IndexMode: {}".format(mode)) + if index < scaled_position: + if mode in (IndexMode.LessOrEqual, IndexMode.Less): + return index + elif mode == IndexMode.GreaterOrEqual: + return index + 1 + else: + raise ValueError("Unknown IndexMode: {}".format(mode)) + else: + if mode in (IndexMode.LessOrEqual, IndexMode.Less): + return index - 1 + elif mode == IndexMode.GreaterOrEqual: + return index + else: + raise ValueError("Unknown IndexMode: {}".format(mode)) def range_indices(self, start_position, end_position, mode=SliceMode.Exclusive): """