From f07021f1eb9be8cd77d89fb636a33d6767536fc9 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Fri, 2 Feb 2018 03:44:38 +0000 Subject: [PATCH 1/6] Fixed bug computing categorical datashader aggregates --- holoviews/operation/datashader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index 69050e750f..ff622935e2 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -374,9 +374,10 @@ def _process(self, element, key=None): raise ValueError("Aggregation column %s not found on %s element. " "Ensure the aggregator references an existing " "dimension." % (column,element)) + name = column if isinstance(agg_fn, ds.count_cat): - name = '%s Count' % agg_fn.column - vdims = [dims[0](column)] + name = '%s Count' % column + vdims = [dims[0](name)] else: vdims = Dimension('Count') params = dict(get_param_values(element), kdims=[x, y], @@ -400,7 +401,7 @@ def _process(self, element, key=None): for c in agg.coords[column].data: cagg = agg.sel(**{column: c}) eldata = cagg if ds_version > '0.5.0' else (xs, ys, cagg.data) - layers[c] = self.p.element_type(eldata, **params) + layers[c] = self.p.element_type(eldata, **dict(params, vdims=vdims)) return NdOverlay(layers, kdims=[data.get_dimension(column)]) @@ -725,12 +726,12 @@ def concatenate(cls, overlay): """ if not isinstance(overlay, NdOverlay): raise ValueError('Only NdOverlays can be concatenated') - xarr = xr.concat([v.data.T for v in overlay.values()], + xarr = xr.concat([v.data.transpose() for v in overlay.values()], pd.Index(overlay.keys(), name=overlay.kdims[0].name)) params = dict(get_param_values(overlay.last), vdims=overlay.last.vdims, kdims=overlay.kdims+overlay.last.kdims) - return Dataset(xarr.T, datatype=['xarray'], **params) + return Dataset(xarr.transpose(), datatype=['xarray'], **params) @classmethod From fbada3ad5830697dc06616f34387c47052070ffb Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Sat, 3 Feb 2018 12:39:27 +0000 Subject: [PATCH 2/6] Added tests for categorical aggregates and shading --- holoviews/core/data/image.py | 4 +- holoviews/operation/datashader.py | 13 +++++++ tests/{ => operation}/testdatashader.py | 50 ++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 4 deletions(-) rename tests/{ => operation}/testdatashader.py (79%) diff --git a/holoviews/core/data/image.py b/holoviews/core/data/image.py index 97193c26de..de8eb80ecd 100644 --- a/holoviews/core/data/image.py +++ b/holoviews/core/data/image.py @@ -66,14 +66,14 @@ def irregular(cls, dataset, dim): @classmethod def shape(cls, dataset, gridded=False): if gridded: - return dataset.data.shape + return dataset.data.shape[:2] else: return cls.length(dataset), len(dataset.dimensions()) @classmethod def length(cls, dataset): - return np.product(dataset.data.shape) + return np.product(dataset.data.shape[:2]) @classmethod diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index ff622935e2..ed0ce65474 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -752,7 +752,20 @@ def rgb2hex(cls, rgb): return "#{0:02x}{1:02x}{2:02x}".format(*(int(v*255) for v in rgb)) + @classmethod + def to_xarray(cls, element): + if issubclass(element.interface, XArrayInterface): + return element + data = tuple(element.dimension_values(kd, expanded=False) + for kd in element.kdims) + data += tuple(element.dimension_values(vd, flat=False) + for vd in element.vdims) + dtypes = [dt for dt in element.datatype if dt != 'xarray'] + return element.clone(data, datatype=['xarray']+dtypes) + + def _process(self, element, key=None): + element = element.map(self.to_xarray, Image) if isinstance(element, NdOverlay): bounds = element.last.bounds element = self.concatenate(element) diff --git a/tests/testdatashader.py b/tests/operation/testdatashader.py similarity index 79% rename from tests/testdatashader.py rename to tests/operation/testdatashader.py index 94a3d62fbc..58f819f4c2 100644 --- a/tests/testdatashader.py +++ b/tests/operation/testdatashader.py @@ -2,13 +2,15 @@ from nose.plugins.attrib import attr import numpy as np -from holoviews import Curve, Points, Image, Dataset, RGB, Path, Graph, TriMesh, QuadMesh +from holoviews import (Dimension, Curve, Points, Image, Dataset, RGB, Path, + Graph, TriMesh, QuadMesh, NdOverlay) from holoviews.element.comparison import ComparisonTestCase try: import datashader as ds from holoviews.operation.datashader import ( - aggregate, regrid, ds_version, stack, directly_connect_edges, rasterize + aggregate, regrid, ds_version, stack, directly_connect_edges, + shade, rasterize ) except: ds_version = None @@ -43,6 +45,17 @@ def test_aggregate_points_sampling(self): x_sampling=0.5, y_sampling=0.5) self.assertEqual(img, expected) + def test_aggregate_points_categorical(self): + points = Points([(0.2, 0.3, 'A'), (0.4, 0.7, 'B'), (0, 0.99, 'C')], vdims='z') + img = aggregate(points, dynamic=False, x_range=(0, 1), y_range=(0, 1), + width=2, height=2, aggregator=ds.count_cat('z')) + xs, ys = [0.25, 0.75], [0.25, 0.75] + expected = NdOverlay({'A': Image((xs, ys, [[1, 0], [0, 0]]), vdims='z Count'), + 'B': Image((xs, ys, [[0, 0], [1, 0]]), vdims='z Count'), + 'C': Image((xs, ys, [[0, 0], [1, 0]]), vdims='z Count')}, + kdims=['z']) + self.assertEqual(img, expected) + def test_aggregate_curve(self): curve = Curve([(0.2, 0.3), (0.4, 0.7), (0.8, 0.99)]) expected = Image(([0.25, 0.75], [0.25, 0.75], [[1, 0], [1, 1]]), @@ -77,6 +90,39 @@ def test_aggregate_dframe_nan_path(self): self.assertEqual(img, expected) +@attr(optional=1) +class DatashaderShadeTests(ComparisonTestCase): + + def test_shade_categorical_images_xarray(self): + xs, ys = [0.25, 0.75], [0.25, 0.75] + data = NdOverlay({'A': Image((xs, ys, [[1, 0], [0, 0]]), datatype=['xarray'], vdims='z Count'), + 'B': Image((xs, ys, [[0, 0], [1, 0]]), datatype=['xarray'], vdims='z Count'), + 'C': Image((xs, ys, [[0, 0], [1, 0]]), datatype=['xarray'], vdims='z Count')}, + kdims=['z']) + shaded = shade(data) + r = [[228, 255], [66, 255]] + g = [[26, 255], [150, 255]] + b = [[28, 255], [129, 255]] + a = [[40, 0], [255, 0]] + expected = RGB((xs, ys, r, g, b, a), datatype=['grid'], + vdims=RGB.vdims+[Dimension('A', range=(0, 1))]) + self.assertEqual(shaded, expected) + + def test_shade_categorical_images_grid(self): + xs, ys = [0.25, 0.75], [0.25, 0.75] + data = NdOverlay({'A': Image((xs, ys, [[1, 0], [0, 0]]), datatype=['grid'], vdims='z Count'), + 'B': Image((xs, ys, [[0, 0], [1, 0]]), datatype=['grid'], vdims='z Count'), + 'C': Image((xs, ys, [[0, 0], [1, 0]]), datatype=['grid'], vdims='z Count')}, + kdims=['z']) + shaded = shade(data) + r = [[228, 255], [66, 255]] + g = [[26, 255], [150, 255]] + b = [[28, 255], [129, 255]] + a = [[40, 0], [255, 0]] + expected = RGB((xs, ys, r, g, b, a), datatype=['grid'], + vdims=RGB.vdims+[Dimension('A', range=(0, 1))]) + self.assertEqual(shaded, expected) + @attr(optional=1) From 1f19405fa2caa6b5f1eb52b2274bf469606326e7 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Sat, 3 Feb 2018 13:02:59 +0000 Subject: [PATCH 3/6] Fixed datetime datashader aggregation --- holoviews/operation/datashader.py | 3 +-- tests/operation/testdatashader.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index ed0ce65474..b908d2f0cb 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -265,8 +265,7 @@ def get_agg_data(cls, obj, category=None): df = df.copy() for d in (x, y): if df[d.name].dtype.kind == 'M': - df[d.name] = df[d.name].astype('datetime64[ns]').astype('int64') * 10e-4 - + df[d.name] = df[d.name].astype('datetime64[ns]').astype('int64') * 1000. return x, y, Dataset(df, kdims=kdims, vdims=vdims), glyph diff --git a/tests/operation/testdatashader.py b/tests/operation/testdatashader.py index 58f819f4c2..bb13b9ccde 100644 --- a/tests/operation/testdatashader.py +++ b/tests/operation/testdatashader.py @@ -5,6 +5,7 @@ from holoviews import (Dimension, Curve, Points, Image, Dataset, RGB, Path, Graph, TriMesh, QuadMesh, NdOverlay) from holoviews.element.comparison import ComparisonTestCase +from holoviews.core.util import pd try: import datashader as ds @@ -64,6 +65,18 @@ def test_aggregate_curve(self): width=2, height=2) self.assertEqual(img, expected) + def test_aggregate_curve_datetimes(self): + dates = pd.date_range(start="2016-01-01", end="2016-01-03", freq='1D') + curve = Curve((dates, [1, 2, 3])) + img = aggregate(curve, width=2, height=2, dynamic=False) + bounds = (np.datetime64('2015-12-31T23:59:59.723518000'), 1.0, + np.datetime64('2016-01-03T00:00:00.276482000'), 3.0) + dates = [np.datetime64('2016-01-01T12:00:00.000000000'), + np.datetime64('2016-01-02T12:00:00.000000000')] + expected = Image((dates, [1.5, 2.5], [[1, 0], [0, 2]]), + datatype=['xarray'], bounds=bounds, vdims='Count') + self.assertEqual(img, expected) + def test_aggregate_ndoverlay(self): ds = Dataset([(0.2, 0.3, 0), (0.4, 0.7, 1), (0, 0.99, 2)], kdims=['x', 'y', 'z']) ndoverlay = ds.to(Points, ['x', 'y'], [], 'z').overlay() From 509baf67f70bacdea022895928de81db9803177b Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Sat, 3 Feb 2018 14:33:55 +0000 Subject: [PATCH 4/6] Added __init__.py file for operation tests --- tests/operation/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/operation/__init__.py diff --git a/tests/operation/__init__.py b/tests/operation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From ba72983f7df27e2d6902f8d38980141fec064438 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Sat, 3 Feb 2018 15:02:11 +0000 Subject: [PATCH 5/6] Fixed py2 bug in dt_to_int utility --- holoviews/core/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 3494517891..d37d1fe6c7 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -207,6 +207,7 @@ def deephash(obj): if sys.version_info.major == 3: basestring = str unicode = str + long = int generator_types = (zip, range, types.GeneratorType) else: basestring = basestring @@ -1562,7 +1563,7 @@ def dt_to_int(value, time_unit='us'): value = value.to_pydatetime() elif isinstance(value, np.datetime64): value = value.tolist() - if isinstance(value, int): + if isinstance(value, (int, long)): # Handle special case of nanosecond precision which cannot be # represented by python datetime return value * 10**-(np.log10(tscale)-3) From 2fa32b125d55ef41d34ee0f93a7aa1c793eb0779 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 5 Feb 2018 12:23:53 +0000 Subject: [PATCH 6/6] Minor code cleanup --- holoviews/operation/datashader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/holoviews/operation/datashader.py b/holoviews/operation/datashader.py index b908d2f0cb..91fc58629e 100644 --- a/holoviews/operation/datashader.py +++ b/holoviews/operation/datashader.py @@ -373,9 +373,7 @@ def _process(self, element, key=None): raise ValueError("Aggregation column %s not found on %s element. " "Ensure the aggregator references an existing " "dimension." % (column,element)) - name = column - if isinstance(agg_fn, ds.count_cat): - name = '%s Count' % column + name = '%s Count' % column if isinstance(agg_fn, ds.count_cat) else column vdims = [dims[0](name)] else: vdims = Dimension('Count')