From 9fcc8067495ae906358d9ff0c63d0508969785c0 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 4 Feb 2022 14:20:38 +0100 Subject: [PATCH 1/4] better multiblock expression --- rio_tiler/expression.py | 31 ++++++++++++++- rio_tiler/io/base.py | 22 +++++------ tests/test_expression.py | 85 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 tests/test_expression.py diff --git a/rio_tiler/expression.py b/rio_tiler/expression.py index 2bed564b..bde19a15 100644 --- a/rio_tiler/expression.py +++ b/rio_tiler/expression.py @@ -1,7 +1,8 @@ """rio-tiler.expression: Parse and Apply expression.""" import re -from typing import Sequence, Tuple, Union +import warnings +from typing import List, Sequence, Tuple, Union import numexpr import numpy @@ -29,6 +30,33 @@ def parse_expression(expression: str, cast: bool = True) -> Tuple: return tuple(map(int, bands)) if cast else tuple(bands) +def get_expression_blocks(expression: str) -> List[str]: + """Split expression in blocks. + + Args: + expression (str): band math/combination expression. + + Returns: + tuple: expression blocks. + + Examples: + >>> parse_expression("b1/b2,b2+b1") + ("b1/b2", "b2+b1") + + """ + if ";" in expression: + return [expr for expr in expression.split(";") if expr] + + expr = [expr for expr in expression.split(",") if expr] + if len(expr) > 1: + warnings.warn( + "Using coma `,` for multiband expression will be deprecated in rio-tiler 4.0. Please use semicolon `;`.", + DeprecationWarning, + ) + + return expr + + def apply_expression( blocks: Sequence[str], bands: Sequence[Union[str, int]], @@ -52,5 +80,6 @@ def apply_expression( numexpr.evaluate(bloc.strip(), local_dict=dict(zip(bands, data))) ) for bloc in blocks + if bloc ] ) diff --git a/rio_tiler/io/base.py b/rio_tiler/io/base.py index 3c2f8fd6..d523fc60 100644 --- a/rio_tiler/io/base.py +++ b/rio_tiler/io/base.py @@ -18,7 +18,7 @@ MissingBands, TileOutsideBounds, ) -from ..expression import apply_expression +from ..expression import apply_expression, get_expression_blocks from ..models import BandStatistics, ImageData, Info from ..tasks import multi_arrays, multi_values from ..types import BBox, Indexes @@ -526,7 +526,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: ) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, assets, output.data) output.band_names = blocks @@ -590,7 +590,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: output = multi_arrays(assets, _reader, bbox, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, assets, output.data) output.band_names = blocks @@ -651,7 +651,7 @@ def _reader(asset: str, **kwargs: Any) -> ImageData: output = multi_arrays(assets, _reader, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, assets, output.data) output.band_names = blocks @@ -716,7 +716,7 @@ def _reader(asset: str, *args, **kwargs: Any) -> Dict: values = numpy.array([d for _, d in data.items()]) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) values = apply_expression(blocks, assets, values) return values.tolist() @@ -779,7 +779,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: output = multi_arrays(assets, _reader, shape, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, assets, output.data) output.band_names = blocks @@ -991,7 +991,7 @@ def _reader(band: str, *args: Any, **kwargs: Any) -> ImageData: output = multi_arrays(bands, _reader, tile_x, tile_y, tile_z, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, bands, output.data) output.band_names = blocks @@ -1043,7 +1043,7 @@ def _reader(band: str, *args: Any, **kwargs: Any) -> ImageData: output = multi_arrays(bands, _reader, bbox, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, bands, output.data) output.band_names = blocks @@ -1093,7 +1093,7 @@ def _reader(band: str, **kwargs: Any) -> ImageData: output = multi_arrays(bands, _reader, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, bands, output.data) output.band_names = blocks @@ -1146,7 +1146,7 @@ def _reader(band: str, *args, **kwargs: Any) -> Dict: values = numpy.array([d for _, d in data.items()]) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) values = apply_expression(blocks, bands, values) return values.tolist() @@ -1197,7 +1197,7 @@ def _reader(band: str, *args: Any, **kwargs: Any) -> ImageData: output = multi_arrays(bands, _reader, shape, **kwargs) if expression: - blocks = expression.split(",") + blocks = get_expression_blocks(expression) output.data = apply_expression(blocks, bands, output.data) output.band_names = blocks diff --git a/tests/test_expression.py b/tests/test_expression.py new file mode 100644 index 00000000..2b3df440 --- /dev/null +++ b/tests/test_expression.py @@ -0,0 +1,85 @@ +"""test rio_tiler.expression functions.""" + +import numpy +import pytest + +from rio_tiler.expression import ( + apply_expression, + get_expression_blocks, + parse_expression, +) + + +@pytest.mark.parametrize( + "expr,expected", + [ + ("b1,b2", [1, 2]), + ("B1,b2", [1, 2]), + ("B1,B2", [1, 2]), + ("where((b1==1) | (b1 > 0.5),1,0);", [1]), + ], +) +def test_parse(expr, expected): + """test parse_expression.""" + assert sorted(parse_expression(expr)) == expected + + +@pytest.mark.parametrize( + "expr,expected", + [ + ("b1,b2", ["1", "2"]), + ("B1,b2", ["1", "2"]), + ("B1,B2", ["1", "2"]), + ], +) +def test_parse_cast(expr, expected): + """test parse_expression without casting.""" + assert sorted(parse_expression(expr, cast=False)) == expected + + +@pytest.mark.parametrize( + "expr,expected", + [ + ("b1,", ["b1"]), + ("b1,b2", ["b1", "b2"]), + ("where((b1==1) | (b1 > 0.5),1,0)", ["where((b1==1) | (b1 > 0.5)", "1", "0)"]), + ("where((b1==1) | (b1 > 0.5),1,0);", ["where((b1==1) | (b1 > 0.5),1,0)"]), + ], +) +def test_get_blocks(expr, expected): + """test get_expression_blocks.""" + with pytest.warns(None): + assert get_expression_blocks(expr) == expected + + +def test_get_blocks_warn(): + """test get_expression_blocks.""" + with pytest.warns(DeprecationWarning): + assert get_expression_blocks("b1,b2") + + +def test_apply_expression(): + """test apply_expression.""" + # divide b1 by b2 + data = numpy.zeros(shape=(2, 10, 10), dtype=numpy.uint8) + data[0] += 1 + data[1] += 2 + d = apply_expression(["b1/b2"], ["b1", "b2"], data) + assert numpy.unique(d) == 0.5 + + # complex expression + data = numpy.zeros(shape=(2, 10, 10), dtype=numpy.uint8) + data[0, 0:5, 0:5] += 1 + d = apply_expression(["where((b1==1) | (b1 > 0.5),1,0)"], ["b1", "b2"], data) + # data has 2 bands but expression just use one + assert d.shape == (1, 10, 10) + assert len(numpy.unique(d)) == 2 + assert numpy.unique(d[0, 0:5, 0:5]) == [1] + + data = numpy.zeros(shape=(2, 10, 10), dtype=numpy.uint8) + data[0, 0:5, 0:5] += 1 + data[1, 0:5, 0:5] += 5 + d = apply_expression( + ["where((b1==1) | (b1 > 0.5),1,0)", "where(b2 > 5,1,0)"], ["b1", "b2"], data + ) + assert d.shape == (2, 10, 10) From 8edf11dd4b21b3d6d7574ff39bbd39ce925e6dbb Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 4 Feb 2022 14:25:23 +0100 Subject: [PATCH 2/4] update changelog --- CHANGES.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 6ca7f274..a68bcb58 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,20 @@ # unreleased * add support for setting the S3 endpoint url scheme via the `AWS_HTTPS` environment variables in `aws_get_object` function using boto3 (https://github.com/cogeotiff/rio-tiler/pull/476) +* Add semicolon `;` support for multi-blocks expression (https://github.com/cogeotiff/rio-tiler/pull/479) +* add `rio_tiler.expression.get_expression_blocks` method to split expression (https://github.com/cogeotiff/rio-tiler/pull/479) + +**future deprecation** + +* using coma `,` in expression to define multiple blocks will be replaced by semicolon `;` + +```python +# before +expression = "b1+b2,b2" + +# new +expression = "b1+b2;b2" +``` # 3.0.3 (2022-01-18) From fc9fe3cca0acb0f9db918652cb66252c66913c97 Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Mon, 7 Feb 2022 08:53:35 +0100 Subject: [PATCH 3/4] Update CHANGES.md Co-authored-by: Kyle Barron --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a68bcb58..7a1cabad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ **future deprecation** -* using coma `,` in expression to define multiple blocks will be replaced by semicolon `;` +* using a comma `,` in an expression to define multiple blocks will be replaced by semicolon `;` ```python # before From 2e01c23b1bfcf26d609bd5e8dcdc8d856e54e960 Mon Sep 17 00:00:00 2001 From: Vincent Sarago Date: Mon, 7 Feb 2022 08:53:40 +0100 Subject: [PATCH 4/4] Update rio_tiler/expression.py Co-authored-by: Kyle Barron --- rio_tiler/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rio_tiler/expression.py b/rio_tiler/expression.py index bde19a15..d5429621 100644 --- a/rio_tiler/expression.py +++ b/rio_tiler/expression.py @@ -50,7 +50,7 @@ def get_expression_blocks(expression: str) -> List[str]: expr = [expr for expr in expression.split(",") if expr] if len(expr) > 1: warnings.warn( - "Using coma `,` for multiband expression will be deprecated in rio-tiler 4.0. Please use semicolon `;`.", + "Using comma `,` for multiband expression will be deprecated in rio-tiler 4.0. Please use semicolon `;`.", DeprecationWarning, )