Skip to content

Commit

Permalink
Merge pull request #2001 from tilezen/dees/keep_n_features_gridded
Browse files Browse the repository at this point in the history 
Adding a function to pick important features by grid
  • Loading branch information
@iandees
iandees authored Dec 10, 2021
2 parents f5a938e + ab0d108 commit 18bf48c
Show file tree
Hide file tree
Showing 4 changed files with 467 additions and 0 deletions.
35 changes: 35 additions & 0 deletions integration-test/1999-keep-n-gridded.py
View file

Large diffs are not rendered by default.

39 changes: 39 additions & 0 deletions queries.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -972,6 +972,45 @@ post_process:
items_matching:
kind: station
rank_key: kind_tile_rank
- fn: vectordatasource.transform.keep_n_features_gridded
params:
source_layer: places
start_zoom: 8
end_zoom: 9
items_matching: { kind: locality }
max_items: 1
grid_width: 3
sorting_keys:
- { sort_key: 'min_zoom', reverse: False }
- { sort_key: 'collision_rank', reverse: False }
- { sort_key: 'population', reverse: True }
- { sort_key: 'id', reverse: True }
- fn: vectordatasource.transform.keep_n_features_gridded
params:
source_layer: places
start_zoom: 9
end_zoom: 11
items_matching: { kind: locality }
max_items: 1
grid_width: 6
sorting_keys:
- { sort_key: 'min_zoom', reverse: False }
- { sort_key: 'collision_rank', reverse: False }
- { sort_key: 'population', reverse: True }
- { sort_key: 'id', reverse: True }
- fn: vectordatasource.transform.keep_n_features_gridded
params:
source_layer: places
start_zoom: 11
end_zoom: 13
items_matching: { kind: locality }
max_items: 1
grid_width: 12
sorting_keys:
- { sort_key: 'min_zoom', reverse: False }
- { sort_key: 'collision_rank', reverse: False }
- { sort_key: 'population', reverse: True }
- { sort_key: 'id', reverse: True }
- fn: vectordatasource.transform.rank_features
params:
source_layer: places
Expand Down
286 changes: 286 additions & 0 deletions test/test_transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,292 @@ def test_short_name(self):
self.assertEquals('foo', props['name:short'])


class KeepNGriddedTest(unittest.TestCase):
longMessage=True

def test_not_points(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Polygon(
[(1, 1), (2, 2), (1, 2), (1, 1)])
test_shape_2 = shapely.geometry.Polygon(
[(10, 10), (20, 20), (10, 20), (10, 10)])
features = [
(test_shape_1, {"foo": "bar"}, "test_shape_1"),
(test_shape_2, {"foo": "bar"}, "test_shape_2"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=1,
grid_width=2,
sorting_keys=[
{"sort_key": "foo"},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEquals(features, output_features, "Non-point features should pass through without modification")

def test_points_keep_1(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.1, 1.0))
test_shape_2 = shapely.geometry.Point((1.1, 1.0))
features = [
(test_shape_1, {"foo": "bar"}, "test_shape_1"),
(test_shape_2, {"foo": "bar"}, "test_shape_2"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=1,
grid_width=2,
sorting_keys=[
{"sort_key": "foo"},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEqual(1, len(output_features), "Should consolidate to a single point in the bucket")
self.assertEqual("test_shape_1", output_features[0][2], "All values equal, should pick first one")

def test_points_keep_1_multisort_second(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.1, 1.0))
test_shape_2 = shapely.geometry.Point((1.1, 1.0))
features = [
(test_shape_2, {"foo": "bar", "min_zoom": 12.0, "population": 20000}, "test_shape_2"),
(test_shape_1, {"foo": "bar", "min_zoom": 12.0, "population": 10000}, "test_shape_1"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=1,
grid_width=2,
sorting_keys=[
{"sort_key": "min_zoom"},
{"sort_key": "population", "reverse": True},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEqual(1, len(output_features), "Should consolidate to a single point in the bucket")
self.assertEqual("test_shape_2", output_features[0][2], "Should pick the shape with higher population")

def test_points_keep_1_multisort_minzoom(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.1, 1.0))
test_shape_2 = shapely.geometry.Point((1.1, 1.0))
features = [
(test_shape_2, {"foo": "bar", "min_zoom": 12.0, "population": 20000}, "test_shape_2"),
(test_shape_1, {"foo": "bar", "min_zoom": 10.0, "population": 10000}, "test_shape_1"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=1,
grid_width=2,
sorting_keys=[
{"sort_key": "min_zoom"},
{"sort_key": "population", "reverse": True},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEqual(1, len(output_features), "Should consolidate to a single point in the bucket")
self.assertEqual("test_shape_1", output_features[0][2], "Should pick the shape with lower min_zoom")

def test_points_keep_1_different_buckets(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.0, 1.0))
test_shape_2 = shapely.geometry.Point((1.0, 1.0))
test_shape_3 = shapely.geometry.Point((75.0, 75.0))
test_shape_4 = shapely.geometry.Point((25.0, 75.0))
features = [
(test_shape_1, {"foo": "bar", "population": 1000}, "test_shape_1"),
(test_shape_2, {"foo": "bar", "population": 2000}, "test_shape_2"),
(test_shape_3, {"foo": "bar", "population": 3000}, "test_shape_3"),
(test_shape_4, {"foo": "bar", "population": 4000}, "test_shape_4"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=1,
grid_width=2,
sorting_keys=[
{"sort_key": "population", "reverse": True},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEqual(3, len(output_features), "Should consolidate to 3 points")
self.assertEqual("test_shape_4", output_features[0][2])
self.assertEqual("test_shape_2", output_features[1][2])
self.assertEqual("test_shape_3", output_features[2][2])

def test_points_keep_more_than_in_one_bucket(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.0, 1.0))
test_shape_2 = shapely.geometry.Point((1.0, 1.0))
test_shape_3 = shapely.geometry.Point((75.0, 75.0))
test_shape_4 = shapely.geometry.Point((25.0, 75.0))
features = [
(test_shape_1, {"foo": "bar", "population": 1000}, "test_shape_1"),
(test_shape_2, {"foo": "bar", "population": 2000}, "test_shape_2"),
(test_shape_3, {"foo": "bar", "population": 3000}, "test_shape_3"),
(test_shape_4, {"foo": "bar", "population": 4000}, "test_shape_4"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=5,
grid_width=2,
sorting_keys=[
{"sort_key": "min_zoom", "reverse": True},
{"sort_key": "population"},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
layer = keep_n_features_gridded(ctx)
output_features = layer['features']
self.assertEqual(4, len(output_features), "Should not consolidate because we're keeping top 5")
self.assertEqual("test_shape_4", output_features[0][2])
self.assertEqual("test_shape_1", output_features[1][2])
self.assertEqual("test_shape_2", output_features[2][2])
self.assertEqual("test_shape_3", output_features[3][2])

def test_fail_on_non_integer_reverse_sort_key(self):
from tilequeue.process import Context
import shapely.geometry

test_shape_1 = shapely.geometry.Point((1.0, 1.0))
test_shape_2 = shapely.geometry.Point((1.0, 1.0))
features = [
(test_shape_1, {"foo": "bar", "population": 1000}, "test_shape_1"),
(test_shape_2, {"foo": "bar", "population": 'error'}, "test_shape_2"),
]
feature_layer = dict(
features=features,
layer_datum=dict(name='test_layer'),
)
feature_layers = [feature_layer]
bounds = (0, 0, 100, 100)
ctx = Context(
feature_layers=feature_layers,
nominal_zoom=0,
unpadded_bounds=bounds,
params=dict(
source_layer="test_layer",
items_matching=dict(foo="bar"),
max_items=5,
grid_width=2,
sorting_keys=[
{"sort_key": "population", "reverse": True},
],
),
resources=None,
log=None,
)
from vectordatasource.transform import keep_n_features_gridded
with self.assertRaises(ValueError):
keep_n_features_gridded(ctx)
self.fail("Should raise an exception when reverse-sorting a non-numeric property")


class TagsPriorityI18nTest(unittest.TestCase):

def _call_fut(self, source, kvs):
Expand Down
Loading

0 comments on commit 18bf48c

Please sign in to comment.