From b0f30b71d398a2439c404e8982b4a9971b264ab7 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 28 Apr 2020 22:56:08 -0700 Subject: [PATCH 1/7] Add geo_shape support for geotile_grid and geohash_grid this commit adds aggregation support for the geo_shape field type on geo*_grid aggregations. --- .../elasticsearch/geometry/utils/Geohash.java | 41 ++ .../bucket/geogrid/GeoGridAggregator.java | 5 +- .../bucket/geogrid/GeoHashGridAggregator.java | 7 +- .../bucket/geogrid/GeoTileGridAggregator.java | 7 +- .../bucket/geogrid/GeoTileUtils.java | 103 +++- .../bucket/geogrid/GeoTileUtilsTests.java | 26 +- .../license/XPackLicenseState.java | 3 +- .../xpack/spatial/SpatialPlugin.java | 56 ++- .../index/fielddata/TriangleTreeReader.java | 4 +- .../bucket/geogrid/AllCellValues.java | 34 ++ .../geogrid/BoundedGeoHashGridTiler.java | 86 ++++ .../geogrid/BoundedGeoTileGridTiler.java | 96 ++++ .../bucket/geogrid/GeoGridTiler.java | 34 ++ .../bucket/geogrid/GeoHashGridTiler.java | 122 +++++ .../bucket/geogrid/GeoShapeCellIdSource.java | 64 +++ .../bucket/geogrid/GeoShapeCellValues.java | 64 +++ .../bucket/geogrid/GeoTileGridTiler.java | 155 ++++++ .../bucket/geogrid/GeoGridTilerTests.java | 446 ++++++++++++++++++ .../geogrid/GeoShapeGeoGridTestCase.java | 300 ++++++++++++ .../GeoShapeGeoHashGridAggregatorTests.java | 45 ++ .../GeoShapeGeoTileGridAggregatorTests.java | 55 +++ .../xpack/spatial/util/GeoTestUtils.java | 27 ++ .../rest-api-spec/test/30_geotile_grid.yml | 57 +++ .../rest-api-spec/test/40_geohash_grid.yml | 59 +++ 24 files changed, 1862 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTiler.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java create mode 100644 x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml create mode 100644 x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java index f67c404dc4188..7ff52f5dc1981 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java @@ -53,6 +53,20 @@ public class Geohash { /** Bit encoded representation of the latitude of north pole */ private static final long MAX_LAT_BITS = (0x1L << (PRECISION * 5 / 2)) - 1; + // Below code is adapted from the spatial4j library (GeohashUtils.java) Apache 2.0 Licensed + private static final double[] precisionToLatHeight, precisionToLonWidth; + static { + precisionToLatHeight = new double[PRECISION + 1]; + precisionToLonWidth = new double[PRECISION + 1]; + precisionToLatHeight[0] = 90*2; + precisionToLonWidth[0] = 180*2; + boolean even = false; + for(int i = 1; i <= PRECISION; i++) { + precisionToLatHeight[i] = precisionToLatHeight[i-1] / (even ? 8 : 4); + precisionToLonWidth[i] = precisionToLonWidth[i-1] / (even ? 4 : 8); + even = ! even; + } + } // no instance: private Geohash() { @@ -97,6 +111,16 @@ public static Rectangle toBoundingBox(final String geohash) { } } + /** Array of geohashes one level below the baseGeohash. Sorted. */ + public static String[] getSubGeohashes(String baseGeohash) { + String[] hashes = new String[BASE_32.length]; + for (int i = 0; i < BASE_32.length; i++) {//note: already sorted + char c = BASE_32[i]; + hashes[i] = baseGeohash+c; + } + return hashes; + } + /** * Calculate all neighbors of a given geohash cell. * @@ -201,6 +225,13 @@ public static final String getNeighbor(String geohash, int level, int dx, int dy } } + /** + * Encode a string geohash to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) + */ + public static final long longEncode(String hash) { + return longEncode(hash, hash.length()); + } + /** * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) */ @@ -290,6 +321,16 @@ public static long mortonEncode(final String hash) { return BitUtil.flipFlop(l); } + /** approximate width of geohash tile for a specific precision in degrees */ + public static double lonWidthInDegrees(int precision) { + return precisionToLonWidth[precision]; + } + + /** approximate height of geohash tile for a specific precision in degrees */ + public static double latHeightInDegrees(int precision) { + return precisionToLatHeight[precision]; + } + private static long encodeLatLon(final double lat, final double lon) { // encode lat/lon flipping the sign bit so negative ints sort before positive ints final int latEnc = encodeLatitude(lat) ^ 0x80000000; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java index 95be75c49f45f..1f72698ac7a26 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java @@ -28,6 +28,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -43,10 +44,10 @@ public abstract class GeoGridAggregator extends Bucke protected final int requiredSize; protected final int shardSize; - protected final CellIdSource valuesSource; + protected final ValuesSource.Numeric valuesSource; protected final LongHash bucketOrds; - GeoGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, + GeoGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, int requiredSize, int shardSize, SearchContext aggregationContext, Aggregator parent, Map metadata) throws IOException { super(name, factories, aggregationContext, parent, metadata); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index e2344314d56f1..f3376c960eb7a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -20,6 +20,7 @@ import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -32,9 +33,9 @@ */ public class GeoHashGridAggregator extends GeoGridAggregator { - GeoHashGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, - int requiredSize, int shardSize, SearchContext aggregationContext, - Aggregator parent, Map metadata) throws IOException { + public GeoHashGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, + int requiredSize, int shardSize, SearchContext aggregationContext, + Aggregator parent, Map metadata) throws IOException { super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, metadata); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java index e6e054c491954..4f7b555b2a1ef 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java @@ -21,6 +21,7 @@ import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -33,9 +34,9 @@ */ public class GeoTileGridAggregator extends GeoGridAggregator { - GeoTileGridAggregator(String name, AggregatorFactories factories, CellIdSource valuesSource, - int requiredSize, int shardSize, SearchContext aggregationContext, - Aggregator parent, Map metadata) throws IOException { + public GeoTileGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, + int requiredSize, int shardSize, SearchContext aggregationContext, + Aggregator parent, Map metadata) throws IOException { super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, metadata); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 03f821296f2a6..125385b1d016b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -18,12 +18,15 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.ESSloppyMath; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.geometry.Rectangle; import java.io.IOException; import java.util.Locale; @@ -43,6 +46,8 @@ public final class GeoTileUtils { private GeoTileUtils() {} + private static final double PI_DIV_2 = Math.PI / 2; + /** * Largest number of tiles (precision) to use. * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) @@ -53,6 +58,18 @@ private GeoTileUtils() {} */ public static final int MAX_ZOOM = 29; + /** + * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 + */ + public static final double LATITUDE_MASK = 85.0511287798066; + + /** + * Since shapes are encoded, their boundaries are to be compared to against the encoded/decoded values of LATITUDE_MASK + */ + public static final double NORMALIZED_LATITUDE_MASK = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(LATITUDE_MASK)); + public static final double NORMALIZED_NEGATIVE_LATITUDE_MASK = + GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-LATITUDE_MASK)); + /** * Bit position of the zoom value within hash - zoom is stored in the most significant 6 bits of a long number. */ @@ -63,6 +80,7 @@ private GeoTileUtils() {} */ private static final long X_Y_VALUE_MASK = (1L << MAX_ZOOM) - 1; + /** * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. * @@ -90,37 +108,65 @@ public static int checkPrecisionRange(int precision) { } /** - * Encode lon/lat to the geotile based long format. - * The resulting hash contains interleaved tile X and Y coordinates. - * The precision itself is also encoded as a few high bits. + * Calculates the x-coordinate in the tile grid for the specified longitude given + * the number of tile columns for a pre-determined zoom-level. + * + * @param longitude the longitude to use when determining the tile x-coordinate + * @param tiles the number of tiles per row for a pre-determined zoom-level */ - public static long longEncode(double longitude, double latitude, int precision) { - // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java - - // Number of tiles for the current zoom level along the X and Y axis - final long tiles = 1 << checkPrecisionRange(precision); - - long xTile = (long) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); + public static int getXTile(double longitude, long tiles) { + // normalizeLon treats this as 180, which is not friendly for tile mapping + if (longitude == -180) { + return 0; + } - double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); - long yTile = (long) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); // Edge values may generate invalid values, and need to be clipped. // For example, polar regions (above/below lat 85.05112878) get normalized. if (xTile < 0) { - xTile = 0; + return 0; } if (xTile >= tiles) { - xTile = tiles - 1; + return (int) tiles - 1; } + + return xTile; + } + + /** + * Calculates the y-coordinate in the tile grid for the specified longitude given + * the number of tile rows for pre-determined zoom-level. + * + * @param latitude the latitude to use when determining the tile y-coordinate + * @param tiles the number of tiles per column for a pre-determined zoom-level + */ + public static int getYTile(double latitude, long tiles) { + double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude))); + int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + if (yTile < 0) { yTile = 0; } if (yTile >= tiles) { - yTile = tiles - 1; + return (int) tiles - 1; } - return longEncode((long) precision, xTile, yTile); + return yTile; + } + + /** + * Encode lon/lat to the geotile based long format. + * The resulting hash contains interleaved tile X and Y coordinates. + * The precision itself is also encoded as a few high bits. + */ + public static long longEncode(double longitude, double latitude, int precision) { + // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java + // Number of tiles for the current zoom level along the X and Y axis + final long tiles = 1 << checkPrecisionRange(precision); + long xTile = getXTile(longitude, tiles); + long yTile = getYTile(latitude, tiles); + return longEncodeTiles(precision, xTile, yTile); } /** @@ -131,7 +177,13 @@ public static long longEncode(double longitude, double latitude, int precision) */ public static long longEncode(String hashAsString) { int[] parsed = parseHash(hashAsString); - return longEncode((long)parsed[0], (long)parsed[1], (long)parsed[2]); + return longEncode((long) parsed[0], (long) parsed[1], (long) parsed[2]); + } + + public static long longEncodeTiles(int precision, long xTile, long yTile) { // Zoom value is placed in front of all the bits used for the geotile + // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), + // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. + return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile; } /** @@ -193,6 +245,23 @@ static GeoPoint keyToGeoPoint(String hashAsString) { return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } + public static Rectangle toBoundingBox(long hash) { + int[] hashAsInts = parseHash(hash); + return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); + } + + public static Rectangle toBoundingBox(int xTile, int yTile, int precision) { + final double tiles = validateZXY(precision, xTile, yTile); + final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; + final double maxN = Math.PI - (2.0 * Math.PI * (yTile)) / tiles; + final double minY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(minN))); + final double minX = ((xTile) / tiles * 360.0) - 180; + final double maxY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(maxN))); + final double maxX = ((xTile + 1) / tiles * 360.0) - 180; + + return new Rectangle(minX, maxX, maxY, minY); + } + /** * Validates Zoom, X, and Y values, and returns the total number of allowed tiles along the x/y axis. */ diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index fc5cf6cb910bd..cf3e8699b6894 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; @@ -28,8 +29,10 @@ import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class GeoTileUtilsTests extends ESTestCase { @@ -219,8 +222,8 @@ public void testGeoTileAsLongRoutines() { * so ensure they are clipped correctly. */ public void testSingularityAtPoles() { - double minLat = -85.05112878; - double maxLat = 85.05112878; + double minLat = -GeoTileUtils.LATITUDE_MASK; + double maxLat = GeoTileUtils.LATITUDE_MASK; double lon = randomIntBetween(-180, 180); double lat = randomBoolean() ? randomDoubleBetween(-90, minLat, true) @@ -231,4 +234,23 @@ public void testSingularityAtPoles() { String clippedTileIndex = stringEncode(longEncode(lon, clippedLat, zoom)); assertEquals(tileIndex, clippedTileIndex); } + + public void testPointToTile() { + int zoom = randomIntBetween(0, MAX_ZOOM); + int tiles = 1 << zoom; + int xTile = randomIntBetween(0, zoom); + int yTile = randomIntBetween(0, zoom); + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, zoom); + // check corners + assertThat(GeoTileUtils.getXTile(rectangle.getMinX(), tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getXTile(rectangle.getMaxX(), tiles), equalTo(Math.min(tiles - 1, xTile + 1))); + assertThat(GeoTileUtils.getYTile(rectangle.getMaxY(), tiles), anyOf(equalTo(yTile - 1), equalTo(yTile))); + assertThat(GeoTileUtils.getYTile(rectangle.getMinY(), tiles), anyOf(equalTo(yTile + 1), equalTo(yTile))); + // check point inside + double x = randomDoubleBetween(rectangle.getMinX(), rectangle.getMaxX(), false); + double y = randomDoubleBetween(rectangle.getMinY(), rectangle.getMaxY(), false); + assertThat(GeoTileUtils.getXTile(x, tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getYTile(y, tiles), equalTo(yTile)); + + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index c497cbe1c007a..8013135ae8829 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -46,7 +46,8 @@ public enum Feature { SECURITY_API_KEY_SERVICE(OperationMode.MISSING, false), SECURITY_AUTHORIZATION_REALM(OperationMode.PLATINUM, true), SECURITY_AUTHORIZATION_ENGINE(OperationMode.PLATINUM, true), - SPATIAL_GEO_CENTROID(OperationMode.GOLD, true); + SPATIAL_GEO_CENTROID(OperationMode.GOLD, true), + SPATIAL_GEO_GRID(OperationMode.GOLD, true); final OperationMode minimumOperationMode; final boolean needsActive; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 54a56f4af368c..a245407bdd895 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -16,6 +16,10 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregator; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregator; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregator; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregatorSupplier; @@ -23,6 +27,7 @@ import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregatorSupplier; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregatorSupplier; +import org.elasticsearch.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregator; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregatorSupplier; @@ -36,6 +41,12 @@ import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder; import org.elasticsearch.xpack.spatial.ingest.CircleProcessor; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHashGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeCellIdSource; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; @@ -80,8 +91,9 @@ public List> getQueries() { @Override public List> getAggregationExtentions() { return List.of( - this::registerGeoShapeBoundsAggregator, this::registerGeoShapeCentroidAggregator, + this::registerGeoShapeGridAggregators, + SpatialPlugin::registerGeoShapeBoundsAggregator, SpatialPlugin::registerValueCountAggregator, SpatialPlugin::registerCardinalityAggregator ); @@ -92,7 +104,7 @@ public Map getProcessors(Processor.Parameters paramet return Map.of(CircleProcessor.TYPE, new CircleProcessor.Factory()); } - public void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { + public static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { builder.register(GeoBoundsAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (GeoBoundsAggregatorSupplier) (name, aggregationContext, parent, valuesSource, wrapLongitude, metadata) -> new GeoShapeBoundsAggregator(name, aggregationContext, parent, (GeoShapeValuesSource) valuesSource, @@ -110,13 +122,49 @@ public void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder buil }); } - public static void registerValueCountAggregator(ValuesSourceRegistry.Builder builder) { + public void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder) { + builder.register(GeoHashGridAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), + (GeoGridAggregatorSupplier) (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, + aggregationContext, parent, metadata) -> { + if (getLicenseState().isAllowed(XPackLicenseState.Feature.SPATIAL_GEO_CENTROID)) { + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoHashGridTiler(); + } else { + tiler = new BoundedGeoHashGridTiler(geoBoundingBox); + } + GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); + return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, + parent, metadata); + } + throw LicenseUtils.newComplianceException("geohash_grid aggregation on geo_shape fields"); + }); + + builder.register(GeoTileGridAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), + (GeoGridAggregatorSupplier) (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, + aggregationContext, parent, metadata) -> { + if (getLicenseState().isAllowed(XPackLicenseState.Feature.SPATIAL_GEO_GRID)) { + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoTileGridTiler(); + } else { + tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + } + GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); + return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, + parent, metadata); + } + throw LicenseUtils.newComplianceException("geotile_grid aggregation on geo_shape fields"); + }); + } + + static void registerValueCountAggregator(ValuesSourceRegistry.Builder builder) { builder.register(ValueCountAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (ValueCountAggregatorSupplier) ValueCountAggregator::new ); } - public static void registerCardinalityAggregator(ValuesSourceRegistry.Builder builder) { + static void registerCardinalityAggregator(ValuesSourceRegistry.Builder builder) { builder.register(CardinalityAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (CardinalityAggregatorSupplier) CardinalityAggregator::new); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java index 5673b6d312591..3f57ffff79229 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/TriangleTreeReader.java @@ -38,7 +38,7 @@ * ----------------------------------------- * ----------------------------------------- */ -class TriangleTreeReader { +public class TriangleTreeReader { private final ByteArrayDataInput input; private final CoordinateEncoder coordinateEncoder; private final Tile2D tile2D; @@ -46,7 +46,7 @@ class TriangleTreeReader { private int treeOffset; private int docValueOffset; - TriangleTreeReader(CoordinateEncoder coordinateEncoder) { + public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.tile2D = new Tile2D(); this.extent = new Extent(); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java new file mode 100644 index 0000000000000..acd0c96d3b668 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +import java.io.IOException; + +/** Sorted numeric doc values for precision 0 */ +class AllCellValues extends AbstractSortingNumericDocValues { + private MultiGeoShapeValues geoValues; + + protected AllCellValues(MultiGeoShapeValues geoValues, GeoGridTiler tiler) { + this.geoValues = geoValues; + resize(1); + values[0] = tiler.encode(0, 0, 0); + } + + // for testing + protected long[] getValues() { + return values; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + resize(1); + return geoValues.advanceExact(docId); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java new file mode 100644 index 0000000000000..f4b0426e4f62a --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +public class BoundedGeoHashGridTiler extends GeoHashGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + public BoundedGeoHashGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + protected int setValue(GeoShapeCellValues docValues, MultiGeoShapeValues.GeoShapeValue geoValue, MultiGeoShapeValues.BoundingBox bounds, + int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + return 0; + } + + @Override + protected GeoRelation relateTile(MultiGeoShapeValues.GeoShapeValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } else { + return GeoRelation.QUERY_DISJOINT; + } + } + + @Override + protected int setValuesForFullyContainedTile(String hash, GeoShapeCellValues values, int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision ) { + if (cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(hashes[i]))) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java new file mode 100644 index 0000000000000..72fad3e836f37 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +public class BoundedGeoTileGridTiler extends GeoTileGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + public BoundedGeoTileGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + long hash = encode(x, y, precision); + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(hash))) { + values[valuesIdx] = hash; + return valuesIdx + 1; + } + return valuesIdx; + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + public GeoRelation relateTile(MultiGeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } + return GeoRelation.QUERY_DISJOINT; + } + + @Override + protected int setValue(GeoShapeCellValues docValues, MultiGeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(xTile, yTile, precision))) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + return 0; + } + + @Override + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(nextX, nextY, zTile))) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTiler.java new file mode 100644 index 0000000000000..fa386ef113b72 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +/** + * The tiler to use to convert a geo value into long-encoded bucket keys for aggregating. + */ +public interface GeoGridTiler { + /** + * encodes a single point to its long-encoded bucket key value. + * + * @param x the x-coordinate + * @param y the y-coordinate + * @param precision the zoom level of tiles + */ + long encode(double x, double y, int precision); + + /** + * + * @param docValues the array of long-encoded bucket keys to fill + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles the geoValue intersects + */ + int setValues(GeoShapeCellValues docValues, MultiGeoShapeValues.GeoShapeValue geoValue, int precision); +} + diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java new file mode 100644 index 0000000000000..bc9fb4920e47d --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +public class GeoHashGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return Geohash.longEncode(x, y, precision); + } + + @Override + public int setValues(GeoShapeCellValues values, MultiGeoShapeValues.GeoShapeValue geoValue, int precision) { + if (precision == 1) { + values.resizeCell(1); + values.add(0, Geohash.longEncode(0, 0, 0)); + } + + MultiGeoShapeValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); + long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); + long count = (numLonCells + 1) * (numLatCells + 1); + if (count == 1) { + return setValue(values, geoValue, bounds, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, bounds); + } else { + return setValuesByRasterization("", values, 0, precision, geoValue); + } + } + + /** + * Sets a singular doc-value for the {@link MultiGeoShapeValues.GeoShapeValue}. To be overriden by {@link BoundedGeoHashGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(GeoShapeCellValues docValues, MultiGeoShapeValues.GeoShapeValue geoValue, MultiGeoShapeValues.BoundingBox bounds, + int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + + protected GeoRelation relateTile(MultiGeoShapeValues.GeoShapeValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + return geoValue.relate(rectangle); + } + + protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoShapeValues.GeoShapeValue geoValue, int precision, + MultiGeoShapeValues.BoundingBox bounds) { + // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can + // see that eventually we will be visiting twice the same cell which should not happen. + int idx = 0; + String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); + String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1); + double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow); + double minX = Geohash.decodeLongitude(min); + double maxY = Geohash.decodeLatitude(max); + double maxX = Geohash.decodeLongitude(max); + for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) { + for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) { + String hash = Geohash.stringEncode(i, j, precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, encode(i, j, precision)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(String hash, GeoShapeCellValues values, int valuesIndex, int targetPrecision, + MultiGeoShapeValues.GeoShapeValue geoValue) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + GeoRelation relation = relateTile(geoValue, hashes[i]); + if (relation == GeoRelation.QUERY_CROSSES) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = + setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue); + } + } else if (relation == GeoRelation.QUERY_INSIDE) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + values.resizeCell(valuesIndex + (int) Math.pow(32, targetPrecision - hash.length()) + 1); + valuesIndex = setValuesForFullyContainedTile(hashes[i],values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(String hash, GeoShapeCellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java new file mode 100644 index 0000000000000..2d09c042a84d4 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; + +public class GeoShapeCellIdSource extends ValuesSource.Numeric { + private final GeoShapeValuesSource valuesSource; + private final int precision; + private final GeoGridTiler encoder; + + public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, int precision, GeoGridTiler encoder) { + this.valuesSource = valuesSource; + this.precision = precision; + this.encoder = encoder; + } + + public int precision() { + return precision; + } + + @Override + public boolean isFloatingPoint() { + return false; + } + + @Override + public SortedNumericDocValues longValues(LeafReaderContext ctx) { + MultiGeoShapeValues geoValues = valuesSource.geoShapeValues(ctx); + if (precision == 0) { + // special case, precision 0 is the whole world + return new AllCellValues(geoValues, encoder); + } + ValuesSourceType vs = geoValues.valuesSourceType(); + if (GeoShapeValuesSourceType.instance() == vs) { + // docValues are geo shapes + return new GeoShapeCellValues(geoValues, precision, encoder); + } else { + throw new IllegalArgumentException("unsupported geo type"); + } + } + + @Override + public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { + throw new UnsupportedOperationException(); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java new file mode 100644 index 0000000000000..9d39492e1774f --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +import java.io.IOException; + +/** Sorted numeric doc values for geo shapes */ +class GeoShapeCellValues extends AbstractSortingNumericDocValues { + private MultiGeoShapeValues geoShapeValues; + protected int precision; + protected GeoGridTiler tiler; + + protected GeoShapeCellValues(MultiGeoShapeValues geoShapeValues, int precision, GeoGridTiler tiler) { + this.geoShapeValues = geoShapeValues; + this.precision = precision; + this.tiler = tiler; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (geoShapeValues.advanceExact(docId)) { + assert geoShapeValues.docValueCount() == 1; + int j = advanceValue(geoShapeValues.nextValue()); + resize(j); + sort(); + return true; + } else { + return false; + } + } + + // for testing + protected long[] getValues() { + return values; + } + + protected void add(int idx, long value) { + values[idx] = value; + } + + void resizeCell(int newSize) { + resize(newSize); + } + + /** + * Sets the appropriate long-encoded value for target + * in values. + * + * @param target the geo-shape to encode + * @return number of buckets for given shape tiling of target. + */ + int advanceValue(MultiGeoShapeValues.GeoShapeValue target) { + return tiler.setValues(this, target, precision); + } + +} + diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java new file mode 100644 index 0000000000000..d15a241cbfc89 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoTileGridTiler.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; + +public class GeoTileGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return GeoTileUtils.longEncode(x, y, precision); + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + values[valuesIdx] = encode(x, y, precision); + return valuesIdx + 1; + } + + /** + * Sets the values of the long[] underlying {@link GeoShapeCellValues}. + * + * If the shape resides between GeoTileUtils.NORMALIZED_LATITUDE_MASK and 90 or + * between GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK and -90 degree latitudes, then + * the shape is not accounted for since geo-tiles are only defined within those bounds. + * + * @param values the bucket values + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles set by the shape + */ + @Override + public int setValues(GeoShapeCellValues values, MultiGeoShapeValues.GeoShapeValue geoValue, int precision) { + MultiGeoShapeValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + + if (precision == 0) { + values.resizeCell(1); + values.add(0, GeoTileUtils.longEncodeTiles(0, 0, 0)); + return 1; + } + + // geo tiles are not defined at the extreme latitudes due to them + // tiling the world as a square. + if ((bounds.top > GeoTileUtils.NORMALIZED_LATITUDE_MASK && bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK) + || (bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK + && bounds.bottom < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + int count = (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); + if (count == 1) { + return setValue(values, geoValue, minXTile, minYTile, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); + } else { + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); + } + } + + protected GeoRelation relateTile(MultiGeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + return geoValue.relate(rectangle); + } + + /** + * Sets a singular doc-value for the {@link MultiGeoShapeValues.GeoShapeValue}. To be overriden by {@link BoundedGeoTileGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(GeoShapeCellValues docValues, MultiGeoShapeValues.GeoShapeValue geoValue, int xTile, int yTile, int precision) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + + /** + * + * @param values the bucket values as longs + * @param geoValue the shape value + * @param precision the target precision to split the shape up into + * @return the number of buckets the geoValue is found in + */ + protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoShapeValues.GeoShapeValue geoValue, + int precision, int minXTile, int minYTile, int maxXTile, int maxYTile) { + int idx = 0; + for (int i = minXTile; i <= maxXTile; i++) { + for (int j = minYTile; j <= maxYTile; j++) { + GeoRelation relation = relateTile(geoValue, i, j, precision); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex, + int targetPrecision, MultiGeoShapeValues.GeoShapeValue geoValue) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + GeoRelation relation = relateTile(geoValue, nextX, nextY, zTile); + if (GeoRelation.QUERY_INSIDE == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + values.resizeCell(valuesIndex + 1 << ( 2 * (targetPrecision - zTile)) + 1); + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } else if (GeoRelation.QUERY_CROSSES == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, targetPrecision, geoValue); + } + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, GeoShapeCellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java new file mode 100644 index 0000000000000..44b47fdd74b33 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -0,0 +1,446 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeCoordinateEncoder; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; +import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeReader; + +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK; +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.encodeDecodeLat; +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.encodeDecodeLon; +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.randomBBox; +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.triangleTreeReader; +import static org.hamcrest.Matchers.equalTo; + +public class GeoGridTilerTests extends ESTestCase { + private static final GeoTileGridTiler GEOTILE = new GeoTileGridTiler(); + private static final GeoHashGridTiler GEOHASH = new GeoHashGridTiler(); + + public void testGeoTile() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + assertThat(GEOTILE.encode(x, y, precision), equalTo(GeoTileUtils.longEncode(x, y, precision))); + + // create rectangle within tile and check bound counts + Rectangle tile = GeoTileUtils.toBoundingBox(1309, 3166, 13); + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + + // test shape within tile bounds + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 13); + assertThat(count, equalTo(1)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 14); + assertThat(count, equalTo(4)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 15); + assertThat(count, equalTo(16)); + } + } + + public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + public void testGeoTileSetValuesBruteAndRecursivePolygon() throws Exception { + Geometry geometry = GeometryTestUtils.randomPolygon(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { + Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + // tests that bounding boxes of shapes crossing the dateline are correctly wrapped + public void testGeoTileSetValuesBoundingBoxes_BoundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 1; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(randomBBox()))); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + GeoBoundingBox geoBoundingBox = randomBBox(); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + GeoShapeCellValues cellValues = new GeoShapeCellValues(null, precision, GEOTILE); + + int numTiles = new BoundedGeoTileGridTiler(geoBoundingBox).setValues(cellValues, value, precision); + int expected = numTiles(value, precision, geoBoundingBox); + + assertThat(numTiles, equalTo(expected)); + } + } + + // test random rectangles that can cross the date-line and verify that there are an expected + // number of tiles returned + public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(randomBBox()))); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + int expected = numTiles(value, precision); + assertThat(numTiles, equalTo(expected)); + } + } + + public void testTilerMatchPoint() throws Exception { + int precision = randomIntBetween(0, 4); + Point originalPoint = GeometryTestUtils.randomPoint(false); + int xTile = GeoTileUtils.getXTile(originalPoint.getX(), 1 << precision); + int yTile = GeoTileUtils.getYTile(originalPoint.getY(), 1 << precision); + Rectangle bbox = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + + Point[] pointCorners = new Point[] { + // tile corners + new Point(bbox.getMinX(), bbox.getMinY()), + new Point(bbox.getMinX(), bbox.getMaxY()), + new Point(bbox.getMaxX(), bbox.getMinY()), + new Point(bbox.getMaxX(), bbox.getMaxY()), + // tile edge midpoints + new Point(bbox.getMinX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point(bbox.getMaxX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMinY()), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMaxY()), + }; + + for (Point point : pointCorners) { + if (point.getX() == GeoUtils.MAX_LON || point.getY() == -LATITUDE_MASK) { + continue; + } + TriangleTreeReader reader = triangleTreeReader(point, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + assertThat(numTiles, equalTo(1)); + long tilerHash = unboundedCellValues.getValues()[0]; + long pointHash = GeoTileUtils.longEncode(encodeDecodeLon(point.getX()), encodeDecodeLat(point.getY()), precision); + assertThat(tilerHash, equalTo(pointHash)); + } + } + + public void testGeoHash() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, 6); + assertThat(GEOHASH.encode(x, y, precision), equalTo(Geohash.longEncode(x, y, precision))); + + Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5)); + + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + + // test shape within tile bounds + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 5); + assertThat(count, equalTo(1)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 6); + assertThat(count, equalTo(32)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 7); + assertThat(count, equalTo(1024)); + } + } + + private boolean tileIntersectsBounds(int x, int y, int precision, GeoBoundingBox bounds) { + if (bounds == null) { + return true; + } + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bounds.right() < bounds.left()) { + boundsWestLeft = -180; + boundsWestRight = bounds.right(); + boundsEastLeft = bounds.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { + boundsEastLeft = bounds.left(); + boundsEastRight = bounds.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + Rectangle tile = GeoTileUtils.toBoundingBox(x, y, precision); + + return (bounds.top() >= tile.getMinY() && bounds.bottom() <= tile.getMaxY() + && (boundsEastLeft <= tile.getMaxX() && boundsEastRight >= tile.getMinX() + || (crossesDateline && boundsWestLeft <= tile.getMaxX() && boundsWestRight >= tile.getMinX()))); + } + + private int numTiles(MultiGeoShapeValues.GeoShapeValue geoValue, int precision, GeoBoundingBox geoBox) throws Exception { + MultiGeoShapeValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } else if ((bounds.top > LATITUDE_MASK && bounds.bottom > LATITUDE_MASK) + || (bounds.top < -LATITUDE_MASK && bounds.bottom < -LATITUDE_MASK)) { + return 0; + } + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return tileIntersectsBounds(minXTile, minYTile, precision, geoBox) ? 1 : 0; + } + + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } + + private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); + int recursiveCount; + { + recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); + int bruteForceCount; + { + final double tiles = 1 << precision; + MultiGeoShapeValues.BoundingBox bounds = value.boundingBox(); + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision, minXTile, minYTile, maxXTile, maxYTile); + } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 3); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); + int recursiveCount; + { + recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); + int bruteForceCount; + { + MultiGeoShapeValues.BoundingBox bounds = value.boundingBox(); + bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); + } + + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + + static Geometry boxToGeo(GeoBoundingBox geoBox) { + // turn into polygon + if (geoBox.right() < geoBox.left() && geoBox.right() != -180) { + return new MultiPolygon(List.of( + new Polygon(new LinearRing( + new double[] { -180, geoBox.right(), geoBox.right(), -180, -180 }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })), + new Polygon(new LinearRing( + new double[] { geoBox.left(), 180, 180, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })) + )); + } else { + double right = GeoUtils.normalizeLon(geoBox.right()); + return new Polygon(new LinearRing( + new double[] { geoBox.left(), right, right, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })); + } + } + + private int numTiles(MultiGeoShapeValues.GeoShapeValue geoValue, int precision) { + MultiGeoShapeValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } + + if ((bounds.top > NORMALIZED_LATITUDE_MASK && bounds.bottom > NORMALIZED_LATITUDE_MASK) + || (bounds.top < NORMALIZED_NEGATIVE_LATITUDE_MASK && bounds.bottom < NORMALIZED_NEGATIVE_LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return 1; + } + + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java new file mode 100644 index 0000000000000..b8ed5d8fab8f5 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -0,0 +1,300 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.apache.lucene.document.Document; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGrid; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeCoordinateEncoder; +import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeReader; +import org.elasticsearch.xpack.spatial.index.mapper.BinaryGeoShapeDocValuesField; +import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.randomBBox; +import static org.elasticsearch.xpack.spatial.util.GeoTestUtils.triangleTreeReader; +import static org.hamcrest.Matchers.equalTo; + +public abstract class GeoShapeGeoGridTestCase> extends AggregatorTestCase { + private static final String FIELD_NAME = "location"; + + /** + * Generate a random precision according to the rules of the given aggregation. + */ + protected abstract int randomPrecision(); + + /** + * Convert geo point into a hash string (bucket string ID) + */ + protected abstract String hashAsString(double lng, double lat, int precision); + + /** + * Return a point within the bounds of the tile grid + */ + protected abstract Point randomPoint(); + + /** + * Return the bounding tile as a {@link Rectangle} for a given point + */ + protected abstract Rectangle getTile(double lng, double lat, int precision); + + /** + * Create a new named {@link GeoGridAggregationBuilder}-derived builder + */ + protected abstract GeoGridAggregationBuilder createBuilder(String name); + + @Override + protected List getSearchPlugins() { + return List.of(new LocalStateSpatialPlugin()); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(GeoShapeValuesSourceType.instance()); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return createBuilder("foo").field(fieldName); + } + + public void testNoDocs() throws IOException { + testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, iw -> { + // Intentionally not writing any docs + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }); + } + + public void testUnmapped() throws IOException { + testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), null, iw -> { + iw.addDocument(Collections.singleton( + new BinaryGeoShapeDocValuesField(FIELD_NAME, GeoTestUtils.toDecodedTriangles(new Point(10D, 10D)), + new CentroidCalculator(new Point(10D, 10D))))); + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }); + } + + + public void testUnmappedMissingGeoShape() throws IOException { + // default value type for agg is GEOPOINT, so missing value is parsed as a GEOPOINT + GeoGridAggregationBuilder builder = createBuilder("_name") + .field("wrong_field") + .missing("-34.0,53.4"); + testCase(new MatchAllDocsQuery(), 1, null, + iw -> { + iw.addDocument(Collections.singleton( + new BinaryGeoShapeDocValuesField(FIELD_NAME, GeoTestUtils.toDecodedTriangles(new Point(10D, 10D)), + new CentroidCalculator(new Point(10D, 10D))))); + }, + geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), builder); + } + + public void testGeoShapeBounds() throws IOException { + final int precision = randomPrecision(); + final int numDocs = randomIntBetween(100, 200); + int numDocsWithin = 0; + final GeoGridAggregationBuilder builder = createBuilder("_name"); + + expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); + expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); + + GeoBoundingBox bbox = randomBBox(); + final double boundsTop = bbox.top(); + final double boundsBottom = bbox.bottom(); + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bbox.right() < bbox.left()) { + boundsWestLeft = -180; + boundsWestRight = bbox.right(); + boundsEastLeft = bbox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = bbox.left(); + boundsEastRight = bbox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + List docs = new ArrayList<>(); + List points = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + Point p; + p = randomPoint(); + double x = GeoTestUtils.encodeDecodeLon(p.getX()); + double y = GeoTestUtils.encodeDecodeLat(p.getY()); + Rectangle pointTile = getTile(x, y, precision); + + + TriangleTreeReader reader = triangleTreeReader(p, GeoShapeCoordinateEncoder.INSTANCE); + GeoRelation tileRelation = reader.relateTile(GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMinX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMinY()), + GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMaxX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMaxY())); + boolean intersectsBounds = boundsTop >= pointTile.getMinY() && boundsBottom <= pointTile.getMaxY() + && (boundsEastLeft <= pointTile.getMaxX() && boundsEastRight >= pointTile.getMinX() + || (crossesDateline && boundsWestLeft <= pointTile.getMaxX() && boundsWestRight >= pointTile.getMinX())); + if (tileRelation != GeoRelation.QUERY_DISJOINT && intersectsBounds) { + numDocsWithin += 1; + } + + + points.add(p); + docs.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(p), new CentroidCalculator(p))); + } + + final long numDocsInBucket = numDocsWithin; + + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> { + for (BinaryGeoShapeDocValuesField docField : docs) { + iw.addDocument(Collections.singletonList(docField)); + } + }, + geoGrid -> { + assertThat(AggregationInspectionHelper.hasValue(geoGrid), equalTo(numDocsInBucket > 0)); + long docCount = 0; + for (int i = 0; i < geoGrid.getBuckets().size(); i++) { + docCount += geoGrid.getBuckets().get(i).getDocCount(); + } + assertThat(docCount, equalTo(numDocsInBucket)); + }); + } + + public void testGeoShapeWithSeveralDocs() throws IOException { + int precision = randomIntBetween(1, 4); + int numShapes = randomIntBetween(8, 128); + Map expectedCountPerGeoHash = new HashMap<>(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, null, iw -> { + List shapes = new ArrayList<>(); + Document document = new Document(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int shapeId = 0; shapeId < numShapes; shapeId++) { + // undefined close to pole + double lat = (170.10225756d * randomDouble()) - 85.05112878d; + double lng = (360d * randomDouble()) - 180d; + + // Precision-adjust longitude/latitude to avoid wrong bucket placement + // Internally, lat/lng get converted to 32 bit integers, loosing some precision. + // This does not affect geohashing because geohash uses the same algorithm, + // but it does affect other bucketing algos, thus we need to do the same steps here. + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + + shapes.add(new Point(lng, lat)); + String hash = hashAsString(lng, lat, precision); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); + iw.addDocument(document); + shapes.clear(); + distinctHashesPerDoc.clear(); + document.clear(); + } + } + if (shapes.size() != 0) { + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); + iw.addDocument(document); + } + }, geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); + } + assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); + }); + } + + private void testCase(Query query, String field, int precision, GeoBoundingBox geoBoundingBox, + CheckedConsumer buildIndex, + Consumer> verify) throws IOException { + testCase(query, precision, geoBoundingBox, buildIndex, verify, createBuilder("_name").field(field)); + } + + @SuppressWarnings("unchecked") + private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, + CheckedConsumer buildIndex, + Consumer> verify, + GeoGridAggregationBuilder aggregationBuilder) throws IOException { + Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + buildIndex.accept(indexWriter); + indexWriter.close(); + + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + + aggregationBuilder.precision(precision); + if (geoBoundingBox != null) { + aggregationBuilder.setGeoBoundingBox(geoBoundingBox); + assertThat(aggregationBuilder.geoBoundingBox(), equalTo(geoBoundingBox)); + } + + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName(FIELD_NAME); + + Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); + aggregator.preCollection(); + indexSearcher.search(query, aggregator); + aggregator.postCollection(); + verify.accept((InternalGeoGrid) aggregator.buildAggregation(0L)); + + indexReader.close(); + directory.close(); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java new file mode 100644 index 0000000000000..01d7cd0e0fdeb --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoHashGridAggregatorTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGridBucket; + +import static org.elasticsearch.geometry.utils.Geohash.stringEncode; + +public class GeoShapeGeoHashGridAggregatorTests extends GeoShapeGeoGridTestCase { + + @Override + protected int randomPrecision() { + return randomIntBetween(1, 12); + } + + @Override + protected String hashAsString(double lng, double lat, int precision) { + return stringEncode(lng, lat, precision); + } + + @Override + protected Point randomPoint() { + return GeometryTestUtils.randomPoint(false); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return Geohash.toBoundingBox(stringEncode(lng, lat, precision)); + } + + @Override + protected GeoGridAggregationBuilder createBuilder(String name) { + return new GeoHashGridAggregationBuilder(name); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java new file mode 100644 index 0000000000000..5ab81dfe90089 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoTileGridAggregatorTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGridBucket; + +public class GeoShapeGeoTileGridAggregatorTests extends GeoShapeGeoGridTestCase { + + @Override + protected int randomPrecision() { + return randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + } + + @Override + protected String hashAsString(double lng, double lat, int precision) { + return GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)); + } + + @Override + protected Point randomPoint() { + return new Point(randomDoubleBetween(GeoUtils.MIN_LON, GeoUtils.MAX_LON, true), + randomDoubleBetween(-GeoTileUtils.LATITUDE_MASK, GeoTileUtils.LATITUDE_MASK, false)); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return GeoTileUtils.toBoundingBox(GeoTileUtils.longEncode(lng, lat, precision)); + } + + @Override + protected GeoGridAggregationBuilder createBuilder(String name) { + return new GeoTileGridAggregationBuilder(name); + } + + public void testPrecision() { + final GeoGridAggregationBuilder builder = createBuilder("_name"); + + expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); + expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); + + int precision = randomIntBetween(0, 29); + builder.precision(precision); + assertEquals(precision, builder.precision()); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/util/GeoTestUtils.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/util/GeoTestUtils.java index a903d5b11867f..65393b2e24898 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/util/GeoTestUtils.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/util/GeoTestUtils.java @@ -9,10 +9,13 @@ import org.apache.lucene.document.ShapeField; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoJson; +import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -22,10 +25,17 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeCoordinateEncoder; +import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeReader; +import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeWriter; import java.io.IOException; +import java.util.Arrays; import java.util.List; public class GeoTestUtils { @@ -45,6 +55,13 @@ public static ShapeField.DecodedTriangle[] toDecodedTriangles(Geometry geometry) return triangles; } + + public static GeoBoundingBox randomBBox() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoBoundingBox(new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())); + } + public static double encodeDecodeLat(double lat) { return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); } @@ -66,4 +83,14 @@ public static Geometry fromGeoJsonString(String geoJson) throws Exception { Geometry geometry = new GeometryParser(true, true, true).parse(parser); return new GeoShapeIndexer(true, "indexer").prepareForIndexing(geometry); } + + public static TriangleTreeReader triangleTreeReader(Geometry geometry, GeoShapeCoordinateEncoder encoder) throws IOException { + ShapeField.DecodedTriangle[] triangles = toDecodedTriangles(geometry); + TriangleTreeWriter writer = new TriangleTreeWriter(Arrays.asList(triangles), encoder, new CentroidCalculator(geometry)); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + writer.writeTo(output); + TriangleTreeReader reader = new TriangleTreeReader(encoder); + reader.reset(new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size()))); + return reader; + } } diff --git a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml new file mode 100644 index 0000000000000..2b7471bdf858e --- /dev/null +++ b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml @@ -0,0 +1,57 @@ +--- +"Test geotile_grid aggregation on geo_shape field": + - do: + indices.create: + index: locations + body: + mappings: + properties: + location: + type: geo_shape + + - do: + bulk: + refresh: true + body: + - index: + _index: locations + _id: 1 + - '{"location": "POINT(4.912350 52.374081)", "city": "Amsterdam", "name": "NEMO Science Museum"}' + - index: + _index: locations + _id: 2 + - '{"location": "POINT(4.901618 52.369219)", "city": "Amsterdam", "name": "Museum Het Rembrandthuis"}' + - index: + _index: locations + _id: 3 + - '{"location": "POINT(4.914722 52.371667)", "city": "Amsterdam", "name": "Nederlands Scheepvaartmuseum"}' + - index: + _index: locations + _id: 4 + - '{"location": "POINT(4.405200 51.222900)", "city": "Antwerp", "name": "Letterenhuis"}' + - index: + _index: locations + _id: 5 + - '{"location": "POINT(2.336389 48.861111)", "city": "Paris", "name": "Musée du Louvre"}' + - index: + _index: locations + _id: 6 + - '{"location": "POINT(2.327000 48.860000)", "city": "Paris", "name": "Musée dOrsay"}' + + - do: + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geotile_grid: + precision: 5 + field: location + - match: {hits.total: 6 } + - length: { aggregations.grid.buckets: 2 } + - match: { aggregations.grid.buckets.0.key: "5/16/10" } + - match: { aggregations.grid.buckets.0.doc_count: 4 } + - match: { aggregations.grid.buckets.1.key: "5/16/11" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } diff --git a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml new file mode 100644 index 0000000000000..4f41f6b75c481 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml @@ -0,0 +1,59 @@ +--- +"Test geohash_grid aggregation on geo_shape field": + - do: + indices.create: + index: locations + body: + mappings: + properties: + location: + type: geo_shape + + - do: + bulk: + refresh: true + body: + - index: + _index: locations + _id: 1 + - '{"location": "POINT(4.912350 52.374081)", "city": "Amsterdam", "name": "NEMO Science Museum"}' + - index: + _index: locations + _id: 2 + - '{"location": "POINT(4.901618 52.369219)", "city": "Amsterdam", "name": "Museum Het Rembrandthuis"}' + - index: + _index: locations + _id: 3 + - '{"location": "POINT(4.914722 52.371667)", "city": "Amsterdam", "name": "Nederlands Scheepvaartmuseum"}' + - index: + _index: locations + _id: 4 + - '{"location": "POINT(4.405200 51.222900)", "city": "Antwerp", "name": "Letterenhuis"}' + - index: + _index: locations + _id: 5 + - '{"location": "POINT(2.336389 48.861111)", "city": "Paris", "name": "Musée du Louvre"}' + - index: + _index: locations + _id: 6 + - '{"location": "POINT(2.327000 48.860000)", "city": "Paris", "name": "Musée dOrsay"}' + + - do: + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geohash_grid: + precision: 4 + field: location + - match: {hits.total: 6 } + - length: { aggregations.grid.buckets: 3 } + - match: { aggregations.grid.buckets.0.key: "u173" } + - match: { aggregations.grid.buckets.0.doc_count: 3 } + - match: { aggregations.grid.buckets.1.key: "u09t" } + - match: { aggregations.grid.buckets.1.doc_count: 2 } + - match: { aggregations.grid.buckets.2.key: "u155" } + - match: { aggregations.grid.buckets.2.doc_count: 1 } From 1dee6ebed310d1c53e7656ff462b6e10a7f41eb6 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 29 Apr 2020 13:11:03 -0700 Subject: [PATCH 2/7] fix style --- .../search/aggregations/bucket/geogrid/GeoTileUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 125385b1d016b..41e6d6ba08e52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -180,7 +180,8 @@ public static long longEncode(String hashAsString) { return longEncode((long) parsed[0], (long) parsed[1], (long) parsed[2]); } - public static long longEncodeTiles(int precision, long xTile, long yTile) { // Zoom value is placed in front of all the bits used for the geotile + public static long longEncodeTiles(int precision, long xTile, long yTile) { + // Zoom value is placed in front of all the bits used for the geotile // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile; From b2d9ed0c68d2e943c1ad6ff7984c4d48c2571b3f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 29 Apr 2020 13:57:43 -0700 Subject: [PATCH 3/7] add missing mapped test case --- .../bucket/geogrid/GeoShapeGeoGridTestCase.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java index b8ed5d8fab8f5..e348eff8ee2d4 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; import org.apache.lucene.document.Document; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -15,6 +16,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.geometry.Geometry; @@ -132,6 +134,17 @@ public void testUnmappedMissingGeoShape() throws IOException { geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), builder); } + public void testMappedMissingGeoShape() throws IOException { + GeoGridAggregationBuilder builder = createBuilder("_name") + .field(FIELD_NAME) + .missing("LINESTRING (30 10, 10 30, 40 40)"); + testCase(new MatchAllDocsQuery(), 1, null, + iw -> { + iw.addDocument(Collections.singleton(new SortedSetDocValuesField("string", new BytesRef("a")))); + }, + geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), builder); + } + public void testGeoShapeBounds() throws IOException { final int precision = randomPrecision(); final int numDocs = randomIntBetween(100, 200); From c8198e0276474c63583cbd02c97039c22c948e1c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 30 Apr 2020 07:53:23 -0700 Subject: [PATCH 4/7] fix scoping of registration --- .../org/elasticsearch/xpack/spatial/SpatialPlugin.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index d6d8050d4f677..6c3ee9bd47d13 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -104,14 +104,14 @@ public Map getProcessors(Processor.Parameters paramet return Map.of(CircleProcessor.TYPE, new CircleProcessor.Factory()); } - public static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { + private static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { builder.register(GeoBoundsAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (GeoBoundsAggregatorSupplier) (name, aggregationContext, parent, valuesSource, wrapLongitude, metadata) -> new GeoShapeBoundsAggregator(name, aggregationContext, parent, (GeoShapeValuesSource) valuesSource, wrapLongitude, metadata)); } - public void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder builder) { + private void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder builder) { builder.register(GeoCentroidAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (GeoCentroidAggregatorSupplier) (name, aggregationContext, parent, valuesSource, metadata) -> { @@ -122,7 +122,7 @@ public void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder buil }); } - public void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder) { + private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder) { builder.register(GeoHashGridAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (GeoGridAggregatorSupplier) (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, aggregationContext, parent, metadata) -> { @@ -158,13 +158,13 @@ public void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder }); } - static void registerValueCountAggregator(ValuesSourceRegistry.Builder builder) { + private static void registerValueCountAggregator(ValuesSourceRegistry.Builder builder) { builder.register(ValueCountAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (ValueCountAggregatorSupplier) ValueCountAggregator::new ); } - static void registerCardinalityAggregator(ValuesSourceRegistry.Builder builder) { + private static void registerCardinalityAggregator(ValuesSourceRegistry.Builder builder) { builder.register(CardinalityAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (CardinalityAggregatorSupplier) CardinalityAggregator::new); } From c7dbb6b03bf702a13bd4ab6eefb19db4d3bb45f3 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 30 Apr 2020 21:03:18 -0700 Subject: [PATCH 5/7] add circuit-breaking logic --- .../bucket/geogrid/GeoGridAggregator.java | 3 +- x-pack/plugin/spatial/build.gradle | 1 + .../xpack/spatial/SpatialPlugin.java | 16 ++-- .../bucket/geogrid/AllCellValues.java | 12 +-- .../ByteTrackingSortingNumericDocValues.java | 16 ++++ .../bucket/geogrid/GeoShapeCellIdSource.java | 16 +++- .../bucket/geogrid/GeoShapeCellValues.java | 19 +++-- .../geogrid/GeoShapeHashGridAggregator.java | 34 +++++++++ .../geogrid/GeoShapeTileGridAggregator.java | 34 +++++++++ .../bucket/geogrid/GeoGridTilerTests.java | 76 +++++++++++++++---- .../geogrid/GeoShapeGeoGridTestCase.java | 1 + .../rest-api-spec/test/30_geotile_grid.yml | 47 ++++++++++++ .../rest-api-spec/test/40_geohash_grid.yml | 46 +++++++++++ 13 files changed, 286 insertions(+), 35 deletions(-) create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java index 1f72698ac7a26..c7e75b27bce31 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java @@ -46,6 +46,7 @@ public abstract class GeoGridAggregator extends Bucke protected final int shardSize; protected final ValuesSource.Numeric valuesSource; protected final LongHash bucketOrds; + protected SortedNumericDocValues values; GeoGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, int requiredSize, int shardSize, SearchContext aggregationContext, @@ -68,7 +69,7 @@ public ScoreMode scoreMode() { @Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException { - final SortedNumericDocValues values = valuesSource.longValues(ctx); + values = valuesSource.longValues(ctx); return new LeafBucketCollectorBase(sub, null) { @Override public void collect(int doc, long bucket) throws IOException { diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index e6630238e78f3..be6c976c346bf 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -27,6 +27,7 @@ restResources { testClusters.integTest { setting 'xpack.license.self_generated.type', 'trial' + setting 'indices.breaker.request.limit', '25kb' testDistribution = 'DEFAULT' } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 6c3ee9bd47d13..0acda1af41be3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -17,9 +17,7 @@ import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregator; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregator; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregator; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregatorSupplier; @@ -46,6 +44,8 @@ import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeCellIdSource; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeHashGridAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; @@ -134,8 +134,10 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde tiler = new BoundedGeoHashGridTiler(geoBoundingBox); } GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); - return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, - parent, metadata); + GeoShapeHashGridAggregator agg = new GeoShapeHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, + aggregationContext, parent, metadata); + cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes); + return agg; } throw LicenseUtils.newComplianceException("geohash_grid aggregation on geo_shape fields"); }); @@ -151,8 +153,10 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde tiler = new BoundedGeoTileGridTiler(geoBoundingBox); } GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); - return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, - parent, metadata); + GeoShapeTileGridAggregator agg = new GeoShapeTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, + aggregationContext, parent, metadata); + cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes); + return agg; } throw LicenseUtils.newComplianceException("geotile_grid aggregation on geo_shape fields"); }); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java index acd0c96d3b668..556d245eb61a4 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/AllCellValues.java @@ -6,24 +6,20 @@ package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; -import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; import java.io.IOException; +import java.util.function.Consumer; /** Sorted numeric doc values for precision 0 */ -class AllCellValues extends AbstractSortingNumericDocValues { +class AllCellValues extends ByteTrackingSortingNumericDocValues { private MultiGeoShapeValues geoValues; - protected AllCellValues(MultiGeoShapeValues geoValues, GeoGridTiler tiler) { + protected AllCellValues(MultiGeoShapeValues geoValues, GeoGridTiler tiler, Consumer circuitBreakerConsumer) { this.geoValues = geoValues; resize(1); values[0] = tiler.encode(0, 0, 0); - } - - // for testing - protected long[] getValues() { - return values; + circuitBreakerConsumer.accept((long) Long.BYTES); } @Override diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java new file mode 100644 index 0000000000000..bee9bc3a217fd --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/ByteTrackingSortingNumericDocValues.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; + +abstract class ByteTrackingSortingNumericDocValues extends AbstractSortingNumericDocValues { + + public long getValuesBytes() { + return values.length * Long.BYTES; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java index 2d09c042a84d4..669213a5f1acc 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java @@ -16,10 +16,13 @@ import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import java.util.function.Consumer; + public class GeoShapeCellIdSource extends ValuesSource.Numeric { private final GeoShapeValuesSource valuesSource; private final int precision; private final GeoGridTiler encoder; + private Consumer circuitBreakerConsumer; public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, int precision, GeoGridTiler encoder) { this.valuesSource = valuesSource; @@ -27,6 +30,15 @@ public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, int precision, Ge this.encoder = encoder; } + /** + * This setter exists since the aggregator's circuit-breaking accounting needs to be + * accessible from within the values-source. Problem is that this values-source needs to + * be created and passed to the aggregator before we have access to this functionality. + */ + public void setCircuitBreakerConsumer(Consumer circuitBreakerConsumer) { + this.circuitBreakerConsumer = circuitBreakerConsumer; + } + public int precision() { return precision; } @@ -41,12 +53,12 @@ public SortedNumericDocValues longValues(LeafReaderContext ctx) { MultiGeoShapeValues geoValues = valuesSource.geoShapeValues(ctx); if (precision == 0) { // special case, precision 0 is the whole world - return new AllCellValues(geoValues, encoder); + return new AllCellValues(geoValues, encoder, circuitBreakerConsumer); } ValuesSourceType vs = geoValues.valuesSourceType(); if (GeoShapeValuesSourceType.instance() == vs) { // docValues are geo shapes - return new GeoShapeCellValues(geoValues, precision, encoder); + return new GeoShapeCellValues(geoValues, precision, encoder, circuitBreakerConsumer); } else { throw new IllegalArgumentException("unsupported geo type"); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java index 9d39492e1774f..fc2472ecb4721 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellValues.java @@ -6,21 +6,25 @@ package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; -import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; import java.io.IOException; +import java.util.function.Consumer; /** Sorted numeric doc values for geo shapes */ -class GeoShapeCellValues extends AbstractSortingNumericDocValues { - private MultiGeoShapeValues geoShapeValues; +class GeoShapeCellValues extends ByteTrackingSortingNumericDocValues { + private final MultiGeoShapeValues geoShapeValues; + private final Consumer circuitBreakerConsumer; protected int precision; protected GeoGridTiler tiler; - protected GeoShapeCellValues(MultiGeoShapeValues geoShapeValues, int precision, GeoGridTiler tiler) { + protected GeoShapeCellValues(MultiGeoShapeValues geoShapeValues, int precision, GeoGridTiler tiler, + Consumer circuitBreakerConsumer) { this.geoShapeValues = geoShapeValues; this.precision = precision; this.tiler = tiler; + this.circuitBreakerConsumer = circuitBreakerConsumer; + circuitBreakerConsumer.accept((long) Long.BYTES); } @Override @@ -46,7 +50,13 @@ protected void add(int idx, long value) { } void resizeCell(int newSize) { + int oldValuesLength = values.length; resize(newSize); + int newValuesLength = values.length; + if (newValuesLength > oldValuesLength) { + long bytesDiff = (newValuesLength - oldValuesLength) * Long.BYTES; + circuitBreakerConsumer.accept(bytesDiff); + } } /** @@ -59,6 +69,5 @@ void resizeCell(int newSize) { int advanceValue(MultiGeoShapeValues.GeoShapeValue target) { return tiler.setValues(this, target, precision); } - } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java new file mode 100644 index 0000000000000..ae6c660df6a3b --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeHashGridAggregator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +public class GeoShapeHashGridAggregator extends GeoHashGridAggregator { + public GeoShapeHashGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, int requiredSize, + int shardSize, SearchContext aggregationContext, Aggregator parent, + Map metadata) throws IOException { + super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, metadata); + } + + /** + * This is a wrapper method to expose this protected method to {@link GeoShapeCellIdSource} + * + * @param bytes the number of bytes to register or negative to deregister the bytes + * @return the cumulative size in bytes allocated by this aggregator to service this request + */ + public long addRequestBytes(long bytes) { + return addRequestCircuitBreakerBytes(bytes); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java new file mode 100644 index 0000000000000..3c925b2324414 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeTileGridAggregator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +public class GeoShapeTileGridAggregator extends GeoTileGridAggregator { + public GeoShapeTileGridAggregator(String name, AggregatorFactories factories, ValuesSource.Numeric valuesSource, int requiredSize, + int shardSize, SearchContext aggregationContext, Aggregator parent, + Map metadata) throws IOException { + super(name, factories, valuesSource, requiredSize, shardSize, aggregationContext, parent, metadata); + } + + /** + * This is a wrapper method to expose this protected method to {@link GeoShapeCellIdSource} + * + * @param bytes the number of bytes to register or negative to deregister the bytes + * @return the cumulative size in bytes allocated by this aggregator to service this request + */ + public long addRequestBytes(long bytes) { + return addRequestCircuitBreakerBytes(bytes); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 44b47fdd74b33..f7925afae3477 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -6,18 +6,26 @@ package org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.indices.breaker.BreakerSettings; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.spatial.index.fielddata.GeoRelation; @@ -25,8 +33,10 @@ import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; import org.elasticsearch.xpack.spatial.index.fielddata.TriangleTreeReader; +import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.function.Consumer; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_LATITUDE_MASK; @@ -40,6 +50,7 @@ public class GeoGridTilerTests extends ESTestCase { private static final GeoTileGridTiler GEOTILE = new GeoTileGridTiler(); private static final GeoHashGridTiler GEOHASH = new GeoHashGridTiler(); + private static final Consumer NOOP_BREAKER = (l) -> {}; public void testGeoTile() throws Exception { double x = randomDouble(); @@ -56,17 +67,17 @@ public void testGeoTile() throws Exception { // test shape within tile bounds { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int count = GEOTILE.setValues(values, value, 13); assertThat(count, equalTo(1)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int count = GEOTILE.setValues(values, value, 14); assertThat(count, equalTo(4)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int count = GEOTILE.setValues(values, value, 15); assertThat(count, equalTo(16)); } @@ -107,7 +118,7 @@ public void testGeoTileSetValuesBoundingBoxes_BoundedGeoShapeCellValues() throws TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); GeoBoundingBox geoBoundingBox = randomBBox(); MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); - GeoShapeCellValues cellValues = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues cellValues = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int numTiles = new BoundedGeoTileGridTiler(geoBoundingBox).setValues(cellValues, value, precision); int expected = numTiles(value, precision, geoBoundingBox); @@ -133,7 +144,7 @@ public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() thro TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); - GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); int expected = numTiles(value, precision); assertThat(numTiles, equalTo(expected)); @@ -166,7 +177,7 @@ public void testTilerMatchPoint() throws Exception { } TriangleTreeReader reader = triangleTreeReader(point, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); - GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); assertThat(numTiles, equalTo(1)); long tilerHash = unboundedCellValues.getValues()[0]; @@ -190,17 +201,17 @@ public void testGeoHash() throws Exception { // test shape within tile bounds { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH, NOOP_BREAKER); int count = GEOHASH.setValues(values, value, 5); assertThat(count, equalTo(1)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH, NOOP_BREAKER); int count = GEOHASH.setValues(values, value, 6); assertThat(count, equalTo(32)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH, NOOP_BREAKER); int count = GEOHASH.setValues(values, value, 7); assertThat(count, equalTo(1024)); } @@ -306,12 +317,12 @@ private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Ex geometry = indexer.prepareForIndexing(geometry); TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); - GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int recursiveCount; { recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); } - GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE, NOOP_BREAKER); int bruteForceCount; { final double tiles = 1 << precision; @@ -336,12 +347,12 @@ private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Ex geometry = indexer.prepareForIndexing(geometry); TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); - GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH, NOOP_BREAKER); int recursiveCount; { recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); } - GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH, NOOP_BREAKER); int bruteForceCount; { MultiGeoShapeValues.BoundingBox bounds = value.boundingBox(); @@ -443,4 +454,43 @@ private int numTiles(MultiGeoShapeValues.GeoShapeValue geoValue, int precision) return count; } } + + public void testGeoHashGridCircuitBreaker() throws IOException { + testCircuitBreaker(GEOHASH); + } + + public void testGeoTileGridCircuitBreaker() throws IOException { + testCircuitBreaker(GEOTILE); + } + + private void testCircuitBreaker(GeoGridTiler tiler) throws IOException { + MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); + int precision = randomIntBetween(0, 6); + TriangleTreeReader reader = triangleTreeReader(multiPoint, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoShapeValues.GeoShapeValue value = new MultiGeoShapeValues.GeoShapeValue(reader); + + final long numBytes; + if (precision == 0) { + AllCellValues values = new AllCellValues(null, tiler, NOOP_BREAKER); + numBytes = values.getValuesBytes(); + } else { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, tiler, NOOP_BREAKER); + tiler.setValues(values, value, precision); + numBytes = values.getValuesBytes(); + } + + CircuitBreakerService service = new HierarchyCircuitBreakerService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)); + service.registerBreaker(new BreakerSettings("limited", numBytes - 1, 1.0)); + CircuitBreaker limitedBreaker = service.getBreaker("limited"); + + Consumer circuitBreakerConsumer = (l) -> limitedBreaker.addEstimateBytesAndMaybeBreak(l, "agg"); + expectThrows(CircuitBreakingException.class, () -> { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, tiler, circuitBreakerConsumer); + tiler.setValues(values, value, precision); + assertThat(values.getValuesBytes(), equalTo(numBytes)); + assertThat(limitedBreaker.getUsed(), equalTo(numBytes)); + }); + + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java index e348eff8ee2d4..6c54c24f45e75 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -305,6 +305,7 @@ private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, aggregator.preCollection(); indexSearcher.search(query, aggregator); aggregator.postCollection(); + verify.accept((InternalGeoGrid) aggregator.buildAggregation(0L)); indexReader.close(); diff --git a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml index 2b7471bdf858e..b36ee664043d6 100644 --- a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml +++ b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/30_geotile_grid.yml @@ -55,3 +55,50 @@ - match: { aggregations.grid.buckets.0.doc_count: 4 } - match: { aggregations.grid.buckets.1.key: "5/16/11" } - match: { aggregations.grid.buckets.1.doc_count: 2 } + +--- +"Test geotile_grid aggregation circuit breaker": + - do: + indices.create: + index: locations + body: + mappings: + properties: + location: + type: geo_shape + + - do: + index: + index: locations + id: box + body: { location: "POLYGON((-173.671875 -77.8418477505252,173.32031249999997 -77.8418477505252,173.32031249999997 83.52016238353205,-173.671875 83.52016238353205,-173.671875 -77.8418477505252))" } + + - do: + indices.refresh: {} + + - do: + catch: /data for \[\] would be \[28648\/27.9kb\], which is larger than the limit of \[25600\/25kb\]/ + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geotile_grid: + precision: 11 + field: location + + - do: + catch: /data for \[\] would be \[28648\/27.9kb\], which is larger than the limit of \[25600\/25kb\]/ + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geotile_grid: + precision: 11 + field: location + diff --git a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml index 4f41f6b75c481..e9f4b17f341a8 100644 --- a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml +++ b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/40_geohash_grid.yml @@ -57,3 +57,49 @@ - match: { aggregations.grid.buckets.1.doc_count: 2 } - match: { aggregations.grid.buckets.2.key: "u155" } - match: { aggregations.grid.buckets.2.doc_count: 1 } + +--- +"Test geohash_grid aggregation circuit breaker": + - do: + indices.create: + index: locations + body: + mappings: + properties: + location: + type: geo_shape + + - do: + index: + index: locations + id: box + body: { location: "POLYGON((-173.671875 -77.8418477505252,173.32031249999997 -77.8418477505252,173.32031249999997 83.52016238353205,-173.671875 83.52016238353205,-173.671875 -77.8418477505252))" } + + - do: + indices.refresh: {} + + - do: + catch: /data for \[\] would be \[26344\/25.7kb\], which is larger than the limit of \[25600\/25kb\]/ + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geotile_grid: + precision: 10 + field: location + + - do: + catch: /data for \[\] would be \[26344\/25.7kb\], which is larger than the limit of \[25600\/25kb\]/ + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + grid: + geotile_grid: + precision: 10 + field: location From d0feb6026abf9ea78cdadc2c1ed1d5d3d32845b0 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 30 Apr 2020 21:04:18 -0700 Subject: [PATCH 6/7] set initial breaker consumer to be a no-op --- .../search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java index 669213a5f1acc..4d7a7de573039 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeCellIdSource.java @@ -28,6 +28,7 @@ public GeoShapeCellIdSource(GeoShapeValuesSource valuesSource, int precision, Ge this.valuesSource = valuesSource; this.precision = precision; this.encoder = encoder; + this.circuitBreakerConsumer = (l) -> {}; } /** From 83da59bf13777c315297b8a2f6df079950402dfa Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 May 2020 14:31:55 -0700 Subject: [PATCH 7/7] fix bug in tiling optimization --- .../xpack/spatial/SpatialPlugin.java | 2 ++ .../bucket/geogrid/GeoHashGridTiler.java | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 0acda1af41be3..2a68215975a9d 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -136,6 +136,7 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); GeoShapeHashGridAggregator agg = new GeoShapeHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, parent, metadata); + // this would ideally be something set in an immutable way on the ValuesSource cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes); return agg; } @@ -155,6 +156,7 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource((GeoShapeValuesSource) valuesSource, precision, tiler); GeoShapeTileGridAggregator agg = new GeoShapeTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, aggregationContext, parent, metadata); + // this would ideally be something set in an immutable way on the ValuesSource cellIdSource.setCircuitBreakerConsumer(agg::addRequestBytes); return agg; } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java index bc9fb4920e47d..806f8f807f404 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoHashGridTiler.java @@ -27,16 +27,16 @@ public int setValues(GeoShapeCellValues values, MultiGeoShapeValues.GeoShapeValu MultiGeoShapeValues.BoundingBox bounds = geoValue.boundingBox(); assert bounds.minX() <= bounds.maxX(); - long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); - long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); - long count = (numLonCells + 1) * (numLatCells + 1); - if (count == 1) { + + // TODO: optimize for when a whole shape (not just point) fits in a single tile an + // for when brute-force is expected to be faster than rasterization, which + // is when the number of tiles expected is less than the precision + + // optimization for setting just one value for when the shape represents a point + if (bounds.minX() == bounds.maxX() && bounds.minY() == bounds.maxY()) { return setValue(values, geoValue, bounds, precision); - } else if (count <= precision) { - return setValuesByBruteForceScan(values, geoValue, precision, bounds); - } else { - return setValuesByRasterization("", values, 0, precision, geoValue); } + return setValuesByRasterization("", values, 0, precision, geoValue); } /**