Add a feature to extract elements from array attributes by zoom level

README.md

@@ -405,6 +405,7 @@ resolution is obtained than by using a smaller _maxzoom_ or _detail_.
  * `-pe` or `--empty-csv-columns-are-null`: Treat empty CSV columns as nulls rather than as empty strings.
  * `-aI` or `--convert-stringified-ids-to-numbers`: If a feature ID is the string representation of a number, convert it to a plain number to use as the feature ID.
  * `--use-attribute-for-id=`*name*: Use the attribute with the specified *name* as if it were specified as the feature ID. (If this attribute is a stringified number, you must also use `-aI` to convert it to a number.)
+ * `--zoom-element=`*name*: If the attribute with the specified *name* is a JSON list, when producing zoom level *z*, use the *z*'th element of the array as the attribute value.
 
 ### Filtering features by attributes
 

main.cpp

@@ -95,6 +95,7 @@ static long long diskfree;
 char **av;
 
 std::vector<clipbbox> clipbboxes;
+std::set<std::string> zoom_elements;
 
 void checkdisk(std::vector<struct reader> *r) {
 	long long used = 0;
@@ -2538,6 +2539,7 @@ int main(int argc, char **argv) {
 		{"empty-csv-columns-are-null", no_argument, &prevent[P_EMPTY_CSV_COLUMNS], 1},
 		{"convert-stringified-ids-to-numbers", no_argument, &additional[A_CONVERT_NUMERIC_IDS], 1},
 		{"use-attribute-for-id", required_argument, 0, '~'},
+		{"zoom-element", required_argument, 0, '~'},
 
 		{"Filtering features by attributes", 0, 0, 0},
 		{"feature-filter-file", required_argument, 0, 'J'},
@@ -2697,6 +2699,8 @@ int main(int argc, char **argv) {
 				}
 			} else if (strcmp(opt, "use-attribute-for-id") == 0) {
 				attribute_for_id = optarg;
+			} else if (strcmp(opt, "zoom-element") == 0) {
+				zoom_elements.insert(optarg);
 			} else {
 				fprintf(stderr, "%s: Unrecognized option --%s\n", argv[0], opt);
 				exit(EXIT_FAILURE);

main.hpp

@@ -4,6 +4,7 @@
 #include <stddef.h>
 #include <atomic>
 #include <string>
+#include <set>
 
 struct index {
 	long long start = 0;
@@ -32,6 +33,7 @@ struct clipbbox {
 };
 
 extern std::vector<clipbbox> clipbboxes;
+extern std::set<std::string> zoom_elements;
 
 void checkdisk(std::vector<struct reader> *r);
 

tests/zoom-element/in.json

@@ -0,0 +1,2 @@
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": [ 5, 4, 3, 2, 1, 0 ], "another": [ true, false, { "boo": "eek" }, [ 1, 2, 3 ], "yes", "no" ], "list": [ 1, 2, 3], "compound": { "one": "two" } }, "geometry": { "type": "Point", "coordinates": [ 1,1 ]} }
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 2,2 ]} }

tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract.json

@@ -0,0 +1,77 @@
+{ "type": "FeatureCollection", "properties": {
+"bounds": "1.000000,1.000000,2.000000,2.000000",
+"center": "1.406250,1.405686,7",
+"description": "tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract.json.check.mbtiles",
+"format": "pbf",
+"generator_options": "./tippecanoe -q -a@ -f -o tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract.json.check.mbtiles -B0 -b0 -z7 --zoom-element another --zoom-element extract tests/zoom-element/in.json",
+"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 7, \"fields\": {\"another\": \"Mixed\", \"compound\": \"String\", \"extract\": \"Mixed\", \"list\": \"String\", \"no\": \"Boolean\", \"yes\": \"Boolean\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 2,\"geometry\": \"Point\",\"attributeCount\": 6,\"attributes\": [{\"attribute\": \"another\",\"count\": 7,\"type\": \"mixed\",\"values\": [\"[1,2,3]\",\"[true,false,{\\\"boo\\\":\\\"eek\\\"},[1,2,3],\\\"yes\\\",\\\"no\\\"]\",false,\"no\",true,\"yes\",\"{\\\"boo\\\":\\\"eek\\\"}\"]},{\"attribute\": \"compound\",\"count\": 1,\"type\": \"string\",\"values\": [\"{\\\"one\\\":\\\"two\\\"}\"]},{\"attribute\": \"extract\",\"count\": 7,\"type\": \"mixed\",\"values\": [0,1,2,3,4,5,\"[5,4,3,2,1,0]\"],\"min\": 0,\"max\": 5},{\"attribute\": \"list\",\"count\": 1,\"type\": \"string\",\"values\": [\"[1,2,3]\"]},{\"attribute\": \"no\",\"count\": 1,\"type\": \"boolean\",\"values\": [false]},{\"attribute\": \"yes\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]}]}}",
+"maxzoom": "7",
+"minzoom": "0",
+"name": "tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract.json.check.mbtiles",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": true, "extract": 5 }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.054628 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.933594, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": false, "extract": 4 }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.977539, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "{\"boo\":\"eek\"}", "extract": 3 }, "geometry": { "type": "Point", "coordinates": [ 0.988770, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 3 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "[1,2,3]", "extract": 2 }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.010086 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 7 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "yes", "extract": 1 }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.005197 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.004596 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 5, "x": 16, "y": 15 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "no", "extract": 0 }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.002451 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.001851 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 6, "x": 32, "y": 31 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "no", "extract": 0 }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.001078 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.000478 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 7, "x": 64, "y": 63 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}", "another": "no", "extract": 0 }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.000392 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.000478 ] } }
+] }
+] }
+] }

tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract_-Ccat.json

@@ -0,0 +1,77 @@
+{ "type": "FeatureCollection", "properties": {
+"bounds": "1.000000,1.000000,2.000000,2.000000",
+"center": "1.406250,1.405686,7",
+"description": "tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract_-Ccat.json.check.mbtiles",
+"format": "pbf",
+"generator_options": "./tippecanoe -q -a@ -f -o tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract_-Ccat.json.check.mbtiles -B0 -b0 -z7 --zoom-element another --zoom-element extract -Ccat tests/zoom-element/in.json",
+"json": "{\"vector_layers\": [ { \"id\": \"in\", \"description\": \"\", \"minzoom\": 0, \"maxzoom\": 7, \"fields\": {\"another\": \"Mixed\", \"compound\": \"String\", \"extract\": \"Mixed\", \"list\": \"String\", \"no\": \"Boolean\", \"yes\": \"Boolean\"} } ],\"tilestats\": {\"layerCount\": 1,\"layers\": [{\"layer\": \"in\",\"count\": 16,\"geometry\": \"Point\",\"attributeCount\": 6,\"attributes\": [{\"attribute\": \"another\",\"count\": 7,\"type\": \"mixed\",\"values\": [\"[1,2,3]\",\"[true,false,{\\\"boo\\\":\\\"eek\\\"},[1,2,3],\\\"yes\\\",\\\"no\\\"]\",false,\"no\",true,\"yes\",\"{\\\"boo\\\":\\\"eek\\\"}\"]},{\"attribute\": \"compound\",\"count\": 1,\"type\": \"string\",\"values\": [\"{\\\"one\\\":\\\"two\\\"}\"]},{\"attribute\": \"extract\",\"count\": 7,\"type\": \"mixed\",\"values\": [0,1,2,3,4,5,\"[5,4,3,2,1,0]\"],\"min\": 0,\"max\": 5},{\"attribute\": \"list\",\"count\": 1,\"type\": \"string\",\"values\": [\"[1,2,3]\"]},{\"attribute\": \"no\",\"count\": 1,\"type\": \"boolean\",\"values\": [false]},{\"attribute\": \"yes\",\"count\": 1,\"type\": \"boolean\",\"values\": [true]}]}]}}",
+"maxzoom": "7",
+"minzoom": "0",
+"name": "tests/zoom-element/out/-B0_-b0_-z7_--zoom-element_another_--zoom-element_extract_-Ccat.json.check.mbtiles",
+"type": "overlay",
+"version": "2"
+}, "features": [
+{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 5, "another": true, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.054628 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.933594, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 1, "x": 1, "y": 0 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 4, "another": false, "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.966797, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.977539, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 2, "x": 2, "y": 1 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 3, "another": "{\"boo\":\"eek\"}", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.988770, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.021065 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 3, "x": 4, "y": 3 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 2, "another": "[1,2,3]", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.010690 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.010086 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 4, "x": 8, "y": 7 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 1, "another": "yes", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.005197 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.004596 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 5, "x": 16, "y": 15 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 0, "another": "no", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.002451 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.001851 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 6, "x": 32, "y": 31 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 0, "another": "no", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.001078 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.000478 ] } }
+] }
+] }
+,
+{ "type": "FeatureCollection", "properties": { "zoom": 7, "x": 64, "y": 63 }, "features": [
+{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
+{ "type": "Feature", "properties": { "yes": true, "no": false, "extract": 0, "another": "no", "list": "[1,2,3]", "compound": "{\"one\":\"two\"}" }, "geometry": { "type": "Point", "coordinates": [ 0.999756, 1.000392 ] } }
+,
+{ "type": "Feature", "properties": {  }, "geometry": { "type": "Point", "coordinates": [ 1.999512, 2.000478 ] } }
+] }
+] }
+] }

tile.cpp

@@ -1710,6 +1710,58 @@ static bool line_is_too_small(drawvec const &geometry, int z, int detail) {
 	return true;
 }
 
+serial_val json_to_serial_val(json_object *o) {
+	serial_val sv;
+
+	if (o->type == JSON_NULL) {
+		sv.type = mvt_null;
+		sv.s = "null";
+	} else if (o->type == JSON_TRUE) {
+		sv.type = mvt_bool;
+		sv.s = "true";
+	} else if (o->type == JSON_FALSE) {
+		sv.type = mvt_bool;
+		sv.s = "false";
+	} else if (o->type == JSON_STRING) {
+		sv.type = mvt_string;
+		sv.s = o->string;
+	} else if (o->type == JSON_NUMBER) {
+		sv.type = mvt_double;
+		sv.s = o->string;
+	} else if (o->type == JSON_HASH || o->type == JSON_ARRAY) {
+		char *s = json_stringify(o);
+		sv.type = mvt_string;
+		sv.s = s;
+		free(s);
+	} else {
+		fprintf(stderr, "Can't happen converting JSON object %s\n", json_stringify(o));
+		exit(EXIT_FAILURE);
+	}
+
+	return sv;
+}
+
+serial_val get_zoom_element(int zoom, std::string key, std::string value) {
+	json_pull *jp = json_begin_string(value.c_str());
+	json_object *o = json_read_tree(jp);
+
+	if (o == NULL || o->type != JSON_ARRAY || o->length == 0) {
+		fprintf(stderr, "Can't parse JSON array \"%s\" for --zoom-element=\"%s\"\n", value.c_str(), key.c_str());
+		exit(EXIT_FAILURE);
+	}
+
+	if ((size_t) zoom >= o->length) {
+		zoom = o->length - 1;
+	}
+
+	serial_val sv = json_to_serial_val(o->array[zoom]);
+
+	json_free(o);
+	json_end(jp);
+
+	return sv;
+}
+
 long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, sqlite3 *outdb, const char *outdir, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, std::atomic<long long> *along, long long alongminus, double gamma, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, std::atomic<int> *running, double simplification, std::vector<std::map<std::string, layermap_entry>> *layermaps, std::vector<std::vector<std::string>> *layer_unmaps, size_t tiling_seg, size_t pass, size_t passes, unsigned long long mingap, long long minextent, double fraction, const char *prefilter, const char *postfilter, struct json_object *filter, write_tile_args *arg) {
 	int line_detail;
 	double merge_fraction = 1;
@@ -1863,6 +1915,29 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
 				break;
 			}
 
+			std::set<std::string> need_tilestats;
+			if (zoom_elements.size() > 0) {
+				for (size_t ii = 0; ii < sf.full_keys.size(); ii++) {
+					if (zoom_elements.count(sf.full_keys[ii]) != 0) {
+						serial_val val = get_zoom_element(z, sf.full_keys[ii], sf.full_values[ii].s);
+						sf.full_values[ii] = val;
+						need_tilestats.insert(sf.full_keys[ii]);
+					}
+				}
+				for (ssize_t ii = sf.keys.size() - 1; ii >= 0; ii--) {
+					char *key = stringpool + pool_off[sf.segment] + sf.keys[ii] + 1;
+					if (zoom_elements.count(key) != 0) {
+						serial_val val = get_zoom_element(z, key, std::string(stringpool + pool_off[sf.segment] + sf.values[ii] + 1));
+						sf.full_keys.push_back(key);
+						sf.full_values.push_back(val);
+
+						sf.keys.erase(sf.keys.begin() + ii);
+						sf.values.erase(sf.values.begin() + ii);
+						need_tilestats.insert(key);
+					}
+				}
+			}
+
 			if (sf.dropped) {
 				if (find_partial(partials, sf, which_partial, layer_unmaps)) {
 					preserve_attributes(arg->attribute_accum, attribute_accum_state, sf, stringpool, pool_off, partials[which_partial]);
@@ -1992,6 +2067,7 @@ long long write_tile(FILE *geoms, std::atomic<long long> *geompos_in, char *meta
 				p.renamed = -1;
 				p.extent = sf.extent;
 				p.clustered = 0;
+				p.need_tilestats = need_tilestats;
 				partials.push_back(p);
 			}