Tippecanoe-decode and tippecanoe-overzoom changes to improve binning …

…performance (#292)

* Add a tippecanoe-decode option to restrict which attributes to decode

* Plumb buffer and feature limit around

* Check the feature limit

* Clarifying cases where output detail can be unspecified

* Clip bins to the tile buffer instead of just passing them through

* Add missing include

* Missed some tests

* Add --no-tile-compression option to tippecanoe-overzoom

* Update version and changelog

CHANGELOG.md

@@ -1,3 +1,11 @@
+# 2.68.0
+
+* Adds `--no-tile-compression` option to `tippecanoe-overzoom`
+* Make `tippecanoe-overzoom` clip the output tile to the tile buffer after assigning points to bins
+* Adds `--include`/`-y` option to `tippecanoe-decode` to decode only specified attributes
+* Cleans up some inconsistent handling of variable tile extents in overzooming code
+* Speeds up overzooming slightly in `tile-join` by doing less preflighting to discover which child tiles contain features
+
 # 2.67.0
 
 * Reduce memory consumption of duplicate attribute names in `serial_feature`

Makefile

@@ -291,8 +291,8 @@ overzoom-test: tippecanoe-overzoom
 	./tippecanoe-decode tests/pbf/13-1310-3166.pbf 13 1310 3166 > tests/pbf/13-1310-3166.pbf.json.check
 	cmp tests/pbf/13-1310-3166.pbf.json.check tests/pbf/13-1310-3166.pbf.json
 	rm tests/pbf/13-1310-3166.pbf tests/pbf/13-1310-3166.pbf.json.check
-	# Multiple inputs
-	./tippecanoe-overzoom -o tests/pbf/13-1310-3166-ne.pbf -t 13/1310/3166 tests/pbf/11-327-791.pbf 11/327/791 tests/pbf/0-0-0.pbf 0/0/0
+	# Multiple inputs, no compression
+	./tippecanoe-overzoom --no-tile-compression -o tests/pbf/13-1310-3166-ne.pbf -t 13/1310/3166 tests/pbf/11-327-791.pbf 11/327/791 tests/pbf/0-0-0.pbf 0/0/0
 	./tippecanoe-decode tests/pbf/13-1310-3166-ne.pbf 13 1310 3166 > tests/pbf/13-1310-3166-ne.pbf.json.check
 	cmp tests/pbf/13-1310-3166-ne.pbf.json.check tests/pbf/13-1310-3166-ne.pbf.json
 	rm tests/pbf/13-1310-3166-ne.pbf tests/pbf/13-1310-3166-ne.pbf.json.check

clip.cpp

@@ -1076,7 +1076,7 @@ bool pnpoly_mp(std::vector<mvt_geometry> const &geom, long long x, long long y)
 }
 
 std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int ny,
-		     int detail, int buffer,
+		     int detail_or_unspecified, int buffer,
 		     std::set<std::string> const &keep,
 		     std::set<std::string> const &exclude,
 		     std::vector<std::string> const &exclude_prefix,
@@ -1087,7 +1087,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     std::vector<std::string> const &unidecode_data, double simplification,
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
-		     std::string const &accumulate_numeric) {
+		     std::string const &accumulate_numeric, size_t feature_limit) {
 	std::vector<source_tile> decoded;
 
 	for (auto const &t : tiles) {
@@ -1113,7 +1113,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		decoded.push_back(out);
 	}
 
-	return overzoom(decoded, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric);
+	return overzoom(decoded, nz, nx, ny, detail_or_unspecified, buffer, keep, exclude, exclude_prefix, do_compress, next_overzoomed_tiles, demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, feature_limit);
 }
 
 // like a minimal serial_feature, but with mvt_feature-style attributes
@@ -1284,18 +1284,54 @@ static bool should_keep(std::string const &key,
 	return false;
 }
 
+static void handle_closepath_from_mvt(drawvec &geom) {
+	// mvt geometries close polygons with a mvt_closepath operation
+	// tippecanoe-internal geometries close polygons with a lineto to the initial point
+
+	size_t last_open = 0;
+
+	for (size_t i = 0; i < geom.size(); i++) {
+		if (geom[i].op == mvt_closepath) {
+			geom[i] = draw(mvt_lineto, geom[last_open].x, geom[last_open].y);
+		} else if (geom[i].op == mvt_moveto) {
+			last_open = i;
+		}
+	}
+}
+
 static void feature_out(std::vector<tile_feature> const &features, mvt_layer &outlayer,
 			std::set<std::string> const &keep,
 			std::set<std::string> const &exclude,
 			std::vector<std::string> const &exclude_prefix,
 			std::unordered_map<std::string, attribute_op> const &attribute_accum,
 			std::string const &accumulate_numeric,
-			key_pool &key_pool) {
+			key_pool &key_pool, int buffer) {
 	// Add geometry to output feature
 
+	drawvec geom = features[0].geom;
+	if (buffer >= 0) {
+		int t = features[0].t;
+
+		if (t == VT_LINE) {
+			geom = clip_lines(geom, 32 - outlayer.detail(), buffer);
+		} else if (t == VT_POLYGON) {
+			drawvec dv;
+			handle_closepath_from_mvt(geom);
+			geom = simple_clip_poly(geom, 32 - outlayer.detail(), buffer, dv, false);
+		} else if (t == VT_POINT) {
+			geom = clip_point(geom, 32 - outlayer.detail(), buffer);
+		}
+
+		geom = remove_noop(geom, t, 0);
+		if (t == VT_POLYGON) {
+			geom = clean_or_clip_poly(geom, 0, 0, false, false);
+			geom = close_poly(geom);
+		}
+	}
+
 	mvt_feature outfeature;
 	outfeature.type = features[0].t;
-	for (auto const &g : features[0].geom) {
+	for (auto const &g : geom) {
 		outfeature.geometry.emplace_back(g.op, g.x, g.y);
 	}
 
@@ -1516,22 +1552,27 @@ static std::vector<size_t> parse_ids_string(mvt_value const &v) {
 
 mvt_tile assign_to_bins(mvt_tile &features,
 			std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
-			int z, int x, int y, int detail,
+			int z, int x, int y,
 			std::unordered_map<std::string, attribute_op> const &attribute_accum,
 			std::string const &accumulate_numeric,
 			std::set<std::string> keep,
 			std::set<std::string> exclude,
-			std::vector<std::string> exclude_prefix) {
+			std::vector<std::string> exclude_prefix,
+			int buffer) {
 	std::vector<index_event> events;
 	key_pool key_pool;
 
+	if (bins.size() == 0) {
+		return mvt_tile();
+	}
+
 	// Index bins
 	for (size_t i = 0; i < bins.size(); i++) {
 		for (size_t j = 0; j < bins[i].features.size(); j++) {
 			long long xmin, ymin, xmax, ymax;
 			unsigned long long start, end;
 
-			get_bbox(bins[i].features[j].geometry, &xmin, &ymin, &xmax, &ymax, z, x, y, detail);
+			get_bbox(bins[i].features[j].geometry, &xmin, &ymin, &xmax, &ymax, z, x, y, bins[i].detail());
 			get_quadkey_bounds(xmin, ymin, xmax, ymax, &start, &end);
 			events.emplace_back(start, index_event::ENTER, i, j, xmin, ymin, xmax, ymax);
 			events.emplace_back(end, index_event::EXIT, i, j, xmin, ymin, xmax, ymax);
@@ -1551,7 +1592,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 					fid_to_feature.emplace(features.layers[i].features[j].id, std::make_pair(i, j));
 				}
 
-				get_bbox(features.layers[i].features[j].geometry, &xmin, &ymin, &xmax, &ymax, z, x, y, detail);
+				get_bbox(features.layers[i].features[j].geometry, &xmin, &ymin, &xmax, &ymax, z, x, y, features.layers[i].detail());
 				get_quadkey_bounds(xmin, ymin, xmax, ymax, &start, &end);
 				events.emplace_back(start, index_event::CHECK, i, j, xmin, ymin, xmax, ymax);
 			}
@@ -1562,7 +1603,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 	std::set<active_bin> active;
 
 	mvt_layer outlayer;
-	outlayer.extent = 1 << detail;
+	outlayer.extent = bins[0].extent;
 	outlayer.version = 2;
 	outlayer.name = features.layers[0].name;
 
@@ -1680,7 +1721,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 		if (outfeatures[i].size() > 1) {
 			feature_out(outfeatures[i], outlayer,
 				    keep, exclude, exclude_prefix, attribute_accum,
-				    accumulate_numeric, key_pool);
+				    accumulate_numeric, key_pool, buffer);
 			mvt_feature &nfeature = outlayer.features.back();
 			mvt_value val;
 			val.type = mvt_uint;
@@ -1702,7 +1743,7 @@ mvt_tile assign_to_bins(mvt_tile &features,
 }
 
 std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int ny,
-		     int detail, int buffer,
+		     int detail_or_unspecified, int buffer,
 		     std::set<std::string> const &keep,
 		     std::set<std::string> const &exclude,
 		     std::vector<std::string> const &exclude_prefix,
@@ -1713,15 +1754,15 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     std::vector<std::string> const &unidecode_data, double simplification,
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
-		     std::string const &accumulate_numeric) {
+		     std::string const &accumulate_numeric, size_t feature_limit) {
 	mvt_tile outtile;
 	key_pool key_pool;
 
 	for (auto const &tile : tiles) {
 		for (auto const &layer : tile.tile.layers) {
 			mvt_layer *outlayer = NULL;
 
-			int det = detail;
+			int det = detail_or_unspecified;
 			if (det <= 0) {
 				det = std::round(log(layer.extent) / log(2));
 			}
@@ -1840,7 +1881,10 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 
 				if (flush_multiplier_cluster) {
 					if (pending_tile_features.size() > 0) {
-						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool);
+						feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1);
+						if (outlayer->features.size() >= feature_limit) {
+							break;
+						}
 						pending_tile_features.clear();
 					}
 				}
@@ -1854,16 +1898,16 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 				if (t == VT_POLYGON && tiny_polygon_size > 0) {
 					bool simplified_away_by_reduction = false;
 
-					geom = reduce_tiny_poly(geom, nz, detail, &still_need_simplification_after_reduction, &simplified_away_by_reduction, &accum_area, tiny_polygon_size);
+					geom = reduce_tiny_poly(geom, nz, det, &still_need_simplification_after_reduction, &simplified_away_by_reduction, &accum_area, tiny_polygon_size);
 				} else {
 					still_need_simplification_after_reduction = true;
 				}
 
 				if (simplification > 0 && still_need_simplification_after_reduction) {
 					if (t == VT_POLYGON) {
-						geom = simplify_lines_basic(geom, nz, detail, simplification, 4);
+						geom = simplify_lines_basic(geom, nz, det, simplification, 4);
 					} else if (t == VT_LINE) {
-						geom = simplify_lines_basic(geom, nz, detail, simplification, 0);
+						geom = simplify_lines_basic(geom, nz, det, simplification, 0);
 					}
 				}
 
@@ -1897,8 +1941,11 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 			}
 
 			if (pending_tile_features.size() > 0) {
-				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool);
+				feature_out(pending_tile_features, *outlayer, keep, exclude, exclude_prefix, attribute_accum, accumulate_numeric, key_pool, -1);
 				pending_tile_features.clear();
+				if (outlayer->features.size() >= feature_limit) {
+					break;
+				}
 			}
 
 			if (preserve_input_order) {
@@ -1927,10 +1974,14 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 					std::vector<source_tile> sts;
 					sts.push_back(st);
 
+					// feature_limit arg is 1, because we just care whether there are any overzoomed features
+					// left after clipping to the child tile, not about their actual content
 					std::string child = overzoom(sts,
 								     nz + 1, nx * 2 + x, ny * 2 + y,
-								     detail, buffer, keep, exclude, exclude_prefix, false, NULL,
-								     demultiply, filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric);
+								     detail_or_unspecified, buffer, keep, exclude, exclude_prefix, false, NULL,
+								     demultiply, filter, preserve_input_order, attribute_accum, unidecode_data,
+								     simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric,
+								     1);
 					if (child.size() > 0) {
 						next_overzoomed_tiles->emplace_back(nx * 2 + x, ny * 2 + y);
 					}
@@ -1940,8 +1991,9 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 	}
 
 	if (bins.size() > 0) {
-		outtile = assign_to_bins(outtile, bins, bin_by_id_list, nz, nx, ny, detail, attribute_accum, accumulate_numeric,
-					 keep, exclude, exclude_prefix);
+		outtile = assign_to_bins(outtile, bins, bin_by_id_list, nz, nx, ny,
+					 attribute_accum, accumulate_numeric,
+					 keep, exclude, exclude_prefix, buffer);
 	}
 
 	for (ssize_t i = outtile.layers.size() - 1; i >= 0; i--) {

decode.cpp

@@ -28,6 +28,7 @@
 int minzoom = 0;
 int maxzoom = 32;
 bool force = false;
+std::set<std::string> include_attr;
 
 bool progress_time() {
 	return false;
@@ -217,7 +218,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
 		} else if (coordinate_mode == 2) {  // integer
 			scale = 1;
 		}
-		layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state, scale);
+		layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state, scale, include_attr);
 
 		if (!pipeline) {
 			if (true) {
@@ -571,6 +572,7 @@ int main(int argc, char **argv) {
 		{"stats", no_argument, 0, 'S'},
 		{"force", no_argument, 0, 'f'},
 		{"exclude-metadata-row", required_argument, 0, 'x'},
+		{"include", required_argument, 0, 'y'},
 		{0, 0, 0, 0},
 	};
 
@@ -630,6 +632,10 @@ int main(int argc, char **argv) {
 			exclude_meta.insert(optarg);
 			break;
 
+		case 'y':
+			include_attr.insert(optarg);
+			break;
+
 		default:
 			usage(argv);
 		}

geometry.hpp

@@ -127,7 +127,7 @@ std::string overzoom(std::vector<source_tile> const &tiles, int nz, int nx, int
 		     std::vector<std::string> const &unidecode_data, double simplification,
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
-		     std::string const &accumulate_numeric);
+		     std::string const &accumulate_numeric, size_t feature_limit);
 
 std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int ny,
 		     int detail, int buffer,
@@ -141,7 +141,7 @@ std::string overzoom(std::vector<input_tile> const &tiles, int nz, int nx, int n
 		     std::vector<std::string> const &unidecode_data, double simplification,
 		     double tiny_polygon_size,
 		     std::vector<mvt_layer> const &bins, std::string const &bin_by_id_list,
-		     std::string const &accumulate_numeric);
+		     std::string const &accumulate_numeric, size_t feature_limit);
 
 draw center_of_mass_mp(const drawvec &dv);
 

mvt.hpp

@@ -9,6 +9,7 @@
 #include <vector>
 #include <optional>
 #include <memory>
+#include <cmath>
 
 #include "errors.hpp"
 #include "text.hpp"
@@ -201,6 +202,10 @@ struct mvt_layer {
 	// For tracking the key-value constants already used in this layer
 	std::vector<ssize_t> key_dedup = std::vector<ssize_t>(65536, -1);
 	std::vector<ssize_t> value_dedup = std::vector<ssize_t>(65536, -1);
+
+	int detail() const {
+		return std::round(std::log(extent) / std::log(2));
+	}
 };
 
 struct mvt_tile {

overzoom.cpp

@@ -17,6 +17,8 @@ extern int optind;
 int detail = 12;  // tippecanoe-style: mvt extent == 1 << detail
 int buffer = 5;	  // tippecanoe-style: mvt buffer == extent * buffer / 256;
 bool demultiply = false;
+bool do_compress = true;
+
 std::string filter;
 bool preserve_input_order = false;
 std::unordered_map<std::string, attribute_op> attribute_accum;
@@ -65,6 +67,7 @@ int main(int argc, char **argv) {
 		{"assign-to-bins", required_argument, 0, 'b' & 0x1F},
 		{"bin-by-id-list", required_argument, 0, 'c' & 0x1F},
 		{"accumulate-numeric-attributes", required_argument, 0, 'a' & 0x1F},
+		{"no-tile-compression", no_argument, 0, 'd' & 0x1F},
 
 		{0, 0, 0, 0},
 	};
@@ -151,6 +154,10 @@ int main(int argc, char **argv) {
 			accumulate_numeric = optarg;
 			break;
 
+		case 'd' & 0x1F:
+			do_compress = false;
+			break;
+
 		default:
 			fprintf(stderr, "Unrecognized flag -%c\n", i);
 			usage(argv);
@@ -221,7 +228,11 @@ int main(int argc, char **argv) {
 			exit(EXIT_OPEN);
 		}
 
-		bins = parse_layers(f, nz, nx, ny, 1LL << detail, true);
+		int det = detail;
+		if (det < 0) {
+			det = 12;
+		}
+		bins = parse_layers(f, nz, nx, ny, 1LL << det, true);
 		fclose(f);
 	}
 
@@ -251,7 +262,7 @@ int main(int argc, char **argv) {
 		its.push_back(std::move(t));
 	}
 
-	std::string out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, true, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric);
+	std::string out = overzoom(its, nz, nx, ny, detail, buffer, keep, exclude, exclude_prefix, do_compress, NULL, demultiply, json_filter, preserve_input_order, attribute_accum, unidecode_data, simplification, tiny_polygon_size, bins, bin_by_id_list, accumulate_numeric, SIZE_MAX);
 
 	FILE *f = fopen(outfile, "wb");
 	if (f == NULL) {

plugin.cpp

@@ -55,7 +55,7 @@ void *run_writer(void *a) {
 
 	json_writer state(fp);
 	for (size_t i = 0; i < wa->layers->size(); i++) {
-		layer_to_geojson((*(wa->layers))[i], wa->z, wa->x, wa->y, false, true, false, true, 0, 0, 0, true, state, 0);
+		layer_to_geojson((*(wa->layers))[i], wa->z, wa->x, wa->y, false, true, false, true, 0, 0, 0, true, state, 0, std::set<std::string>());
 	}
 
 	if (fclose(fp) != 0) {