Skip to content

Commit

Permalink
Add support for pmtiles output format (#55)
Browse files Browse the repository at this point in the history 
* add pmtiles.hpp from github.com/protomaps/PMTiles [#10]

* tippecanoe main writes pmtiles output. [#10]

* detect output format using suffix
* after mbtiles is done writing, replace with pmtiles based on map/image tables.
* add method to write_json for writing json sub-object.

* tippecanoe-decode reads pmtiles input. [#10]

* tile-join reads and writes pmtiles. [#10]

* pmtiles test suite for decode and tile-join [#10]

* add base GitHub CI action for compiling and test suite.

* update pmtiles.hpp with z>15 fix

* Fix some ordering problems with pmtiles decode

* Pmtiles should also pass the raw tiles tests

* Eradicate spaces from tileset metadata JSON fields

* Eradicate spaces from more test fixtures

* Update more tests

* Pmtiles tests pass now too

* Remove unnecessary sort (and make indent)

* Update changelog

* The allow-existing test for pmtiles needs -o, not -e

* Declare --allow-existing to be unsupported for pmtiles.

It was always a bad idea even for mbtiles.

Co-authored-by: Brandon Liu <[email protected]>
  • Loading branch information
@e-n-f @bdon
e-n-f and bdon authored Dec 29, 2022
1 parent a374263 commit 3b2599f
Show file tree
Hide file tree
Showing 197 changed files with 2,515 additions and 323 deletions.
11 changes: 11 additions & 0 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
name: test

on: [push]

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: make
- run: make test
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## 2.17.0

* Add pmtiles output format

## 2.16.0

* During tiling, limit the size of the statistics that are kept for -as-needed calculations, because they can get quite large for sources with hundreds of millions of features.
Expand Down
72 changes: 67 additions & 5 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ C = $(wildcard *.c) $(wildcard *.cpp)
INCLUDES = -I/usr/local/include -I.
LIBS = -L/usr/local/lib

tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o
tippecanoe: geojson.o jsonpull/jsonpull.o tile.o pool.o mbtiles.o geometry.o projection.o memfile.o mvt.o serial.o main.o text.o dirtiles.o pmtiles_file.o plugin.o read_json.o write_json.o geobuf.o flatgeobuf.o evaluator.o geocsv.o csv.o geojson-loop.o json_logger.o visvalingam.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-enumerate: enumerate.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lsqlite3

tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o
tippecanoe-decode: decode.o projection.o mvt.o write_json.o text.o jsonpull/jsonpull.o dirtiles.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3

tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o
tile-join: tile-join.o projection.o pool.o mbtiles.o mvt.o memfile.o dirtiles.o jsonpull/jsonpull.o text.o evaluator.o csv.o write_json.o pmtiles_file.o
$(CXX) $(PG) $(LIBS) $(FINAL_FLAGS) $(CXXFLAGS) -o $@ $^ $(LDFLAGS) -lm -lz -lsqlite3 -lpthread

tippecanoe-json-tool: jsontool.o jsonpull/jsonpull.o csv.o text.o geojson-loop.o
Expand All @@ -82,7 +82,7 @@ indent:
TESTS = $(wildcard tests/*/out/*.json)
SPACE = $(NULL) $(NULL)

test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test
test: tippecanoe tippecanoe-decode $(addsuffix .check,$(TESTS)) raw-tiles-test parallel-test pbf-test join-test enumerate-test decode-test join-filter-test unit json-tool-test allow-existing-test csv-test layer-json-test pmtiles-test decode-pmtiles-test
./unit

suffixes = json json.gz
Expand Down Expand Up @@ -156,6 +156,43 @@ raw-tiles-test:
cmp tests/raw-tiles/nothing.json.check tests/raw-tiles/nothing.json
rm -r tests/raw-tiles/nothing tests/raw-tiles/nothing.json.check

pmtiles-test:
./tippecanoe -q -f -o tests/pmtiles/hackspots.pmtiles -r1 -pC tests/raw-tiles/hackspots.geojson
./tippecanoe-decode -x generator tests/pmtiles/hackspots.pmtiles > tests/pmtiles/hackspots.json.check
cmp tests/pmtiles/hackspots.json.check tests/pmtiles/hackspots.json
# Test generating pmtiles first and then converting to mbtiles with tile-join.
./tile-join -q -f -pC -o tests/pmtiles/joined.mbtiles tests/pmtiles/hackspots.pmtiles
./tippecanoe-decode -x generator tests/pmtiles/joined.mbtiles > tests/pmtiles/joined.json.check
cmp tests/pmtiles/joined.json.check tests/pmtiles/joined.json
rm -r tests/pmtiles/hackspots.json.check tests/pmtiles/hackspots.pmtiles

# Test generating mbtiles first and then converting to pmtiles with tile-join. (Changes bounds)
./tippecanoe -q -f -o tests/pmtiles/hackspots.mbtiles -r1 -pC tests/raw-tiles/hackspots.geojson
./tile-join -q -f -pC -o tests/pmtiles/joined.pmtiles tests/pmtiles/hackspots.mbtiles

# decode changes order (ZXY vs TMS order)
./tippecanoe-decode -x generator tests/pmtiles/joined.pmtiles > tests/pmtiles/joined_reordered.json.check
cmp tests/pmtiles/joined_reordered.json.check tests/pmtiles/joined_reordered.json
rm -r tests/pmtiles/joined_reordered.json.check tests/pmtiles/hackspots.mbtiles tests/pmtiles/joined.pmtiles

# From raw-tiles-test:
./tippecanoe -q -f -o tests/raw-tiles/raw-tiles.pmtiles -r1 -pC tests/raw-tiles/hackspots.geojson
./tippecanoe-decode -x generator tests/raw-tiles/raw-tiles.pmtiles | sed 's/\.pmtiles//g' | sed 's/ -o / -e /g' > tests/raw-tiles/raw-tiles.json.check
cmp tests/raw-tiles/raw-tiles.json.check tests/raw-tiles/raw-tiles.json
# Test that -z and -Z work in tippecanoe-decode
./tippecanoe-decode -x generator -Z6 -z7 tests/raw-tiles/raw-tiles.pmtiles | sed 's/\.pmtiles//g' | sed 's/ -o / -e /g' > tests/raw-tiles/raw-tiles-z67.json.check
cmp tests/raw-tiles/raw-tiles-z67.json.check tests/raw-tiles/raw-tiles-z67.json
# Test that -z and -Z work in tile-join
./tile-join -q -f -Z6 -z7 -o tests/raw-tiles/raw-tiles-z67.pmtiles tests/raw-tiles/raw-tiles.pmtiles
./tippecanoe-decode -x generator tests/raw-tiles/raw-tiles-z67.pmtiles | sed 's/\.pmtiles//g' | sed 's/ -o / -e /g' > tests/raw-tiles/raw-tiles-z67-join.json.check
cmp tests/raw-tiles/raw-tiles-z67-join.json.check tests/raw-tiles/raw-tiles-z67-join.json
rm -rf tests/raw-tiles/raw-tiles.pmtiles tests/raw-tiles/raw-tiles-z67.pmtiles tests/raw-tiles/raw-tiles.json.check raw-tiles-z67.json.check tests/raw-tiles/raw-tiles-z67-join.json.check
# Test that metadata.json is created even if all features are clipped away
./tippecanoe -q -f -o tests/raw-tiles/nothing.pmtiles tests/raw-tiles/nothing.geojson
./tippecanoe-decode -x generator tests/raw-tiles/nothing.pmtiles | sed 's/\.pmtiles//g' | sed 's/ -o / -e /g' > tests/raw-tiles/nothing.json.check
cmp tests/raw-tiles/nothing.json.check tests/raw-tiles/nothing.json
rm -r tests/raw-tiles/nothing.pmtiles tests/raw-tiles/nothing.json.check

decode-test:
mkdir -p tests/muni/decode
./tippecanoe -q -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json
Expand All @@ -173,6 +210,23 @@ decode-test:
cmp tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json
rm -f tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.onetile.json.check

decode-pmtiles-test:
mkdir -p tests/muni/decode
./tippecanoe -q -z11 -Z11 -f -o tests/muni/decode/multi.pmtiles tests/muni/*.json
./tippecanoe-decode -x generator -l subway tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.json.check
./tippecanoe-decode -x generator -l subway --integer tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.integer.json.check
./tippecanoe-decode -x generator -l subway --fraction tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.fraction.json.check
./tippecanoe-decode -x generator -c tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.pipeline.json.check
./tippecanoe-decode -x generator tests/muni/decode/multi.pmtiles 11 327 791 | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.onetile.json.check
./tippecanoe-decode -x generator --stats tests/muni/decode/multi.pmtiles | sed 's/pmtiles/mbtiles/g' > tests/muni/decode/multi.pmtiles.stats.json.check
cmp tests/muni/decode/multi.pmtiles.json.check tests/muni/decode/multi.mbtiles.json
cmp tests/muni/decode/multi.pmtiles.integer.json.check tests/muni/decode/multi.mbtiles.integer.json
cmp tests/muni/decode/multi.pmtiles.fraction.json.check tests/muni/decode/multi.mbtiles.fraction.json
cmp tests/muni/decode/multi.pmtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.pipeline.json
cmp tests/muni/decode/multi.pmtiles.onetile.json.check tests/muni/decode/multi.mbtiles.onetile.json
cmp tests/muni/decode/multi.pmtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json
rm -f tests/muni/decode/multi.pmtiles.json.check tests/muni/decode/multi.pmtiles tests/muni/decode/multi.pmtiles.pipeline.json.check tests/muni/decode/multi.pmtiles.stats.json.check tests/muni/decode/multi.pmtiles.onetile.json.check

pbf-test:
./tippecanoe-decode -x generator tests/pbf/11-328-791.vector.pbf 11 328 791 > tests/pbf/11-328-791.vector.pbf.out
cmp tests/pbf/11-328-791.json tests/pbf/11-328-791.vector.pbf.out
Expand Down Expand Up @@ -318,7 +372,15 @@ allow-existing-test:
./tippecanoe -q -Z10 -z11 -F -e tests/allow-existing/both.dir tests/coalesce-tract/tl_2010_06001_tract10.json
./tippecanoe-decode -x generator -x generator_options tests/allow-existing/both.dir | sed 's/both\.dir/both.mbtiles/g' > tests/allow-existing/both.dir.json.check
cmp tests/allow-existing/both.dir.json.check tests/allow-existing/both.mbtiles.json
rm -r tests/allow-existing/both.dir.json.check tests/allow-existing/both.dir tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles
# Make a tileset
./tippecanoe -q -Z0 -z0 -f -o tests/allow-existing/both.pmtiles tests/coalesce-tract/tl_2010_06001_tract10.json
# Writing to existing should fail
if ./tippecanoe -q -Z1 -z1 -o tests/allow-existing/both.pmtiles tests/coalesce-tract/tl_2010_06001_tract10.json; then exit 1; else exit 0; fi
# Replace existing
./tippecanoe -q -Z8 -z9 -f -o tests/allow-existing/both.pmtiles tests/coalesce-tract/tl_2010_06001_tract10.json
# Allow-existing is not supported for pmtiles
if ./tippecanoe -q -Z10 -z11 -F -o tests/allow-existing/both.pmtiles tests/coalesce-tract/tl_2010_06001_tract10.json; then exit 1; else exit 0; fi
rm -r tests/allow-existing/both.pmtiles tests/allow-existing/both.dir.json.check tests/allow-existing/both.dir tests/allow-existing/both.mbtiles.json.check tests/allow-existing/both.mbtiles

csv-test:
# Reading from named CSV
Expand Down
102 changes: 77 additions & 25 deletions decode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@
#include <sys/stat.h>
#include <sys/mman.h>
#include <protozero/pbf_reader.hpp>
#include <sys/stat.h>
#include "mvt.hpp"
#include "projection.hpp"
#include "geometry.hpp"
#include "write_json.hpp"
#include "jsonpull/jsonpull.h"
#include "mbtiles.hpp"
#include "dirtiles.hpp"
#include "pmtiles_file.hpp"
#include "errors.hpp"

int minzoom = 0;
Expand Down Expand Up @@ -245,7 +245,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
if (st.st_size < 50 * 1024 * 1024) {
char *map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
if (map != NULL && map != MAP_FAILED) {
if (strcmp(map, "SQLite format 3") != 0) {
if (strcmp(map, "SQLite format 3") != 0 && strncmp(map, "PMTiles", 7) != 0) {
if (z >= 0) {
std::string s = std::string(map, st.st_size);
handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
Expand All @@ -272,11 +272,30 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

struct stat st;
std::vector<zxy> tiles;

char *pmtiles_map;
std::vector<pmtiles::entry_zxy> entries;
bool is_pmtiles = false;

if (stat(fname, &st) == 0 && (st.st_mode & S_IFDIR) != 0) {
isdir = true;

db = dirmeta2tmp(fname);
tiles = enumerate_dirtiles(fname, minzoom, maxzoom);
} else if (pmtiles_has_suffix(fname)) {
int pmtiles_fd = open(fname, O_RDONLY | O_CLOEXEC);
pmtiles_map = (char *) mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, pmtiles_fd, 0);
if (pmtiles_map == MAP_FAILED) {
perror("mmap in decode");
exit(EXIT_MEMORY);
}
if (close(pmtiles_fd) != 0) {
perror("close");
exit(EXIT_CLOSE);
}
db = pmtilesmeta2tmp(fname, pmtiles_map);
entries = pmtiles_entries_tms(pmtiles_map, minzoom, maxzoom);
is_pmtiles = true;
} else {
if (sqlite3_open(fname, &db) != SQLITE_OK) {
fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
Expand Down Expand Up @@ -382,6 +401,26 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co

handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode);
}
} else if (is_pmtiles) {
within = 0;

for (auto const &entry : entries) {
if (!pipeline && !stats) {
if (within) {
state.json_comma_newline();
}
within = 1;
}
if (stats) {
if (within) {
state.json_comma_newline();
}
within = 1;
}

std::string s{pmtiles_map + entry.offset, entry.length};
handle(s, entry.z, entry.x, entry.y, to_decode, pipeline, stats, state, coordinate_mode);
}
} else {
const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
sqlite3_stmt *stmt;
Expand Down Expand Up @@ -447,36 +486,49 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
} else {
int handled = 0;
while (z >= 0 && !handled) {
const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}
if (is_pmtiles) {
uint64_t tile_offset;
uint32_t tile_length;
std::tie(tile_offset, tile_length) = pmtiles_get_tile(pmtiles_map, z, x, y);
if (tile_length > 0) {
if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}
std::string s{pmtiles_map + tile_offset, tile_length};
handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
}
} else {
const char *sql = "SELECT tile_data from tiles where zoom_level = ? and tile_column = ? and tile_row = ?;";
sqlite3_stmt *stmt;
if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
exit(EXIT_SQLITE);
}

sqlite3_bind_int(stmt, 1, z);
sqlite3_bind_int(stmt, 2, x);
sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y);
sqlite3_bind_int(stmt, 1, z);
sqlite3_bind_int(stmt, 2, x);
sqlite3_bind_int(stmt, 3, (1LL << z) - 1 - y);

while (sqlite3_step(stmt) == SQLITE_ROW) {
int len = sqlite3_column_bytes(stmt, 0);
const char *s = (const char *) sqlite3_column_blob(stmt, 0);
while (sqlite3_step(stmt) == SQLITE_ROW) {
int len = sqlite3_column_bytes(stmt, 0);
const char *s = (const char *) sqlite3_column_blob(stmt, 0);

if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
exit(EXIT_SQLITE);
}
if (s == NULL) {
fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
exit(EXIT_SQLITE);
}

if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}
if (z != oz) {
fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
}

handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
handled = 1;
}
sqlite3_finalize(stmt);
}

sqlite3_finalize(stmt);

z--;
x /= 2;
y /= 2;
Expand Down
9 changes: 4 additions & 5 deletions dirtiles.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,8 +309,7 @@ void dir_write_metadata(const char *outdir, const metadata &m) {
out(state, "minzoom", std::to_string(m.minzoom));
out(state, "maxzoom", std::to_string(m.maxzoom));
out(state, "center", std::to_string(m.center_lon) + "," + std::to_string(m.center_lat) + "," + std::to_string(m.center_z));
out(state, "bounds", std::to_string(m.minlon) + "," + std::to_string(m.minlat) + "," +
std::to_string(m.maxlon) + "," + std::to_string(m.maxlat));
out(state, "bounds", std::to_string(m.minlon) + "," + std::to_string(m.minlat) + "," + std::to_string(m.maxlon) + "," + std::to_string(m.maxlat));
out(state, "type", m.type);
if (m.attribution.size() > 0) {
out(state, "attribution", m.attribution);
Expand All @@ -326,14 +325,14 @@ void dir_write_metadata(const char *outdir, const metadata &m) {
std::string json = "{";

if (m.vector_layers_json.size() > 0) {
json += "\"vector_layers\": " + m.vector_layers_json;
json += "\"vector_layers\":" + m.vector_layers_json;

if (m.tilestats_json.size() > 0) {
json += ",\"tilestats\": " + m.tilestats_json;
json += ",\"tilestats\":" + m.tilestats_json;
}
} else {
if (m.tilestats_json.size() > 0) {
json += "\"tilestats\": " + m.tilestats_json;
json += "\"tilestats\":" + m.tilestats_json;
}
}

Expand Down
2 changes: 1 addition & 1 deletion geometry.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1622,7 +1622,7 @@ drawvec checkerboard_anchors(drawvec const &geom, int tx, int ty, int z, unsigne
// upper left of tile in world coordinates
long long tx1 = 0, ty1 = 0;
// lower right of tile in world coordinates;
long long tx2 = 1LL << 32; // , ty2 = 1LL << 32;
long long tx2 = 1LL << 32; // , ty2 = 1LL << 32;
if (z != 0) {
tx1 = (long long) tx << (32 - z);
ty1 = (long long) ty << (32 - z);
Expand Down
Loading

0 comments on commit 3b2599f

Please sign in to comment.