Skip to content

Commit

Permalink
feat: added option to save geometry in the wkt format
Browse files Browse the repository at this point in the history
  • Loading branch information
RaczeQ committed Mar 13, 2024
1 parent bbe3327 commit da1b0c1
Show file tree
Hide file tree
Showing 7 changed files with 153 additions and 35 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@ When the `keep_all_tags` parameter is passed while filtering by OSM tags, and ad
General schema of multiple segments that are concatenated together:
`pbf_file_name`\_(`osm_filter_tags_hash_part`/`nofilter`)(\_`alltags`)\_(`clipping_geometry_hash_part`/`noclip`)\_(`compact`/`exploded`)(\_`filter_osm_ids_hash_part`).geoparquet

> If the WKT mode is turned on, then the result file will be saved with a `parquet` extension and a `_wkt` suffix.

### Memory usage

DuckDB queries requiring `JOIN`, `GROUP` and `ORDER BY` operations are very memory intensive. Because of that, some steps are divided into chunks (groups) with a set number of rows per chunk.
Expand Down
35 changes: 35 additions & 0 deletions examples/command_line_interface.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,41 @@
" \"files/andorra_8a275d4edddd035eb6a5d8120a8b42a320b25cf93577335600faba8c2d69d85a_noclip_compact.geoparquet\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## WKT mode\n",
"\n",
"By default, QuackOSM saves parsed files in the `GeoParquet` format with the geometry in the `WKB` format.\n",
"\n",
"There is also an option to save the file as a `Parquet` file with the geometry in the `WKT` format using `--wkt-result` (or `--wkt`) parameter."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"vscode": {
"languageId": "shellscript"
}
},
"outputs": [],
"source": [
"! QuackOSM andorra.osm.pbf --wkt-result"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"duckdb.read_parquet(\n",
" \"files/andorra_nofilter_noclip_compact_wkt.parquet\"\n",
")"
]
}
],
"metadata": {
Expand Down
11 changes: 11 additions & 0 deletions quackosm/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,15 @@ def main(
callback=_filter_osm_ids_callback,
),
] = None,
wkt_result: Annotated[
bool,
typer.Option(
"--wkt-result/",
"--wkt/",
help="Whether to save the geometry as a WKT string instead of WKB blob.",
show_default=False,
),
] = False,
silent_mode: Annotated[
bool,
typer.Option(
Expand Down Expand Up @@ -583,6 +592,7 @@ def main(
else None
),
filter_osm_ids=filter_osm_ids, # type: ignore
save_as_wkt=wkt_result,
silent_mode=silent_mode,
)
else:
Expand All @@ -601,6 +611,7 @@ def main(
else None
),
filter_osm_ids=filter_osm_ids, # type: ignore
save_as_wkt=wkt_result,
silent_mode=silent_mode,
)
typer.secho(geoparquet_path, fg="green")
18 changes: 14 additions & 4 deletions quackosm/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def convert_pbf_to_gpq(
filter_osm_ids: Optional[list[str]] = None,
working_directory: Union[str, Path] = "files",
osm_way_polygon_features_config: Optional[Union[OsmWayPolygonConfig, dict[str, Any]]] = None,
save_as_wkt: bool = False,
silent_mode: bool = False,
) -> Path:
"""
Expand Down Expand Up @@ -69,7 +70,10 @@ def convert_pbf_to_gpq(
Config used to determine which closed way features are polygons.
Modifications to this config left are left for experienced OSM users.
Defaults to predefined "osm_way_polygon_features.json".
silent_mode (bool): Disable progress bars.
save_as_wkt (bool): Whether to save the file with geometry in the WKT form instead of WKB.
If `True`, it will be saved as a `.parquet` file, because it won't be in the GeoParquet
standard. Defaults to `False`.
silent_mode (bool): Disable progress bars. Defaults to `False`.
Returns:
Path: Path to the generated GeoParquet file.
Expand Down Expand Up @@ -229,6 +233,7 @@ def convert_pbf_to_gpq(
explode_tags=explode_tags,
ignore_cache=ignore_cache,
filter_osm_ids=filter_osm_ids,
save_as_wkt=save_as_wkt,
)


Expand All @@ -243,6 +248,7 @@ def convert_geometry_to_gpq(
filter_osm_ids: Optional[list[str]] = None,
working_directory: Union[str, Path] = "files",
osm_way_polygon_features_config: Optional[Union[OsmWayPolygonConfig, dict[str, Any]]] = None,
save_as_wkt: bool = False,
silent_mode: bool = False,
) -> Path:
"""
Expand Down Expand Up @@ -287,7 +293,10 @@ def convert_geometry_to_gpq(
Config used to determine which closed way features are polygons.
Modifications to this config left are left for experienced OSM users.
Defaults to predefined "osm_way_polygon_features.json".
silent_mode (bool): Disable progress bars.
save_as_wkt (bool): Whether to save the file with geometry in the WKT form instead of WKB.
If `True`, it will be saved as a `.parquet` file, because it won't be in the GeoParquet
standard. Defaults to `False`.
silent_mode (bool): Disable progress bars. Defaults to `False`.
Returns:
Path: Path to the generated GeoParquet file.
Expand Down Expand Up @@ -399,6 +408,7 @@ def convert_geometry_to_gpq(
explode_tags=explode_tags,
ignore_cache=ignore_cache,
filter_osm_ids=filter_osm_ids,
save_as_wkt=save_as_wkt,
)


Expand Down Expand Up @@ -454,7 +464,7 @@ def get_features_gdf(
Config used to determine which closed way features are polygons.
Modifications to this config left are left for experienced OSM users.
Defaults to predefined "osm_way_polygon_features.json".
silent_mode (bool): Disable progress bars.
silent_mode (bool): Disable progress bars. Defaults to `False`.
Returns:
gpd.GeoDataFrame: GeoDataFrame with OSM features.
Expand Down Expand Up @@ -634,7 +644,7 @@ def get_features_gdf_from_geometry(
Config used to determine which closed way features are polygons.
Modifications to this config left are left for experienced OSM users.
Defaults to predefined "osm_way_polygon_features.json".
silent_mode (bool): Disable progress bars.
silent_mode (bool): Disable progress bars. Defaults to `False`.
Returns:
gpd.GeoDataFrame: GeoDataFrame with OSM features.
Expand Down
Loading

0 comments on commit da1b0c1

Please sign in to comment.