GDAL Filesystem rework, updates, fixes, ST_MakeValid.

.github/workflows/Linux.yml

@@ -97,7 +97,7 @@ jobs:
     - name: Setup vcpkg
       uses: lukka/[email protected]
       with:
-        vcpkgGitCommitId: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+        vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2
 
       # Build extension
     - name: Build extension

.github/workflows/MacOS.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Setup vcpkg
         uses: lukka/run-vcpkg@v11
         with:
-          vcpkgGitCommitId: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+          vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2
 
       - name: Setup ccache
         uses: hendrikmuhs/[email protected]

.github/workflows/MainDistributionPipeline.yml

@@ -22,7 +22,7 @@ jobs:
     name: Build extension binaries
     uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@60ddc316ca0c1585f14d55aa73f9db59d8fc05d1
     with:
-      duckdb_version: v0.9.1
+      duckdb_version: v0.9.2
       extension_name: spatial
       vcpkg_commit: 9edb1b8e590cc086563301d735cae4b6e732d2d2 # TODO: remove pinned vcpkg commit when updating duckdb version
       build_duckdb_shell: false
@@ -33,6 +33,6 @@ jobs:
     uses: ./.github/workflows/_extension_deploy.yml
     secrets: inherit
     with:
-      duckdb_version: v0.9.1
+      duckdb_version: v0.9.2
       extension_name: spatial
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

.github/workflows/Windows.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup vcpkg
         uses: lukka/run-vcpkg@v11
         with:
-          vcpkgGitCommitId: 501db0f17ef6df184fcdbfbe0f87cde2313b6ab1
+          vcpkgGitCommitId: 9edb1b8e590cc086563301d735cae4b6e732d2d2
 
       - name: Setup Python
         uses: actions/setup-python@v2

Makefile

@@ -49,7 +49,7 @@ EXTENSION_FLAGS=\
 -DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test"
 
 #### Add more of the DuckDB in-tree extensions here that you need (also feel free to remove them when not needed)
-EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet"
+EXTRA_EXTENSIONS_FLAG=-DBUILD_EXTENSIONS="parquet;json"
 
 BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 $(EXTENSION_FLAGS) ${EXTRA_EXTENSIONS_FLAG} $(OSX_BUILD_FLAG) $(TOOLCHAIN_FLAGS)
 ifeq (${BUILD_SHELL}, 0)

deps/CMakeLists.txt

@@ -199,8 +199,9 @@ ExternalProject_Add(
 ExternalProject_Add(
     GDAL
     DEPENDS ${GDAL_DEPENDENCIES}
-    URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/gdal363.zip
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/vendor/gdal380.zip
     CONFIGURE_HANDLED_BY_BUILD TRUE
+    PATCH_COMMAND patch -p1 < "${CMAKE_CURRENT_LIST_DIR}/patches/remove_filehandler.patch"
     CMAKE_ARGS 
     # CMake options
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
@@ -288,6 +289,8 @@ ExternalProject_Add(
     -DOGR_ENABLE_DRIVER_LVBAG=ON
     -DOGR_ENABLE_DRIVER_VFK=ON
     -DOGR_ENABLE_DRIVER_MVT=ON
+    -DOGR_ENABLE_DRIVER_PMTILES=ON
+    -DOGR_ENABLE_DRIVER_JSONFG=ON
 
     # Drivers requiring network/curl
     -DOGR_ENABLE_DRIVER_AMIGOCLOUD=${SPATIAL_USE_NETWORK}

deps/patches/remove_filehandler.patch

@@ -0,0 +1,72 @@
+diff --git a/port/cpl_vsi.h b/port/cpl_vsi.h
+index 80c66795a3..47751df4f1 100644
+--- a/port/cpl_vsi.h
++++ b/port/cpl_vsi.h
+@@ -719,6 +719,13 @@ void CPL_DLL VSIFreeFilesystemPluginCallbacksStruct(
+ int CPL_DLL VSIInstallPluginHandler(
+     const char *pszPrefix, const VSIFilesystemPluginCallbacksStruct *poCb);
+
++/**
++ * Unregister a handler previously installed with VSIInstallPluginHandler() on
++ * the given prefix.
++ * @since GDAL 3.9
++ */
++int CPL_DLL VSIRemovePluginHandler(const char *pszPrefix);
++
+ /* ==================================================================== */
+ /*      Time querying.                                                  */
+ /* ==================================================================== */
+diff --git a/port/cpl_vsi_virtual.h b/port/cpl_vsi_virtual.h
+index 7c9b9ca304..bf41ee6bbd 100644
+--- a/port/cpl_vsi_virtual.h
++++ b/port/cpl_vsi_virtual.h
+@@ -327,8 +327,7 @@ class CPL_DLL VSIFileManager
+     static VSIFilesystemHandler *GetHandler(const char *);
+     static void InstallHandler(const std::string &osPrefix,
+                                VSIFilesystemHandler *);
+-    /* RemoveHandler is never defined. */
+-    /* static void RemoveHandler( const std::string& osPrefix ); */
++    static void RemoveHandler(const std::string& osPrefix);
+
+     static char **GetPrefixes();
+ };
+diff --git a/port/cpl_vsil.cpp b/port/cpl_vsil.cpp
+index 3a649d50bf..74683b840d 100644
+--- a/port/cpl_vsil.cpp
++++ b/port/cpl_vsil.cpp
+@@ -3253,6 +3253,18 @@ void VSIFileManager::InstallHandler(const std::string &osPrefix,
+         Get()->oHandlers[osPrefix] = poHandler;
+ }
+
++/************************************************************************/
++/*                          RemoveHandler()                             */
++/************************************************************************/
++
++void VSIFileManager::RemoveHandler(const std::string& osPrefix)
++{
++    if (osPrefix == "")
++        Get()->poDefaultHandler = nullptr;
++    else
++        Get()->oHandlers.erase(osPrefix);
++}
++
+ /************************************************************************/
+ /*                       VSICleanupFileManager()                        */
+ /************************************************************************/
+diff --git a/port/cpl_vsil_plugin.cpp b/port/cpl_vsil_plugin.cpp
+index 945f878f55..f35d0aaf74 100644
+--- a/port/cpl_vsil_plugin.cpp
++++ b/port/cpl_vsil_plugin.cpp
+@@ -470,6 +470,12 @@ int VSIInstallPluginHandler(const char *pszPrefix,
+     return 0;
+ }
+
++int VSIRemovePluginHandler(const char *pszPrefix)
++{
++    VSIFileManager::RemoveHandler(pszPrefix);
++    return 0;
++}
++
+ VSIFilesystemPluginCallbacksStruct *
+ VSIAllocFilesystemPluginCallbacksStruct(void)
+ {

spatial/include/spatial/gdal/file_handler.hpp

@@ -1,18 +1,23 @@
 #pragma once
 
 #include "spatial/common.hpp"
+#include "duckdb/main/database.hpp"
 
 namespace spatial {
 
 namespace gdal {
 
-struct GdalFileHandler {
-	static void Register();
+class DuckDBFileSystemHandler;
 
-	// This is a workaround to allow the global file handler to access the current client context
-	// by storing it in a thread_local variable before executing a GDAL IO operation
-	static void SetLocalClientContext(ClientContext &context);
-	static ClientContext &GetLocalClientContext();
+class GDALClientContextState : public ClientContextState {
+	string client_prefix;
+	DuckDBFileSystemHandler* fs_handler;
+public:
+	explicit GDALClientContextState(ClientContext& context);
+	~GDALClientContextState() override;
+	void QueryEnd() override;
+	const string& GetPrefix() const;
+	static GDALClientContextState& GetOrCreate(ClientContext& context);
 };
 
 } // namespace gdal

spatial/include/spatial/geos/functions/scalar.hpp

@@ -31,6 +31,7 @@ struct GEOSScalarFunctions {
 		RegisterStIsSimple(db);
 		RegisterStIsValid(db);
 		RegisterStLineMerge(db);
+		RegisterStMakeValid(db);
 		RegisterStNormalize(db);
 		RegisterStOverlaps(db);
 		RegisterStPointOnSurface(db);
@@ -74,6 +75,7 @@ struct GEOSScalarFunctions {
 	static void RegisterStRemoveRepeatedPoints(DatabaseInstance &db);
 	static void RegisterStReverse(DatabaseInstance &db);
 	static void RegisterStLineMerge(DatabaseInstance &db);
+	static void RegisterStMakeValid(DatabaseInstance &db);
 	static void RegisterStSimplifyPreserveTopology(DatabaseInstance &db);
 	static void RegisterStSimplify(DatabaseInstance &db);
 	static void RegisterStTouches(DatabaseInstance &db);

spatial/src/spatial/core/functions/aggregate/st_envelope_agg.cpp

@@ -90,7 +90,7 @@ struct EnvelopeAggFunction {
 //------------------------------------------------------------------------
 void CoreAggregateFunctions::RegisterStEnvelopeAgg(DatabaseInstance &db) {
 
-	AggregateFunctionSet st_envelope_agg("st_envelope_agg");
+	AggregateFunctionSet st_envelope_agg("ST_Envelope_Agg");
 	st_envelope_agg.AddFunction(
 	    AggregateFunction::UnaryAggregate<EnvelopeAggState, string_t, string_t, EnvelopeAggFunction>(
 	        core::GeoTypes::GEOMETRY(), core::GeoTypes::GEOMETRY()));

spatial/src/spatial/core/functions/scalar/st_contains.cpp

@@ -146,8 +146,8 @@ static void PointWithinPolygonFunction(DataChunk &args, ExpressionState &state,
 void CoreScalarFunctions::RegisterStContains(DatabaseInstance &db) {
 
 	// ST_Within is the inverse of ST_Contains
-	ScalarFunctionSet contains_function_set("st_contains");
-	ScalarFunctionSet within_function_set("st_within");
+	ScalarFunctionSet contains_function_set("ST_Contains");
+	ScalarFunctionSet within_function_set("ST_Within");
 
 	// POLYGON_2D - POINT_2D
 	contains_function_set.AddFunction(ScalarFunction({GeoTypes::POLYGON_2D(), GeoTypes::POINT_2D()},

spatial/src/spatial/core/functions/scalar/st_intersects_extent.cpp

@@ -31,7 +31,7 @@ static void IntersectsExtentFunction(DataChunk &args, ExpressionState &state, Ve
 }
 
 void CoreScalarFunctions::RegisterStIntersectsExtent(DatabaseInstance &db) {
-	ScalarFunction intersects_func("st_intersects_extent", {GeoTypes::GEOMETRY(), GeoTypes::GEOMETRY()},
+	ScalarFunction intersects_func("ST_Intersects_Extent", {GeoTypes::GEOMETRY(), GeoTypes::GEOMETRY()},
 	                               LogicalType::BOOLEAN, IntersectsExtentFunction);
 
 	ExtensionUtil::RegisterFunction(db, intersects_func);

spatial/src/spatial/core/functions/scalar/st_perimeter.cpp

@@ -131,7 +131,7 @@ static void GeometryPerimeterFunction(DataChunk &args, ExpressionState &state, V
 void CoreScalarFunctions::RegisterStPerimeter(DatabaseInstance &db) {
 
 	// Perimiter
-	ScalarFunctionSet set("st_perimeter");
+	ScalarFunctionSet set("ST_Perimeter");
 	set.AddFunction(ScalarFunction({GeoTypes::BOX_2D()}, LogicalType::DOUBLE, Box2DPerimeterFunction));
 	set.AddFunction(ScalarFunction({GeoTypes::POLYGON_2D()}, LogicalType::DOUBLE, Polygon2DPerimeterFunction));
 	set.AddFunction(ScalarFunction({GeoTypes::GEOMETRY()}, LogicalType::DOUBLE, GeometryPerimeterFunction, nullptr,

spatial/src/spatial/core/functions/scalar/st_xyzm.cpp

@@ -217,7 +217,7 @@ static void GeometryAccessFunction(DataChunk &args, ExpressionState &state, Vect
 //------------------------------------------------------------------------------
 void CoreScalarFunctions::RegisterStX(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_x("st_x");
+	ScalarFunctionSet st_x("ST_X");
 	st_x.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<0>));
 	st_x.AddFunction(ScalarFunction({GeoTypes::GEOMETRY()}, LogicalType::DOUBLE, GeometryAccessFunction<0>));
 
@@ -226,7 +226,7 @@ void CoreScalarFunctions::RegisterStX(DatabaseInstance &db) {
 
 void CoreScalarFunctions::RegisterStXMax(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_xmax("st_xmax");
+	ScalarFunctionSet st_xmax("ST_XMax");
 	st_xmax.AddFunction(ScalarFunction({GeoTypes::BOX_2D()}, LogicalType::DOUBLE, Box2DFunction<2>));
 	st_xmax.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<0>));
 	st_xmax.AddFunction(
@@ -239,7 +239,7 @@ void CoreScalarFunctions::RegisterStXMax(DatabaseInstance &db) {
 
 void CoreScalarFunctions::RegisterStXMin(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_xmin("st_xmin");
+	ScalarFunctionSet st_xmin("ST_XMin");
 	st_xmin.AddFunction(ScalarFunction({GeoTypes::BOX_2D()}, LogicalType::DOUBLE, Box2DFunction<0>));
 	st_xmin.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<0>));
 	st_xmin.AddFunction(
@@ -252,7 +252,7 @@ void CoreScalarFunctions::RegisterStXMin(DatabaseInstance &db) {
 
 void CoreScalarFunctions::RegisterStY(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_y("st_y");
+	ScalarFunctionSet st_y("ST_Y");
 	st_y.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<1>));
 	st_y.AddFunction(ScalarFunction({GeoTypes::GEOMETRY()}, LogicalType::DOUBLE, GeometryAccessFunction<1>));
 
@@ -261,7 +261,7 @@ void CoreScalarFunctions::RegisterStY(DatabaseInstance &db) {
 
 void CoreScalarFunctions::RegisterStYMax(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_ymax("st_ymax");
+	ScalarFunctionSet st_ymax("ST_YMax");
 	st_ymax.AddFunction(ScalarFunction({GeoTypes::BOX_2D()}, LogicalType::DOUBLE, Box2DFunction<3>));
 	st_ymax.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<1>));
 	st_ymax.AddFunction(
@@ -274,7 +274,7 @@ void CoreScalarFunctions::RegisterStYMax(DatabaseInstance &db) {
 
 void CoreScalarFunctions::RegisterStYMin(DatabaseInstance &db) {
 
-	ScalarFunctionSet st_ymin("st_ymin");
+	ScalarFunctionSet st_ymin("ST_YMin");
 	st_ymin.AddFunction(ScalarFunction({GeoTypes::BOX_2D()}, LogicalType::DOUBLE, Box2DFunction<1>));
 	st_ymin.AddFunction(ScalarFunction({GeoTypes::POINT_2D()}, LogicalType::DOUBLE, Point2DFunction<1>));
 	st_ymin.AddFunction(