[python]Robust URI handling for Ingestion-performance improvement

apis/python/src/tiledbsoma/__init__.py

@@ -147,7 +147,12 @@
 from ._constants import SOMA_JOINID
 from ._dataframe import DataFrame
 from ._dense_nd_array import DenseNDArray
-from ._exception import AlreadyExistsError, DoesNotExistError, SOMAError
+from ._exception import (
+    AlreadyExistsError,
+    DoesNotExistError,
+    NotCreateableError,
+    SOMAError,
+)
 from ._experiment import Experiment
 from ._factory import open
 from ._general_utilities import (
@@ -192,6 +197,7 @@
     "get_storage_engine",
     "IntIndexer",
     "Measurement",
+    "NotCreateableError",
     "open",
     "ResultOrder",
     "show_package_versions",

apis/python/src/tiledbsoma/_collection.py

@@ -39,9 +39,11 @@
 from ._dense_nd_array import DenseNDArray
 from ._exception import (
     AlreadyExistsError,
+    NotCreateableError,
     SOMAError,
     is_already_exists_error,
     is_does_not_exist_error,
+    is_not_createable_error,
 )
 from ._funcs import typeguard_ignore
 from ._sparse_nd_array import SparseNDArray
@@ -119,6 +121,8 @@
         Raises:
             tiledbsoma.AlreadyExistsError:
                 If the underlying object already exists at the given URI.
+            tiledbsoma.NotCreateableError:
+                If the URI is malformed for a particular storage backend.
             TileDBError:
                 If unable to create the underlying object.
 
@@ -137,6 +141,8 @@
         except tiledb.TileDBError as tdbe:
             if is_already_exists_error(tdbe):
                 raise AlreadyExistsError(f"{uri!r} already exists")
+            if is_not_createable_error(tdbe):
+                raise NotCreateableError(f"{uri!r} cannot be created")
             raise
 
     @classmethod

apis/python/src/tiledbsoma/_common_nd_array.py

@@ -16,7 +16,12 @@
 import tiledb
 
 from . import _arrow_types, _util
-from ._exception import AlreadyExistsError, is_already_exists_error
+from ._exception import (
+    AlreadyExistsError,
+    NotCreateableError,
+    is_already_exists_error,
+    is_not_createable_error,
+)
 from ._tiledb_array import TileDBArray
 from ._types import OpenTimestamp
 from .options._soma_tiledb_context import (
@@ -80,6 +85,8 @@
                 If the ``shape`` is unsupported.
             tiledbsoma.AlreadyExistsError:
                 If the underlying object already exists at the given URI.
+            tiledbsoma.NotCreateableError:
+                If the URI is malformed for a particular storage backend.
             TileDBError:
                 If unable to create the underlying object.
 
@@ -103,6 +110,8 @@
         except tiledb.TileDBError as tdbe:
             if is_already_exists_error(tdbe):
                 raise AlreadyExistsError(f"{uri!r} already exists")
+            if is_not_createable_error(tdbe):
+                raise NotCreateableError(f"{uri!r} cannot be created")
             raise
 
     @property

apis/python/src/tiledbsoma/_dataframe.py

@@ -20,7 +20,12 @@
 from . import _arrow_types, _util
 from . import pytiledbsoma as clib
 from ._constants import SOMA_JOINID
-from ._exception import AlreadyExistsError, is_already_exists_error
+from ._exception import (
+    AlreadyExistsError,
+    NotCreateableError,
+    is_already_exists_error,
+    is_not_createable_error,
+)
 from ._query_condition import QueryCondition
 from ._read_iters import TableReadIter
 from ._tdb_handles import DataFrameWrapper
@@ -190,6 +195,8 @@
                 If the ``schema`` specifies illegal column names.
             tiledbsoma.AlreadyExistsError:
                 If the underlying object already exists at the given URI.
+            tiledbsoma.NotCreateableError:
+                If the URI is malformed for a particular storage backend.
             TileDBError:
                 If unable to create the underlying object.
 
@@ -229,6 +236,8 @@
         except tiledb.TileDBError as tdbe:
             if is_already_exists_error(tdbe):
                 raise AlreadyExistsError(f"{uri!r} already exists")
+            if is_not_createable_error(tdbe):
+                raise NotCreateableError(f"{uri!r} cannot be created")
             raise
 
     def keys(self) -> Tuple[str, ...]:

apis/python/src/tiledbsoma/_exception.py

@@ -85,6 +85,51 @@
     return "already exists" in stre.lower()
 
 
+class NotCreateableError(SOMAError):
+    """Raised when attempting to create an already existing SOMA object.
+
+    Lifecycle: experimental
+    """
+
+    pass
+
+
+def is_not_createable_error(e: tiledb.TileDBError) -> bool:
+    """Given a TileDBError, return true if it indicates the object cannot be created
+
+    Lifecycle: experimental
+
+    Example:
+        try:
+            tiledb.Array.create(uri, schema, ctx=ctx)
+                ...
+        except tiledb.TileDBError as e:
+            if is_not_createable_error(e):
+                ...
+            raise e
+    """
+    stre = str(e)
+    # Context:
+    # * A recurring paradigm in tiledbsoma.io is open for write (if exists) else create --
+    #   or, equivalently, create (if doesn't already exist), else open for write
+    # * A priori either seems fine
+    # * There are performance implications for trying the create first: when an
+    #   object _does_ already exist we get that quickly.
+    # * Therefore it's more performant to try-create-catch-open-for-write
+    # * However we have the following semantics for cloud URIs:
+    #   o For writes: must be "creation URIs" of the form "tiledb://namespace/s3://bucket/some/path"
+    #   o For read:   can be "creation URIs" _or_ non-creation URIs of the form
+    #     "tiledb://namespace/groupname" or "tiledb://namespace/uuid"
+    # * Put together: when we try-create-catch-open-for-write, _and when_ the URI provided
+    #   is a non-creation URI, we need to catch that fact and treat it as a non-error.
+    stre = stre.lower()
+    if "storage backend local not supported" in stre:
+        return True
+    if "storage backend not supported: local" in stre:
+        return True
+    return False
+
+
 def is_duplicate_group_key_error(e: tiledb.TileDBError) -> bool:
     """Given a TileDBError, return try if it indicates a duplicate member
     add request in a tiledb.Group.

apis/python/src/tiledbsoma/io/ingest.py

@@ -61,6 +61,7 @@
 from .._exception import (
     AlreadyExistsError,
     DoesNotExistError,
+    NotCreateableError,
     SOMAError,
 )
 from .._tdb_handles import RawHandle
@@ -989,7 +990,7 @@
 ) -> CollectionBase[_TDBO]:
     try:
         coll = cls.create(uri, context=context)
-    except AlreadyExistsError:
+    except (AlreadyExistsError, NotCreateableError):
         # It already exists. Are we resuming?
         if ingestion_params.error_if_already_exists:
             raise SOMAError(f"{uri} already exists")
@@ -1200,7 +1201,7 @@
             platform_config=platform_config,
             context=context,
         )
-    except AlreadyExistsError:
+    except (AlreadyExistsError, NotCreateableError):
         if ingestion_params.error_if_already_exists:
             raise SOMAError(f"{soma_df.uri} already exists")
 
@@ -1301,7 +1302,7 @@
             platform_config=platform_config,
             context=context,
         )
-    except AlreadyExistsError:
+    except (AlreadyExistsError, NotCreateableError):
         if ingestion_params.error_if_already_exists:
             raise SOMAError(f"{soma_ndarray.uri} already exists")
         soma_ndarray = cls.open(
@@ -2756,7 +2757,7 @@
             platform_config=platform_config,
             context=context,
         )
-    except AlreadyExistsError:
+    except (AlreadyExistsError, NotCreateableError):
         soma_arr = DenseNDArray.open(arr_uri, "w", context=context)
 
     # If resume mode: don't re-write existing data. This is the user's explicit request