From bdf5c260471f3cf6f712274bd7760d8a611dbd3c Mon Sep 17 00:00:00 2001 From: John Kerl Date: Wed, 27 Mar 2024 11:38:46 -0400 Subject: [PATCH] [Backport release-1.8] [python/ci] `typeguard==4.2.1`, make `requirements_dev.txt` canonical (#2333) * merge * update for core 2.20 --------- Co-authored-by: Ryan Williams --- .github/workflows/python-ci-single.yml | 5 +--- .../workflows/r-python-interop-testing.yml | 5 +--- apis/python/requirements_dev.txt | 8 +++--- apis/python/setup.py | 7 +---- apis/python/src/tiledbsoma/_collection.py | 5 ++-- apis/python/src/tiledbsoma/_factory.py | 3 +-- .../options/_soma_tiledb_context.py | 22 +++++++++------- apis/python/tests/test_experiment_query.py | 26 +------------------ 8 files changed, 24 insertions(+), 57 deletions(-) diff --git a/.github/workflows/python-ci-single.yml b/.github/workflows/python-ci-single.yml index 7c65d91161..01974aa571 100644 --- a/.github/workflows/python-ci-single.yml +++ b/.github/workflows/python-ci-single.yml @@ -100,11 +100,8 @@ jobs: # dist # key: libtiledbsoma-build-dist-${{ inputs.os }}-${{ inputs.python_version }}-${{ hashFiles('libtiledbsoma', 'scripts/bld') }} - - name: Install testing prereqs - run: python -m pip -v install -U pip pytest-cov 'typeguard>=4' types-setuptools sparse - - name: Install tiledbsoma - run: python -m pip -v install -e apis/python + run: python -m pip -v install -e apis/python[dev] env: CC: ${{ inputs.cc }} CXX: ${{ inputs.cxx }} diff --git a/.github/workflows/r-python-interop-testing.yml b/.github/workflows/r-python-interop-testing.yml index 0b867ddfe6..1a6e1956c7 100644 --- a/.github/workflows/r-python-interop-testing.yml +++ b/.github/workflows/r-python-interop-testing.yml @@ -76,11 +76,8 @@ jobs: cache: pip cache-dependency-path: ./apis/python/setup.py - - name: Install testing prereqs - run: python -m pip -v install -U pip pytest-cov 'typeguard>=4' types-setuptools - - name: Install tiledbsoma - run: python -m pip -v install -e apis/python + run: python -m pip -v install -e apis/python[dev] - name: Show package versions run: | diff --git a/apis/python/requirements_dev.txt b/apis/python/requirements_dev.txt index 914243bb03..509e738cbd 100644 --- a/apis/python/requirements_dev.txt +++ b/apis/python/requirements_dev.txt @@ -1,5 +1,7 @@ -cmake >= 3.21 -pybind11-global >= 2.10.0 -typeguard>=4 +black +ruff pytest +pytest-cov sparse +typeguard==4.2.1 +types-setuptools diff --git a/apis/python/setup.py b/apis/python/setup.py index bd68f46779..ef2fccc914 100644 --- a/apis/python/setup.py +++ b/apis/python/setup.py @@ -332,12 +332,7 @@ def run(self): "typing-extensions", # Note "-" even though `import typing_extensions` ], extras_require={ - "dev": [ - "black", - "ruff", - "pytest", - "typeguard>=4", - ] + "dev": open("requirements_dev.txt").read(), }, python_requires=">=3.8", cmdclass={"build_ext": build_ext, "bdist_wheel": bdist_wheel}, diff --git a/apis/python/src/tiledbsoma/_collection.py b/apis/python/src/tiledbsoma/_collection.py index ebd5afeec8..2869165e1d 100644 --- a/apis/python/src/tiledbsoma/_collection.py +++ b/apis/python/src/tiledbsoma/_collection.py @@ -185,7 +185,7 @@ def add_new_collection( def add_new_collection( self, key: str, - kind: Optional[Type[AnyTileDBCollection]] = None, + kind: Optional[Type[CollectionBase]] = None, # type: ignore[type-arg] *, uri: Optional[str] = None, platform_config: Optional[options.PlatformConfig] = None, @@ -377,7 +377,6 @@ def add_new_sparse_ndarray(self, key: str, **kwargs: Any) -> SparseNDArray: """ return self._add_new_ndarray(SparseNDArray, key, **kwargs) - @typeguard_ignore def _add_new_element( self, key: str, @@ -444,7 +443,7 @@ def __getitem__(self, key: str) -> CollectionElementType: entry.entry.wrapper_type.open, uri, mode, context, timestamp ) # Since we just opened this object, we own it and should close it. - self._close_stack.enter_context(entry.soma) # type: ignore[arg-type] + self._close_stack.enter_context(entry.soma) return cast(CollectionElementType, entry.soma) def set( diff --git a/apis/python/src/tiledbsoma/_factory.py b/apis/python/src/tiledbsoma/_factory.py index 4ccee18ec3..4369145f0e 100644 --- a/apis/python/src/tiledbsoma/_factory.py +++ b/apis/python/src/tiledbsoma/_factory.py @@ -122,7 +122,7 @@ def open( Experimental. """ context = _validate_soma_tiledb_context(context) - obj: TileDBObject[_Wrapper] = _open_internal( # type: ignore[no-untyped-call,valid-type] + obj: TileDBObject[_Wrapper] = _open_internal( # type: ignore[valid-type] _tdb_handles.open, uri, mode, context, tiledb_timestamp ) try: @@ -143,7 +143,6 @@ def open( raise -@no_type_check def _open_internal( opener: Callable[ [str, options.OpenMode, SOMATileDBContext, Optional[OpenTimestamp]], _Wrapper diff --git a/apis/python/src/tiledbsoma/options/_soma_tiledb_context.py b/apis/python/src/tiledbsoma/options/_soma_tiledb_context.py index 2dcd7f9df3..c78faf03a3 100644 --- a/apis/python/src/tiledbsoma/options/_soma_tiledb_context.py +++ b/apis/python/src/tiledbsoma/options/_soma_tiledb_context.py @@ -7,8 +7,8 @@ import functools import threading import time -from concurrent import futures -from typing import Any, Dict, Mapping, Optional, Union +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, Literal, Mapping, Optional, Union import tiledb from somacore import ContextBase @@ -46,8 +46,8 @@ def _maybe_timestamp_ms(input: Optional[OpenTimestamp]) -> Optional[int]: return to_timestamp_ms(input) -_SENTINEL = object() -"""Sentinel object to distinguish default parameters from None.""" +_Unset = Literal["__unset__"] +_UNSET: _Unset = "__unset__" class SOMATileDBContext(ContextBase): @@ -68,7 +68,7 @@ def __init__( tiledb_ctx: Optional[tiledb.Ctx] = None, tiledb_config: Optional[Dict[str, Union[str, float]]] = None, timestamp: Optional[OpenTimestamp] = None, - threadpool: Optional[futures.ThreadPoolExecutor] = None, + threadpool: Optional[ThreadPoolExecutor] = None, ) -> None: """Initializes a new SOMATileDBContext. @@ -138,7 +138,7 @@ def __init__( """The TileDB context to use, either provided or lazily constructed.""" self._timestamp_ms = _maybe_timestamp_ms(timestamp) - self.threadpool = threadpool or futures.ThreadPoolExecutor() + self.threadpool = threadpool or ThreadPoolExecutor() """User specified threadpool. If None, we'll instantiate one ourselves.""" self._native_context: Optional[clib.SOMAContext] = None """Lazily construct clib.SOMAContext.""" @@ -219,8 +219,8 @@ def replace( *, tiledb_config: Optional[Dict[str, Any]] = None, tiledb_ctx: Optional[tiledb.Ctx] = None, - timestamp: Optional[OpenTimestamp] = _SENTINEL, # type: ignore[assignment] - threadpool: Optional[futures.ThreadPoolExecutor] = _SENTINEL, # type: ignore[assignment] + timestamp: Union[None, OpenTimestamp, _Unset] = _UNSET, + threadpool: Union[None, ThreadPoolExecutor, _Unset] = _UNSET, ) -> Self: """Create a copy of the context, merging changes. @@ -259,12 +259,14 @@ def replace( new_config = self._internal_tiledb_config() new_config.update(tiledb_config) tiledb_config = {k: v for (k, v) in new_config.items() if v is not None} - if timestamp is _SENTINEL: + + if timestamp == _UNSET: # Keep the existing timestamp if not overridden. timestamp = self._timestamp_ms - if threadpool is _SENTINEL: + if threadpool == _UNSET: # Keep the existing threadpool if not overridden. threadpool = self.threadpool + return type(self)( tiledb_config=tiledb_config, tiledb_ctx=tiledb_ctx, diff --git a/apis/python/tests/test_experiment_query.py b/apis/python/tests/test_experiment_query.py index 0b249ebd96..8aefdcdbde 100644 --- a/apis/python/tests/test_experiment_query.py +++ b/apis/python/tests/test_experiment_query.py @@ -1,5 +1,4 @@ from concurrent import futures -from contextlib import nullcontext from typing import Tuple from unittest import mock @@ -7,10 +6,8 @@ import pandas as pd import pyarrow as pa import pytest -import tiledb -from pyarrow import ArrowInvalid from scipy import sparse -from somacore import AxisQuery, options +from somacore import options import tiledbsoma as soma from tests._util import raises_no_typeguard @@ -908,24 +905,3 @@ def test_experiment_query_uses_threadpool_from_context(soma_experiment): assert adata is not None pool.submit.assert_called() - - -def test_empty_categorical_query(pbmc_small): - q = pbmc_small.axis_query( - measurement_name="RNA", obs_query=AxisQuery(value_filter='groups == "g1"') - ) - obs = q.obs().concat() - assert len(obs) == 44 - - q = pbmc_small.axis_query( - measurement_name="RNA", obs_query=AxisQuery(value_filter='groups == "foo"') - ) - # Empty query on a categorical column raised ArrowInvalid before TileDB 2.21; see https://github.com/single-cell-data/TileDB-SOMA/pull/2299 - ctx = ( - nullcontext() - if tiledb.libtiledb.version() >= (2, 21) - else pytest.raises(ArrowInvalid) - ) - with ctx: - obs = q.obs().concat() - assert len(obs) == 0