Merge pull request #214 from dan-fritchman/pydantic_v2

Pydantic v2 Compatibility

.github/workflows/test.yaml

@@ -16,28 +16,32 @@ jobs:
 
     # Notes on this matrix, particularly the dependencies part:
     # Hdl21 has two external dependencies: Pydantic and VLSIR.
-    # 
+    #
     # 1. VLSIR
     # This sets up testing with dependencies from both (a) PyPi and (b) "dev" versions from GitHub.
     # Not every version of Hdl21 is designed to work with both.
     # Eventually this should know which *should* work.
     # For now it asserts that the dev-mode passes, and allows failer (`continue-on-error`) for the PyPi version.
-    # 
+    #
     # 2. Pydantic
     # Test with both the minimum supported version, and whatever pip selects, which is generally the latest.
     # Some languages/ libraries (ahem, Rust) find a way to build in this "test them min supported version" thing;
-    # we haven't seen one for Python, and can only really do this "manually" because there is only one. 
+    # we haven't seen one for Python, and can only really do this "manually" because there is only one.
     # Note the combinations pf `python-version` and `pydantic-version` are often relevant;
-    # typing-stuff generally evolves materially with each python version, and pydantic makes heavy use 
+    # typing-stuff generally evolves materially with each python version, and pydantic makes heavy use
     # of checking the current interpreter-version to try to make maximally detailed type-checking.
-    # 
+    #
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-        pydantic-version: ["==1.9.0", ""] 
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        pydantic-version: ["==1.9.0", "==2.0", "==2.6", ""]
         dep-installer: ["dev", "pypi"]
-    continue-on-error: ${{ matrix.dep-installer == 'pypi' || matrix.python-version == '3.11' }}
+    # Issues for these:
+    # VLSIR/ PyPi: https://github.com/dan-fritchman/Hdl21/issues/216
+    # Python 3.12: https://github.com/dan-fritchman/Hdl21/issues/215
+    # Python 3.7-3.8: https://github.com/dan-fritchman/Hdl21/issues/217
+    continue-on-error: ${{ matrix.dep-installer == 'pypi' || matrix.python-version == '3.12' || matrix.python-version == '3.7'|| matrix.python-version == '3.8' }}
 
     steps:
       - name: Checkout Repo

conftest.py

@@ -4,9 +4,16 @@
 Primarily adds the simulator-focused options enabled by `VlsirTools`. 
 """
 
+import sys
+from pydantic import __version__ as pydantic_version
 
 from vlsirtools.pytest import (
     pytest_configure,
     pytest_addoption,
     pytest_collection_modifyitems,
 )
+
+# Print some config info: the interpreter version, and the pydantic version
+print("Hdl21 PyTest Configuration")
+print("Python " + sys.version)
+print("Pydantic " + pydantic_version)

hdl21/datatype.py

@@ -27,15 +27,53 @@
   - Notable exceptions include *union types* thereof, which do not have the necessary fields/ methods. 
 """
 
-from pydantic import Extra
-from pydantic.dataclasses import dataclass
 from typing import TypeVar, Type, Optional
+from pydantic import __version__ as _pydantic_version
+
+
+_pydantic_major_version = int(_pydantic_version.split(".")[0])
+if _pydantic_major_version > 2 or _pydantic_major_version < 1:
+    msg = "Error reading Pydantic version. Should be either 1.x or 2.x."
+    raise ImportError(msg)
+
+if _pydantic_major_version == 1:
+    from pydantic import Extra, BaseModel
+    from pydantic.json import pydantic_encoder as pydantic_json_encoder
+
+    PYDANTIC_V2 = False
+
+    class OurBaseConfig:
+        allow_extra = Extra.forbid
+
+    class AllowArbConfig(OurBaseConfig):
+        arbitrary_types_allowed = True
+
+    def _update_forward_refs():
+        """Update all the forward type-references"""
+        for tp in datatypes:
+            tp.__pydantic_model__.update_forward_refs()
+
+else:  # _pydantic_major_version==2
+    from pydantic import Extra, BaseModel, RootModel, BeforeValidator
+    from pydantic.deprecated.json import pydantic_encoder as pydantic_json_encoder
+
+    PYDANTIC_V2 = True
+    OurBaseConfig = dict(allow_extra="forbid", validate_default=True)
+    AllowArbConfig = dict(
+        allow_extra="forbid", validate_default=True, arbitrary_types_allowed=True
+    )
+
+    def _update_forward_refs():
+        """Update all the forward type-references"""
+        ...
+        # for tp in datatypes:
+        #     tp.model_rebuild()
 
 
-# The list of defined datatypes
-datatypes = []
+from pydantic.dataclasses import dataclass
 
 T = TypeVar("T")
+datatypes = []  # The list of defined datatypes
 
 
 def _datatype(cls: Type[T], *, config: Optional[Type] = None, **kwargs) -> Type[T]:
@@ -63,22 +101,3 @@ def datatype(cls: Optional[Type[T]] = None, **kwargs) -> Type[T]:
     if cls is None:
         return inner  # Called with parens, e.g. `@datatype()`
     return inner(cls)  # Called without parens
-
-
-def _update_forward_refs():
-    """Update all the forward type-references"""
-    for tp in datatypes:
-        tp.__pydantic_model__.update_forward_refs()
-
-
-"""
-# Define a few common pydantic model `Config`s
-"""
-
-
-class OurBaseConfig:
-    allow_extra = Extra.forbid
-
-
-class AllowArbConfig(OurBaseConfig):
-    arbitrary_types_allowed = True

hdl21/elab/passes/flatten_bundles.py

@@ -90,9 +90,6 @@ def add_subscope(self, name: str, scope: "BundleScope"):
             self.signals[path_from_self] = sig
 
 
-BundleScope.__pydantic_model__.update_forward_refs()
-
-
 @datatype(config=AllowArbConfig)
 class BundlePortEntry:
     """# Bundle-Port Entry in the Cache

hdl21/external_module.py

@@ -54,6 +54,8 @@ def Params(self) -> Type:
         return self.paramtype
 
     def __post_init__(self):
+        """Post-Constructor Checks"""
+
         # Check for a valid parameter-type
         if not isparamclass(self.paramtype) and self.paramtype not in (dict, Dict):
             msg = f"Invalid parameter type {self.paramtype} for {self}. "
@@ -64,8 +66,7 @@ def __post_init__(self):
         self._source_info: Optional[SourceInfo] = source_info(get_pymodule=True)
         self._importpath = None
 
-    def __post_init_post_parse__(self):
-        """After type-checking, do some more checks on values"""
+        # Now do some more checks on values
         for p in self.port_list:
             if not p.name:
                 raise ValueError(f"Unnamed Primitive Port {p} for {self.name}")
@@ -99,7 +100,7 @@ class ExternalModuleCall:
     module: ExternalModule
     params: Any
 
-    def __post_init_post_parse__(self):
+    def __post_init__(self):
         # Type-validate our parameters
         if not isinstance(self.params, self.module.paramtype):
             msg = f"Invalid parameter type {type(self.params)} for ExternalModule {self.module.name}. Must be {self.module.paramtype}"

hdl21/flatten.py

@@ -37,7 +37,7 @@ class FlattenedInstance:
     path: List[h.Instance] = field(default_factory=list)
     conns: Dict[str, h.Signal] = field(default_factory=dict)
 
-    def __post_init_post_parse__(self):
+    def __post_init__(self):
         # Assert that this instance's target is either a primitive, or external
         if not isinstance(self.inst.of, (h.PrimitiveCall, h.ExternalModuleCall)):
             raise ValueError(f"Invalid flattened instance {self}")

hdl21/instantiable.py

@@ -3,12 +3,13 @@
 and thus supports its "connect by call" and "connect by assignment" semantics. 
 """
 
-import copy
+# Std-Lib Imports
+from __future__ import annotations
 from typing import Any, Union, Dict
+import copy
 
-from pydantic import BaseModel
-
-from .datatype import AllowArbConfig
+# Local Imports
+from .datatype import AllowArbConfig, _pydantic_major_version
 from .module import Module
 from .primitives import PrimitiveCall
 from .external_module import ExternalModuleCall
@@ -18,31 +19,7 @@
 InstantiableUnion = Union[Module, ExternalModuleCall, PrimitiveCall]
 
 
-class Instantiable(BaseModel):
-    """
-    # Instantiable
-
-    Generally this means
-    ````python
-    Union[Module, ExternalModuleCall, PrimitiveCall]
-    ```
-    with some customized checking and error handling.
-    """
-
-    __root__: InstantiableUnion
-    Config = AllowArbConfig
-
-    def __init__(self, *_, **__):
-        # Brick any attempts to create instances
-        msg = f"Invalid attempt to instantiate an `Instantiable` directly. "
-        raise RuntimeError(msg)
-
-    @classmethod
-    def __get_validators__(cls):
-        yield assert_instantiable
-
-
-def assert_instantiable(i: Any) -> Instantiable:
+def assert_instantiable(i: Any) -> "Instantiable":
     """# Assert that `i` is an `Instantiable` type."""
     if not is_instantiable(i):
         return invalid(i)
@@ -74,7 +51,7 @@ def invalid(val: Any) -> None:
     raise TypeError(msg)
 
 
-def qualname(i: Instantiable) -> str:
+def qualname(i: "Instantiable") -> str:
     """Path-qualified name of Instantiable `i`"""
     from .qualname import qualname as module_qualname
 
@@ -91,7 +68,7 @@ def qualname(i: Instantiable) -> str:
     raise TypeError(f"Invalid Instantiable {i}")
 
 
-def io(i: Instantiable) -> Dict[str, "Connectable"]:
+def io(i: "Instantiable") -> Dict[str, "Connectable"]:
     """
     Get a complete dictionary of IO ports for `i`, including all types: Signals and Bundles.
     Copies the Instantiable's top-level dictionary so that it is not modified by consumers.
@@ -103,4 +80,41 @@ def io(i: Instantiable) -> Dict[str, "Connectable"]:
     return rv
 
 
+_doc = """
+# Instantiable
+
+Generally this means
+````python
+Union[Module, ExternalModuleCall, PrimitiveCall]
+```
+with some customized checking and error handling.
+"""
+
+if _pydantic_major_version == 1:
+    from .datatype import BaseModel
+
+    class Instantiable(BaseModel):
+        # "Custom root types" implementation
+
+        __doc__ = _doc
+        __root__: InstantiableUnion
+        Config = AllowArbConfig
+
+        def __init__(self, *_, **__):
+            # Brick any attempts to create instances
+            msg = f"Invalid attempt to instantiate an `Instantiable` directly. "
+            raise RuntimeError(msg)
+
+        @classmethod
+        def __get_validators__(cls):
+            yield assert_instantiable
+
+else:
+    from .datatype import BeforeValidator
+    from typing import Annotated
+
+    Instantiable = Annotated[InstantiableUnion, BeforeValidator(assert_instantiable)]
+    Instantiable.__doc__ = _doc
+
+
 __all__ = ["Instantiable", "is_instantiable"]

hdl21/noconn.py

@@ -33,7 +33,7 @@ class NoConn:
 
     name: Optional[str] = None
 
-    def __post_init_post_parse__(self) -> None:
+    def __post_init__(self) -> None:
         # Internal management data
         # Connected port references
         self._connected_ports: Set[PortRef] = set()

hdl21/params.py

@@ -11,7 +11,7 @@
 
 # Local Imports
 from .default import Default
-from .datatype import AllowArbConfig
+from .datatype import AllowArbConfig, pydantic_json_encoder
 
 T = TypeVar("T")
 
@@ -293,7 +293,7 @@ def hdl21_naming_encoder(obj: Any) -> Any:
         return {f.name: getattr(obj, f.name) for f in dataclasses.fields(obj)}
 
     # Not an Hdl21 type. Hand off to pydantic.
-    return pydantic.json.pydantic_encoder(obj)
+    return pydantic_json_encoder(obj)
 
 
 # Shortcut for parameter-less generators.

hdl21/pdk/sample_pdk/pdk.py

@@ -47,7 +47,7 @@ class SamplePdkMosParams:
     nf = h.Param(dtype=h.Scalar, desc="Number of parallel fingers", default=1)
     m = h.Param(dtype=h.Scalar, desc="Number of parallel fingers", default=1)
 
-    def __post_init_post_parse__(self):
+    def __post_init__(self):
         """Value Checks"""
         if self.w <= 0:
             raise ValueError(f"MosParams with invalid width {self.w}")

hdl21/portref.py

@@ -24,7 +24,7 @@ class PortRef:
     inst: _Instance
     portname: str
 
-    def __post_init_post_parse__(self):
+    def __post_init__(self):
         # Inner management data
         self._connected_ports: Set[PortRef] = set()
         self.resolved: Union[None, "Signal", "BundleInstance"] = None

hdl21/prefix.py

@@ -183,17 +183,20 @@ def new(cls, number: Decimal, prefix: Prefix = Prefix.UNIT) -> "Prefixed":
         for using arguments by-position, which `pydantic.BaseModel` does not support."""
         return cls(number=number, prefix=prefix)
 
-    @classmethod
-    def validate(cls, v: Union["Prefixed", "ToPrefixed"]) -> "Prefixed":
-        """Validate `v` as a `Prefixed` number, or convert to `Prefixed` if applicable.
-        While usable elsewhere, `validate` is primarily intended for use in type-validated
-        dataclass trees, such as those generated in `paramclass`es."""
-
-        return to_prefixed(v)
-
-    @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
+    #
+    # FIXME: pending potential deprecation #157
+    #
+    # @classmethod
+    # def validate(cls, v: Union["Prefixed", "ToPrefixed"]) -> "Prefixed":
+    #     """Validate `v` as a `Prefixed` number, or convert to `Prefixed` if applicable.
+    #     While usable elsewhere, `validate` is primarily intended for use in type-validated
+    #     dataclass trees, such as those generated in `paramclass`es."""
+    #
+    #     return to_prefixed(v)
+    #
+    # @classmethod
+    # def __get_validators__(cls):
+    #     yield cls.validate
 
     def __hash__(self):
         return hash((self.number, self.prefix))
@@ -335,7 +338,7 @@ def __ge__(self, other) -> bool:
 ToPrefixed = Union[int, float, str, Decimal]
 
 
-def to_prefixed(v: Union[Prefixed, "ToPrefixed"]) -> Prefixed:
+def to_prefixed(v: Union[Prefixed, ToPrefixed]) -> Prefixed:
     """Convert any convertible type to a `Prefixed` number."""
 
     if isinstance(v, Prefixed):