refactor: update to pydantic v2.x (#298)

* refactor: update to pydantic v2.x
* refactor: use typing_extensions.Annotated instead of typing.Annotated for compatibility with python 3.7

Resolves https://github.ibm.com/st4sd/st4sd-runtime-core/issues/250

Signed-off-by: Vassilis Vassiladis <[email protected]>

python/experiment/cli/configuration.py

@@ -8,7 +8,8 @@
 
 import pydantic
 import typer
-from pydantic import AnyHttpUrl, BaseModel, BaseSettings
+from pydantic import AnyHttpUrl, BaseModel
+from pydantic_settings import BaseSettings
 
 #
 APP_NAME = "stp"

python/experiment/model/frontends/dsl.py

@@ -103,7 +103,7 @@
 import typing
 
 import experiment.model.codes
-import pydantic.typing
+import typing_extensions
 import pydantic
 import re
 
@@ -129,15 +129,15 @@
 RevampedReferencePattern = fr'"(?P<reference>([.a-zA-Z0-9_/-])+)"{PatternReferenceMethod}'
 LegacyReferencePattern = fr'(?P<reference>([.a-zA-Z0-9_/-])+){PatternReferenceMethod}'
 
-TargetReference = pydantic.constr(regex=fr"<{StepNamePattern}>")
-ParameterReference = pydantic.constr(regex=ParameterPattern)
-MaxRestarts = pydantic.conint(gt=-2)
-BackendType = pydantic.constr(regex=fr'({ParameterPattern}|docker|local|lsf|kubernetes)')
-K8sQosType = pydantic.constr(regex=fr'({ParameterPattern}|guaranteed|burstable|besteffort)')
-DockerImagePullPolicy = pydantic.constr(regex=fr'({ParameterPattern}|Always|Never|IfNotPresent)')
-ResourceRequestFloat = pydantic.confloat(ge=0)
-ResourceRequestInt = pydantic.conint(ge=0)
-EnvironmentReference = pydantic.constr(regex=fr'({ParameterPattern}|none|environment)')
+TargetReference = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=fr"<{StepNamePattern}>")]
+ParameterReference = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=ParameterPattern)]
+MaxRestarts = typing_extensions.Annotated[int, pydantic.Field(gt=-2)]
+BackendType = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=fr'({ParameterPattern}|docker|local|lsf|kubernetes)')]
+K8sQosType = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=fr'({ParameterPattern}|guaranteed|burstable|besteffort)')]
+DockerImagePullPolicy = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=fr'({ParameterPattern}|Always|Never|IfNotPresent)')]
+ResourceRequestFloat = typing_extensions.Annotated[int, pydantic.Field(ge=0)]
+ResourceRequestInt = typing_extensions.Annotated[int, pydantic.Field(ge=0)]
+EnvironmentReference = typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=fr'({ParameterPattern}|none|environment)')]
 
 
 class OutputReference:
@@ -253,10 +253,28 @@ class Config:
         min_length=1
     )
     default: typing.Optional[ParameterValueType] = pydantic.Field(
+        None,
         description="The default value of the parameter",
     )
 
 
+class InstantiatedParameter(Parameter):
+    value: typing.Optional[ParameterValueType] = pydantic.Field(
+        None,
+        description="The value of the parameter, if unset defaults to @default"
+    )
+
+    def get_value(self) -> typing.Optional[ParameterValueType]:
+        """The value of the parameter, if unset defaults to @default
+
+        Returns:
+            The value of the parameter, if unset defaults to @default
+        """
+        if 'value' in self.__fields_set__:
+            return self.value
+        return self.default
+
+
 class Signature(pydantic.BaseModel):
     class Config:
         extra = "forbid"
@@ -265,7 +283,7 @@ class Config:
         description="The name of the template, must be unique in the parent namespace",
         min_length=1,
         # VV: Names cannot end in digits - FlowIR has a special meaning for digits at the end of component names
-        regex=SignatureNamePattern
+        pattern=SignatureNamePattern
     )
 
     description: typing.Optional[str] = pydantic.Field(
@@ -284,7 +302,7 @@ class Config:
 
     target: TargetReference = pydantic.Field(
         description="Reference to a step name. A string enclosed in <> e.g. <foo>", min_length=3,
-        regex=fr"<{StepNamePattern}>",
+        pattern=fr"<{StepNamePattern}>",
     )
 
     args: typing.Dict[str, ParameterValueType] = pydantic.Field(
@@ -298,7 +316,7 @@ def get_target(self) -> str:
 
 
 class ExecuteStepEntryInstance(ExecuteStep):
-    target: pydantic.typing.Literal["<entry-instance>"] = pydantic.Field(
+    target: typing_extensions.Literal["<entry-instance>"] = pydantic.Field(
         "<entry-instance>", description="The entry point step name. Must be <entry-instance>."
     )
 
@@ -312,7 +330,7 @@ class Config:
         description="The Signature of the Workflow template"
     )
 
-    steps: typing.Dict[str, pydantic.constr(regex=SignatureNamePattern)] = pydantic.Field(
+    steps: typing.Dict[str, typing_extensions.Annotated[str, pydantic.StringConstraints(pattern=SignatureNamePattern)]] = pydantic.Field(
         description="Instantiated Templates that execute as steps of the parent workflow. "
                     "key: value pairs where the key is the name of the Instance and the value is the name "
                     "of the Template from which to create the Instance."
@@ -372,10 +390,8 @@ class CResourceManagerLSF(pydantic.BaseModel):
     class Config:
         extra = "forbid"
 
-    statusRequestInterval: pydantic.conint(ge=20) = pydantic.Field(
-        20,
-        description="How many seconds to wait between polling the status of LSF tasks"
-    )
+    statusRequestInterval: typing_extensions.Annotated[int, pydantic.Field(
+        description="How many seconds to wait between polling the status of LSF tasks", ge=20)] = 20
 
     queue: str = pydantic.Field(
         "normal",
@@ -674,7 +690,7 @@ class CExecutorPre(pydantic.BaseModel):
     class Config:
         extra = "forbid"
 
-    name: pydantic.typing.Literal['lsf-dm-in'] = pydantic.Field(
+    name: typing_extensions.Literal['lsf-dm-in'] = pydantic.Field(
         'lsf-dm-in',
         description="The name of the pre-executor"
     )
@@ -692,7 +708,7 @@ class CExecutorPost(pydantic.BaseModel):
     class Config:
         extra = "forbid"
 
-    name: pydantic.typing.Literal['lsf-dm-out'] = pydantic.Field(
+    name: typing_extensions.Literal['lsf-dm-out'] = pydantic.Field(
         'lsf-dm-out',
         description="The name of the post-executor"
     )
@@ -737,7 +753,7 @@ class Config:
 
     expandArguments: typing.Optional[
         typing.Union[ParameterReference,
-        pydantic.typing.Literal["double-quote", "none"]]
+        typing_extensions.Literal["double-quote", "none"]]
     ] = pydantic.Field(
         "double-quote",
         description="When set to \"double-quote\" it instructs the runtime to expand the arguments to tasks using the "

python/experiment/settings.py

@@ -13,6 +13,8 @@
 import os
 
 import experiment.model.errors
+from pydantic import Field, ConfigDict
+from typing_extensions import Annotated
 
 
 class Orchestrator(pydantic.BaseModel):
@@ -21,24 +23,23 @@ class Orchestrator(pydantic.BaseModel):
     Use the load_settings_orchestrator() method to parse and validate settings (method raises errors that
     include information about offending environment variables)
     """
-    class Config:
-        extra = pydantic.Extra.forbid
+    model_config = ConfigDict(extra=pydantic.Extra.forbid)
 
-    workers_default_all: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_default_all: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         None, description="If set, sets the default value to all worker polls. This will override the default of "
                           "other individual fields. Users can still override those defaults by specifying an explicit "
                           "value.")
-    workers_controller: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_controller: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         40, description="Number of workers in the Controller threadPool")
-    workers_component_state: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_component_state: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         50, description="Number of workers in the ComponentState threadPool")
-    workers_engine: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_engine: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         50, description="Number of workers in the Engine threadPool")
-    workers_engine_trigger: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_engine_trigger: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         20, description="Number of workers in the Engine threadPool that Engines use to trigger immediate emissions")
-    workers_engine_task: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_engine_task: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         100, description="Number of workers in the Engine threadPool that Engines use to wait for tasks to complete")
-    workers_backend_k8s: Optional[pydantic.conint(ge=1)] = pydantic.Field(
+    workers_backend_k8s: Optional[Annotated[int, Field(ge=1)]] = pydantic.Field(
         50, description="Number of workers in the Backend.Kubernetes threadPool")
 
     @pydantic.root_validator(pre=True)

setup.py

@@ -76,7 +76,8 @@
     # rx version dep. Rx moved to 3.x after 1.6.1 which introduced new API
     install_requires=['reactivex>=4.0.0', 'pyyaml', 'pytest', 'pytest-xdist', 'pytest-timeout',
                       'networkx', 'matplotlib', 'requests', 'six', 'kubernetes', 'psutil', 'boto3',
-                      'pyrsistent', 'js2py', 'pymongo>=4.0', 'papermill', 'pandas', 'future', 'pydantic<2.0',
+                      'pyrsistent', 'js2py', 'pymongo>=4.0', 'papermill', 'pandas', 'future', 'pydantic>=2.0.0',
+                      'pydantic-settings',
                       'keyring', 'typer[all]', 'jsonschema'], #, 'pygraphviz'],
 
     # List additional groups of dependencies here (e.g. development