Merge pull request #282 from dmgav/pydantic_v2

Compatibility with Pydantic v2

.github/workflows/testing.yml

@@ -10,7 +10,13 @@ jobs:
       matrix:
         use-ipykernel: [false, true]
         python-version: ["3.9", "3.10", "3.11"]
+        pydantic-version: ["<2.0.0", ">=2.0.0"]
         group: [1, 2, 3]
+        exclude:
+          - python-version: "3.9"
+            pydantic-version: "<2.0.0"
+          - python-version: "3.10"
+            pydantic-version: "<2.0.0"
       fail-fast: false
 
     steps:
@@ -30,6 +36,7 @@ jobs:
 
         pip install .
         pip install -r requirements-dev.txt
+        pip install "pydantic${{ matrix.pydantic-version }}"
         pip list
     - name: Test with pytest
       env:

bluesky_queueserver/manager/profile_ops.py

@@ -3,6 +3,7 @@
 import enum
 import glob
 import importlib
+import importlib.resources
 import inspect
 import logging
 import numbers
@@ -17,7 +18,6 @@
 from collections.abc import Iterable
 
 import jsonschema
-import pkg_resources
 import pydantic
 import yaml
 from numpydoc.docscrape import NumpyDocString
@@ -173,7 +173,7 @@ def get_default_startup_dir():
     Returns the path to the default profile collection that is distributed with the package.
     The function does not guarantee that the directory exists. Used for demo with Python-based worker.
     """
-    pc_path = pkg_resources.resource_filename("bluesky_queueserver", "profile_collection_sim/")
+    pc_path = os.path.join(importlib.resources.files("bluesky_queueserver"), "profile_collection_sim", "")
     return pc_path
 
 
@@ -2180,6 +2180,7 @@ def _process_annotation(encoded_annotation, *, ns=None):
             for d in items[item_name]:
                 type_code += f"'{d}': '{d}',"
             type_code += "})"
+
             ns[type_name] = eval(type_code, ns, ns)
 
         # Once all the types are created,  execute the code for annotation.
@@ -2420,6 +2421,8 @@ def _validate_plan_parameters(param_list, call_args, call_kwargs):
         #   recommended 'm.dict()', because 'm.dict()' was causing performance problems
         #   when validating large batches of plans. Based on testing, 'm.__dict__' seems
         #   to work fine in this application.
+        # NOTE: the following step may not be needed once Pydantic 1 is deprecated,
+        #   because Pydantic 2 performs strict type checking.
         success, msg = _compare_in_out(bound_args.arguments, m.__dict__)
         if not success:
             raise ValueError(f"Error in argument types: {msg}")

bluesky_queueserver/manager/tests/test_profile_ops.py

@@ -13,9 +13,11 @@
 
 import ophyd
 import ophyd.sim
+import pydantic
 import pytest
 import yaml
 from bluesky import protocols
+from packaging import version
 
 from bluesky_queueserver import gen_list_of_plans_and_devices, register_device, register_plan
 from bluesky_queueserver.manager.annotation_decorator import parameter_annotation_decorator
@@ -72,6 +74,7 @@
 )
 
 python_version = sys.version_info  # Can be compare to tuples (such as 'python_version >= (3, 9)')
+pydantic_version_major = version.parse(pydantic.__version__).major
 
 
 def test_get_default_startup_dir():
@@ -3094,11 +3097,14 @@ def test_find_and_replace_built_in_types_1(
 
 
 def _create_schema_for_testing(annotation_type):
-    import pydantic
-
     model_kwargs = {"par": (annotation_type, ...)}
     func_model = pydantic.create_model("func_model", **model_kwargs)
-    schema = func_model.schema()
+
+    if pydantic_version_major == 2:
+        schema = func_model.model_json_schema()
+    else:
+        schema = func_model.schema()
+
     return schema
 
 
@@ -3890,6 +3896,13 @@ def test_plan(param):
 """
 
 
+# Error messages may be different for Pydantic 1 and 2
+if pydantic_version_major == 2:
+    err_msg_tpp1 = r"Input should be 'det1','det2' or 'det3' \[type=enum, input_value='det4', input_type=str\]"
+else:
+    err_msg_tpp1 = "value is not a valid enumeration member; permitted: 'det1', 'det2', 'det3'"
+
+
 # fmt: off
 @pytest.mark.parametrize(
     "plan, exp_args, exp_kwargs, success, err_msg",
@@ -3910,8 +3923,7 @@ def test_plan(param):
         ({"name": "move_then_count", "user_group": _user_group,
           "args": [["motor1"], ["det1", "det4"]],
           "kwargs": {"positions": [5, 7]}},
-         [[ophyd.sim.motor1], [ophyd.sim.det1], [5, 7]], {}, False,
-         "value is not a valid enumeration member; permitted: 'det1', 'det2', 'det3'"),
+         [[ophyd.sim.motor1], [ophyd.sim.det1], [5, 7]], {}, False, err_msg_tpp1),
         # Use default value for 'detectors' defined in parameter annotation
         ({"name": "move_then_count", "user_group": _user_group, "args": [["motor1"]],
           "kwargs": {"positions": [5, 7]}},
@@ -4260,6 +4272,29 @@ def plan7(a, b: str):
     return plans_in_nspace, devices_in_nspace, allowed_plans, allowed_devices
 
 
+# Error messages may be different for Pydantic 1 and 2
+if pydantic_version_major == 2:
+    err_msg_tpp2a = (
+        r"Input should be a valid integer, got a number with a fractional part "
+        r"\[type=int_from_float, input_value=2.6, input_type=float\]"
+    )
+    err_msg_tpp2b = (
+        r"Input should be a valid integer, got a number with a fractional part "
+        r"\[type=int_from_float, input_value=2.8, input_type=float\]"
+    )
+    err_msg_tpp2c = "Input should be '_pp_dev1','_pp_dev2' or '_pp_dev3'"
+    err_msg_tpp2d = "Input should be '_pp_p1','_pp_p2' or '_pp_p3'"
+    err_msg_tpp2e = "Input should be 'one','two' or 'three'"
+    err_msg_tpp2f = r"Input should be a valid string \[type=string_type, input_value=50, input_type=int\]"
+else:
+    err_msg_tpp2a = "Incorrect parameter type: key='a', value='2.6'"
+    err_msg_tpp2b = "Incorrect parameter type: key='b', value='2.8'"
+    err_msg_tpp2c = "value is not a valid enumeration member"
+    err_msg_tpp2d = "value is not a valid enumeration member"
+    err_msg_tpp2e = "value is not a valid enumeration member"
+    err_msg_tpp2f = "Incorrect parameter type: key='b', value='50'"
+
+
 # fmt: off
 @pytest.mark.parametrize(
     "plan_name, plan, remove_objs, exp_args, exp_kwargs, exp_meta, success, err_msg",
@@ -4284,7 +4319,7 @@ def plan7(a, b: str):
         ("plan2", {"user_group": _user_group, "args": [3, 2.6]}, [],
          [3, 2.6], {}, {}, True, ""),
         ("plan2", {"user_group": _user_group, "args": [2.6, 3]}, [],
-         [2.6, 3], {}, {}, False, " Incorrect parameter type: key='a', value='2.6'"),
+         [2.6, 3], {}, {}, False, err_msg_tpp2a),
         ("plan2", {"user_group": _user_group, "kwargs": {"b": 9.9, "s": "def"}}, [],
          [], {"b": 9.9, "s": "def"}, {}, True, ""),
 
@@ -4299,7 +4334,7 @@ def plan7(a, b: str):
         ("plan3", {"user_group": _user_group, "args": [], "kwargs": {"b": 30}}, [],
          [], {'a': 0.5, 'b': 30, 's': 50}, {}, True, ""),
         ("plan3", {"user_group": _user_group, "args": [], "kwargs": {"b": 2.8}}, [],
-         [], {'a': 0.5, 'b': 2.8, 's': 50}, {}, False, "Incorrect parameter type: key='b', value='2.8'"),
+         [], {'a': 0.5, 'b': 2.8, 's': 50}, {}, False, err_msg_tpp2b),
 
         # Plan with a single parameter, which is a list of detector. The list of detector names (enum)
         #   and default value (list of detectors) is specified in 'parameter_annotation_decorator'.
@@ -4312,7 +4347,7 @@ def plan7(a, b: str):
         ("plan4", {"user_group": _user_group, "kwargs": {"detectors": ["_pp_dev1", "_pp_dev3"]}}, [],
          [[_pp_dev1, _pp_dev3]], {}, {}, True, ""),
         ("plan4", {"user_group": _user_group, "kwargs": {"detectors": ["nonexisting_dev", "_pp_dev3"]}}, [],
-         [[_pp_dev1, _pp_dev3]], {}, {}, False, "value is not a valid enumeration member"),
+         [[_pp_dev1, _pp_dev3]], {}, {}, False, err_msg_tpp2c),
 
         # Passing subdevice names to plans. Parameter annotation contains fixed lists of device names,
         #   so the passed devices should be converted to objects or parameter validation should fail
@@ -4459,7 +4494,7 @@ def plan7(a, b: str):
         ("plan5", {"user_group": _user_group, "kwargs": {"plan_to_execute": "_pp_p3"}}, [],
          [_pp_p3], {}, {}, True, ""),
         ("plan5", {"user_group": _user_group, "args": ["nonexisting_plan"]}, [],
-         [_pp_p3], {}, {}, False, "value is not a valid enumeration member"),
+         [_pp_p3], {}, {}, False, err_msg_tpp2d),
         # Remove plan from the list of allowed plans
         ("plan5", {"user_group": _user_group}, ["_pp_p2"],
          [], {"plan_to_execute": "_pp_p2"}, {}, True, ""),
@@ -4523,7 +4558,7 @@ def plan7(a, b: str):
         ("plan6", {"user_group": _user_group, "args": [("one", "two")]}, [],
          [("one", "two")], {}, {}, True, ""),
         ("plan6", {"user_group": _user_group, "args": [("one", "nonexisting")]}, [],
-         [("one", "two")], {}, {}, False, "value is not a valid enumeration member"),
+         [("one", "two")], {}, {}, False, err_msg_tpp2e),
 
         # Plan has no custom annotation. All strings must be converted to objects whenever possible.
         ("plan1", {"user_group": _user_group, "args": ["a", ":a*", "a-b-c"]}, [],
@@ -4537,8 +4572,7 @@ def plan7(a, b: str):
         ("plan7", {"user_group": _user_group, "args": [["_pp_dev1", "_pp_dev3", "some_str"], "_pp_dev2"]}, [],
          [[_pp_dev1, _pp_dev3, "some_str"], "_pp_dev2"], {}, {}, True, ""),
         ("plan7", {"user_group": _user_group, "args": [["_pp_dev1", "_pp_dev3", "some_str"], 50]}, [],
-         [[_pp_dev1, _pp_dev3, "some_str"], "_pp_dev2"], {}, {}, False,
-         "Incorrect parameter type: key='b', value='50'"),
+         [[_pp_dev1, _pp_dev3, "some_str"], "_pp_dev2"], {}, {}, False, err_msg_tpp2f),
 
         # General failing cases
         ("nonexisting_plan", {"user_group": _user_group}, [],
@@ -6662,8 +6696,42 @@ def _vp3a(
     yield from ["one", "two", "three"]
 
 
+# Error messages may be different for Pydantic 1 and 2
+if pydantic_version_major == 2:
+    err_msg_tvp3a = "Input should be 'm2' [type=enum, input_value='m4', input_type=str]"
+    err_msg_tvp3b = "Input should be 'm2' [type=enum, input_value='m3', input_type=str]"
+    err_msg_tvp3c = "Input should be an instance of Motors [type=is_instance_of, input_value='m2', input_type=str]"
+    err_msg_tvp3d = "Input should be an instance of Motors [type=is_instance_of, input_value='m2', input_type=str]"
+    err_msg_tvp3e = "Input should be a valid list [type=list_type, input_value='m2', input_type=str]"
+    err_msg_tvp3f = (
+        "Input should be an instance of Detectors2 [type=is_instance_of, input_value='d4', input_type=str]"
+    )
+    err_msg_tvp3f1 = "Input should be 'd5' [type=enum, input_value='d4', input_type=str]"
+    err_msg_tvp3g = "Input should be a valid list [type=list_type, input_value='d2', input_type=str]"
+    err_msg_tvp3h = "Input should be 'd1' or 'd2' [type=enum, input_value='d4', input_type=str]"
+    err_msg_tvp3i = "Input should be 'p1' [type=enum, input_value='p3', input_type=str]"
+    err_msg_tvp3j = "Input should be 'p1' [type=enum, input_value='p2', input_type=str]"
+    err_msg_tvp3k = "Input should be 'p1' [type=enum, input_value='p2', input_type=str]"
+    err_msg_tvp3l = "Input should be 'm1' or 'm2' [type=enum, input_value=0, input_type=int]"
+else:
+    err_msg_tvp3a = "value is not a valid enumeration member; permitted: 'm2'"
+    err_msg_tvp3b = "value is not a valid enumeration member; permitted: 'm2'"
+    err_msg_tvp3c = "value is not a valid enumeration member; permitted:"
+    err_msg_tvp3d = "value is not a valid enumeration member; permitted:"
+    err_msg_tvp3e = "value is not a valid list"
+    err_msg_tvp3f = "value is not a valid enumeration member; permitted:"
+    err_msg_tvp3f1 = "value is not a valid enumeration member; permitted:"
+    err_msg_tvp3g = "value is not a valid list"
+    err_msg_tvp3h = "value is not a valid enumeration member; permitted: 'd1', 'd2'"
+    err_msg_tvp3i = "value is not a valid enumeration member; permitted: 'p1'"
+    err_msg_tvp3j = "value is not a valid enumeration member; permitted: 'p1'"
+    err_msg_tvp3k = "value is not a valid enumeration member; permitted: 'p1'"
+    err_msg_tvp3l = "value is not a valid enumeration member"
+
+
 # fmt: off
 @pytest.mark.parametrize("plan_func, plan, allowed_devices, success, errmsg", [
+
     # Basic use of the function.
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
      ("m1", "m2", "d1", "d2"), True, ""),
@@ -6686,42 +6754,46 @@ def _vp3a(
 
     # Use motor that is not listed in the annotation (but exists in the list of allowed devices).
     (_vp3a, {"args": [("m2", "m4"), ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m2", "m4", "d1", "d2"), False, "value is not a valid enumeration member; permitted: 'm2'"),
+     ("m2", "m4", "d1", "d2"), False, err_msg_tvp3a),
     # The motor is not in the list of allowed devices.
     (_vp3a, {"args": [("m2", "m3"), ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m2", "m4", "d1", "d2"), False, "value is not a valid enumeration member; permitted: 'm2'"),
+     ("m2", "m4", "d1", "d2"), False, err_msg_tvp3b),
     # Both motors are not in the list of allowed devices.
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m4", "m5", "d1", "d2"), False, "value is not a valid enumeration member; permitted:"),
+     ("m4", "m5", "d1", "d2"), False, err_msg_tvp3c),
     # Empty list of allowed devices (should be the same result as above).
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     (), False, "value is not a valid enumeration member; permitted:"),
+     (), False, err_msg_tvp3d),
     # Single motor is passed as a scalar (instead of a list element)
     (_vp3a, {"args": ["m2", ("d1", "d2"), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m2", "m4", "d1", "d2"), False, "value is not a valid list"),
+     ("m2", "m4", "d1", "d2"), False, err_msg_tvp3e),
 
     # Pass single detector (allowed).
     (_vp3a, {"args": [("m1", "m2"), "d4", ("p1",), (10.0, 20.0)], "kwargs": {}},
      ("m1", "m2", "d1", "d2", "d4"), True, ""),
+    # Pass single detector from 'Detectors2' group, no devices from 'Detector2' group are allowed.
+    (_vp3a, {"args": [("m1", "m2"), "d4", ("p1",), (10.0, 20.0)], "kwargs": {}},
+     ("m1", "m2", "d1", "d2"), False, err_msg_tvp3f),
     # Pass single detector from 'Detectors2' group, which is not in the list of allowed devices.
+    #   Detector2 group is not empty.
     (_vp3a, {"args": [("m1", "m2"), "d4", ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2"), False, "value is not a valid enumeration member; permitted:"),
+     ("m1", "m2", "d1", "d2", "d5"), False, err_msg_tvp3f1),
     # Pass single detector from 'Detectors1' group (not allowed).
     (_vp3a, {"args": [("m1", "m2"), "d2", ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2", "d4"), False, " value is not a valid list"),
+     ("m1", "m2", "d1", "d2", "d4"), False, err_msg_tvp3g),
     # Pass a detector from a group 'Detector2' as a list element.
     (_vp3a, {"args": [("m1", "m2"), ("d4",), ("p1",), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2", "d4"), False, "value is not a valid enumeration member; permitted: 'd1', 'd2'"),
+     ("m1", "m2", "d1", "d2", "d4"), False, err_msg_tvp3h),
 
     # Plan 'p3' is not in the list of allowed plans
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p3",), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2"), False, "value is not a valid enumeration member; permitted: 'p1'"),
+     ("m1", "m2", "d1", "d2"), False, err_msg_tvp3i),
     # Plan 'p2' is in the list of allowed plans, but not listed in the annotation.
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p2",), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2"), False, "value is not a valid enumeration member; permitted: 'p1'"),
+     ("m1", "m2", "d1", "d2"), False, err_msg_tvp3j),
     # Plan 'p2' is in the list of allowed plans, but not listed in the annotation.
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), ("p1", "p2"), (10.0, 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2"), False, "value is not a valid enumeration member; permitted: 'p1'"),
+     ("m1", "m2", "d1", "d2"), False, err_msg_tvp3k),
     # Single plan is passed as a scalar (allowed in the annotation).
     (_vp3a, {"args": [("m1", "m2"), ("d1", "d2"), "p1", (10.0, 20.0)], "kwargs": {}},
      ("m1", "m2", "d1", "d2"), True, ""),
@@ -6731,7 +6803,7 @@ def _vp3a(
      ("m1", "m2", "d1", "d2"), False, "Incorrect parameter type"),
     # Int instead of a motor name (validation should fail)
     (_vp3a, {"args": [(0, "m2"), ("d1", "d2"), ("p1",), ("10.0", 20.0)], "kwargs": {}},
-     ("m1", "m2", "d1", "d2"), False, "value is not a valid enumeration member"),
+     ("m1", "m2", "d1", "d2"), False, err_msg_tvp3l),
 ])
 # fmt: on
 def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg):