From 6e9c80d57480d40068dea9e8cbb905162988db58 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Tue, 5 Sep 2023 22:41:48 -0400 Subject: [PATCH 1/5] TST: additional test for 'validate_plan' --- .../manager/tests/test_profile_ops.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bluesky_queueserver/manager/tests/test_profile_ops.py b/bluesky_queueserver/manager/tests/test_profile_ops.py index 502cd5ae..ede0ade1 100644 --- a/bluesky_queueserver/manager/tests/test_profile_ops.py +++ b/bluesky_queueserver/manager/tests/test_profile_ops.py @@ -6952,6 +6952,13 @@ def _vp3a( yield from ["one", "two", "three"] +def _vp3b(detectors: typing.Iterable[protocols.Readable]): + """ + Test if type Iterable can be used for the detector (device) list. + """ + yield from ["one", "two", "three"] + + # Error messages may be different for Pydantic 1 and 2 if pydantic_version_major == 2: err_msg_tvp3a = "Input should be 'm2' [type=enum, input_value='m4', input_type=str]" @@ -7060,6 +7067,19 @@ def _vp3a( # Int instead of a motor name (validation should fail) (_vp3a, {"args": [(0, "m2"), ("d1", "d2"), ("p1",), ("10.0", 20.0)], "kwargs": {}}, ("m1", "m2", "d1", "d2"), False, err_msg_tvp3l), + + # Parameter "detectors" is 'typing.Iterable[protocols.Readable]' + (_vp3b, {"args": [("d1", "d2")], "kwargs": {}}, + ("d1", "d2"), True, ""), + (_vp3b, {"args": [], "kwargs": {"detectors": ("d1", "d2")}}, + ("d1", "d2"), True, ""), + (_vp3b, {"args": [("d1",)], "kwargs": {}}, + ("d1", "d2"), True, ""), + (_vp3b, {"args": ["d1"], "kwargs": {}}, + ("d1", "d2"), False, "Incorrect parameter type: key='detectors'"), + (_vp3b, {"args": [], "kwargs": {"detectors": 'd1'}}, + ("d1", "d2"), False, "Incorrect parameter type: key='detectors'"), + ]) # fmt: on def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg): @@ -7070,6 +7090,7 @@ def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg): plan["name"] = plan_func.__name__ allowed_plans = { "_vp3a": _process_plan(_vp3a, existing_devices={}, existing_plans={}), + "_vp3b": _process_plan(_vp3b, existing_devices={}, existing_plans={}), "p1": {}, # The plan is used only as a parameter value "p2": {}, # The plan is used only as a parameter value } From b2605bb2d97225d7a2203bc521f01a2a83de18d4 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Wed, 6 Sep 2023 09:37:47 -0400 Subject: [PATCH 2/5] TST: more tests for 'validate_plan' --- .../manager/tests/test_profile_ops.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bluesky_queueserver/manager/tests/test_profile_ops.py b/bluesky_queueserver/manager/tests/test_profile_ops.py index ede0ade1..dd31c891 100644 --- a/bluesky_queueserver/manager/tests/test_profile_ops.py +++ b/bluesky_queueserver/manager/tests/test_profile_ops.py @@ -6952,7 +6952,10 @@ def _vp3a( yield from ["one", "two", "three"] -def _vp3b(detectors: typing.Iterable[protocols.Readable]): +def _vp3b( + detectors: typing.Iterable[protocols.Readable], + motors: typing.Optional[typing.Union[protocols.Movable, typing.Iterable[protocols.Movable]]] = None, +): """ Test if type Iterable can be used for the detector (device) list. """ @@ -7080,6 +7083,13 @@ def _vp3b(detectors: typing.Iterable[protocols.Readable]): (_vp3b, {"args": [], "kwargs": {"detectors": 'd1'}}, ("d1", "d2"), False, "Incorrect parameter type: key='detectors'"), + (_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": ("m1", "m2")}}, + ("m1", "m2"), True, ""), + (_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": ("m1",)}}, + ("m1", "m2"), True, ""), + (_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": "m1"}}, # String IS allowed !! + ("m1", "m2"), True, ""), + ]) # fmt: on def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg): From f62305d3d931860fb6c51d25c34ea5dd2433e71c Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Wed, 6 Sep 2023 13:21:45 -0400 Subject: [PATCH 3/5] ENH: extended 'convert_annotation_to_string' --- bluesky_queueserver/manager/profile_ops.py | 90 +++++++++++++++------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/bluesky_queueserver/manager/profile_ops.py b/bluesky_queueserver/manager/profile_ops.py index a591e50a..dc343462 100644 --- a/bluesky_queueserver/manager/profile_ops.py +++ b/bluesky_queueserver/manager/profile_ops.py @@ -2918,23 +2918,56 @@ def convert_annotation_to_string(annotation): Ignore the type if it can not be properly reconstructed using 'eval' """ import bluesky - from bluesky.protocols import Flyable, Movable, Readable + from bluesky.protocols import ( + Checkable, + Configurable, + Flyable, + Locatable, + Movable, + Pausable, + Readable, + Stageable, + Stoppable, + Subscribable, + Triggerable, + ) - # Patterns for callables - callables_patterns = (r"typing.Callable", r"collections.abc.Callable") - n_callables = 0 # Number of detected callables + protocols_mapping = { + "__READABLE__": [Readable], + "__MOVABLE__": [Movable], + "__FLYABLE__": [Flyable], + "__DEVICE__": [ + Configurable, + Triggerable, + Locatable, + Stageable, + Pausable, + Stoppable, + Subscribable, + Checkable, + ], + } + protocols_inv = {_: k for k, v in protocols_mapping.items() for _ in v} - protocols_mapping = {"__READABLE__": Readable, "__MOVABLE__": Movable, "__FLYABLE__": Flyable} - protocols_inv = {v: k for k, v in protocols_mapping.items()} + protocols_patterns = { + k: [f"bluesky.protocols.{_.__name__}" for _ in v] for k, v in protocols_mapping.items() + } + callables_patterns = {"__CALLABLE__": [r"typing.Callable", r"collections.abc.Callable"]} + type_patterns = {**callables_patterns, **protocols_patterns} ns = { "typing": typing, "collections": collections, "bluesky": bluesky, "NoneType": type(None), - **protocols_mapping, + "__READABLE__": Readable, + "__MOVABLE__": Movable, + "__FLYABLE__": Flyable, + "__DEVICE__": Movable, # It doesn't matter what type is assigned } + substitutions_dict = {} + # This will work for generic types like 'typing.List[int]' a_str = f"{annotation!r}" # The following takes care of Python base types, such as 'int', 'float' @@ -2949,25 +2982,25 @@ def convert_annotation_to_string(annotation): a_str = mapping[a_str] else: # Replace each expression with a unique string in the form of '__CALLABLE__' - while True: - pattern = _get_full_type_name(callables_patterns, a_str) - if not pattern: - break - try: - p_type = eval(pattern, ns, ns) - except Exception: - p_type = None - p_str = f"__CALLABLE{n_callables + 1}__" - a_str = re.sub(re.escape(pattern), p_str, a_str) - if p_type: - # Evaluation of the annotation will fail later and the parameter - # will have no type annotation - ns[p_str] = p_type - n_callables += 1 - - mapping = {f"bluesky.protocols.{k.__name__}": v for k, v in protocols_inv.items()} - for pattern, replacement in mapping.items(): - a_str = re.sub(pattern, replacement, a_str) + n_patterns = 0 # Number of detected callables + for type_name, type_patterns in type_patterns.items(): + while True: + pattern = _get_full_type_name(type_patterns, a_str) + if not pattern: + break + try: + p_type = eval(pattern, ns, ns) + except Exception: + p_type = None + p_str_prefix, p_str_suffix = re.findall(r".*[a-zA-Z0-9]|__$", type_name) + p_str = f"{p_str_prefix}{n_patterns + 1}{p_str_suffix}" + a_str = re.sub(re.escape(pattern), p_str, a_str) + if p_type: + # Evaluation of the annotation will fail later and the parameter + # will have no type annotation + ns[p_str] = p_type + substitutions_dict[p_str] = type_name + n_patterns += 1 # Verify if the type could be recreated by evaluating the string during validation. try: @@ -2976,9 +3009,8 @@ def convert_annotation_to_string(annotation): raise Exception() # Replace all callables ('__CALLABLE1__', '__CALLABLE2__', etc.) with '__CALLABLE__' string - for n in range(n_callables): - p_str = f"__CALLABLE{n + 1}__" - a_str = re.sub(p_str, "__CALLABLE__", a_str) + for k, v in substitutions_dict.items(): + a_str = re.sub(k, v, a_str) except Exception: # Ignore the type if it can not be recreated. From 411e9e93cb563f0b1206962ce4fbe23bf201d4df Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Wed, 6 Sep 2023 13:48:22 -0400 Subject: [PATCH 4/5] ENH: support for additional data type in 'convert_annotation_to_string' --- bluesky_queueserver/manager/profile_ops.py | 6 +- .../manager/tests/test_profile_ops.py | 85 ++++++++++++++++++- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/bluesky_queueserver/manager/profile_ops.py b/bluesky_queueserver/manager/profile_ops.py index dc343462..91c805da 100644 --- a/bluesky_queueserver/manager/profile_ops.py +++ b/bluesky_queueserver/manager/profile_ops.py @@ -2953,6 +2953,7 @@ def convert_annotation_to_string(annotation): k: [f"bluesky.protocols.{_.__name__}" for _ in v] for k, v in protocols_mapping.items() } callables_patterns = {"__CALLABLE__": [r"typing.Callable", r"collections.abc.Callable"]} + # Patterns for Callable MUST go first type_patterns = {**callables_patterns, **protocols_patterns} ns = { @@ -2960,10 +2961,6 @@ def convert_annotation_to_string(annotation): "collections": collections, "bluesky": bluesky, "NoneType": type(None), - "__READABLE__": Readable, - "__MOVABLE__": Movable, - "__FLYABLE__": Flyable, - "__DEVICE__": Movable, # It doesn't matter what type is assigned } substitutions_dict = {} @@ -2980,6 +2977,7 @@ def convert_annotation_to_string(annotation): mapping = {k.__name__: v for k, v in protocols_inv.items()} if a_str in mapping: a_str = mapping[a_str] + ns[a_str] = annotation # 'a_str' is __DEVICE__, __READABLE__ or similar else: # Replace each expression with a unique string in the form of '__CALLABLE__' n_patterns = 0 # Number of detected callables diff --git a/bluesky_queueserver/manager/tests/test_profile_ops.py b/bluesky_queueserver/manager/tests/test_profile_ops.py index dd31c891..810c839a 100644 --- a/bluesky_queueserver/manager/tests/test_profile_ops.py +++ b/bluesky_queueserver/manager/tests/test_profile_ops.py @@ -2059,6 +2059,88 @@ def _pf2i( def _pf2j( + val1: bluesky.protocols.Configurable, + val2: bluesky.protocols.Triggerable, + val3: bluesky.protocols.Locatable, + val4: bluesky.protocols.Stageable, + val5: bluesky.protocols.Pausable, + val6: bluesky.protocols.Stoppable, + val7: bluesky.protocols.Subscribable, + val8: bluesky.protocols.Checkable, + val9: Optional[bluesky.protocols.Configurable], + val10: typing.Union[bluesky.protocols.Triggerable, list[bluesky.protocols.Locatable]], +): + yield from [val1, val2, val3, val4, val5, val6, val7, val8] + + +_pf2j_processed = { + "parameters": [ + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val1", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val2", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val3", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val4", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val5", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val6", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val7", + }, + { + "annotation": {"type": "__DEVICE__"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val8", + }, + { + "annotation": {"type": "typing.Optional[__DEVICE__]"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val9", + }, + { + "annotation": {"type": "typing.Union[__DEVICE__, list[__DEVICE__]]"}, + "convert_device_names": True, + "kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1}, + "name": "val10", + }, + ], + "properties": {"is_generator": True}, +} + + +def _pf2k( val1: typing.Callable, val2: typing.Callable[[int, float], str], val3: typing.Union[ @@ -2076,7 +2158,7 @@ def _pf2j( yield from [val1, val2, val3, val4, val5] -_pf2j_processed = { +_pf2k_processed = { "parameters": [ { "annotation": {"type": "__CALLABLE__"}, @@ -2125,6 +2207,7 @@ def _pf2j( (_pf2h, _pf2h_processed), (_pf2i, _pf2i_processed), (_pf2j, _pf2j_processed), + (_pf2k, _pf2k_processed), ]) # fmt: on def test_process_plan_2(plan_func, plan_info_expected): From 3d995320d047e9bd925d697712bcfd5b5631db7a Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Wed, 6 Sep 2023 14:50:21 -0400 Subject: [PATCH 5/5] DOC: updated documentation to include supported types --- docs/source/plan_annotation.rst | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/source/plan_annotation.rst b/docs/source/plan_annotation.rst index 8694aa52..5fdc9a19 100644 --- a/docs/source/plan_annotation.rst +++ b/docs/source/plan_annotation.rst @@ -295,9 +295,27 @@ The server can recognize and properly handle the following types used in the pla * ``bluesky.protocols.Readable`` (replaced by ``__READABLE__`` built-in type); * ``bluesky.protocols.Movable`` (replaced by ``__MOVABLE__`` built-in type); * ``bluesky.protocols.Flyable`` (replaced by ``__FLYABLE__`` built-in type); +* ``bluesky.protocols.Configurable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Triggerable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Locatable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Stageable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Pausable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Stoppable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Subscribable`` (replaced by ``__DEVICE__`` built-in type); +* ``bluesky.protocols.Checkable`` (replaced by ``__DEVICE__`` built-in type); * ``collections.abc.Callable`` (replaced by ``__CALLABLE__`` built-in type); * ``typing.Callable`` (replaced by ``__CALLABLE__`` built-in type). +.. note:: + + Note, that ``typing.Iterable`` can be used with the types listed above with certain restrictions. + If a parameter is annotated as ``typing.Iterable[bluesky.protocols.Readable]``, then the validation + will succeed for a list of devices (names of devices), but fails if a single device name is + passed to a plan. If a parameter is expected to accept a single device or a list (iterable) of devices, + the parameter should be annotated as + ``typing.Union[bluesky.protocols.Readable, typing.Iterable[bluesky.protocols.Readable]]``. + Validation will fail for a single device if the order of types in the union is reversed. + **Supported types of default values.** The default values can be objects of native Python types and literal expressions with objects of native Python types. The default value should be reconstructable with ``ast.literal_eval()``, i.e. for the default value ``vdefault``, @@ -375,9 +393,9 @@ of plans with type hints: # correctly processed by the Queue Server. -The server can process the annotations containing Bluesky protocols ``bluesky.protocols.Readable``, -``bluesky.protocols.Movable`` and ``bluesky.protocols.flyable`` and callable types -``collections.abc.Callable`` and ``typing.Callable`` with or without type parameters. +The server can process the annotations containing Bluesky protocols such as +``bluesky.protocols.Readable``, ```bluesky.protocols.Movable`` and ``bluesky.protocols.Flyable`` +and callable types ``collections.abc.Callable`` and ``typing.Callable`` with or without type parameters. Those types are replaced with ``__READABLE__``, ``__MOVABLE__``, ``__FLYABLE__`` and ``__CALLABLE__`` built-in types respectively. See the details on built-in types in :ref:`parameter_annotation_decorator_parameter_types`.