Skip to content

Commit

Permalink
Merge pull request #290 from dmgav/more-type-comprehension
Browse files Browse the repository at this point in the history 
Support for Additional Types in Plan Header Annotations
  • Loading branch information
@dmgav
dmgav authored Sep 11, 2023
2 parents f7d489f + 3d99532 commit 7de5539
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 33 deletions.
88 changes: 59 additions & 29 deletions bluesky_queueserver/manager/profile_ops.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2918,23 +2918,53 @@ def convert_annotation_to_string(annotation):
Ignore the type if it can not be properly reconstructed using 'eval'
"""
import bluesky
from bluesky.protocols import Flyable, Movable, Readable
from bluesky.protocols import (
Checkable,
Configurable,
Flyable,
Locatable,
Movable,
Pausable,
Readable,
Stageable,
Stoppable,
Subscribable,
Triggerable,
)

# Patterns for callables
callables_patterns = (r"typing.Callable", r"collections.abc.Callable")
n_callables = 0 # Number of detected callables
protocols_mapping = {
"__READABLE__": [Readable],
"__MOVABLE__": [Movable],
"__FLYABLE__": [Flyable],
"__DEVICE__": [
Configurable,
Triggerable,
Locatable,
Stageable,
Pausable,
Stoppable,
Subscribable,
Checkable,
],
}
protocols_inv = {_: k for k, v in protocols_mapping.items() for _ in v}

protocols_mapping = {"__READABLE__": Readable, "__MOVABLE__": Movable, "__FLYABLE__": Flyable}
protocols_inv = {v: k for k, v in protocols_mapping.items()}
protocols_patterns = {
k: [f"bluesky.protocols.{_.__name__}" for _ in v] for k, v in protocols_mapping.items()
}
callables_patterns = {"__CALLABLE__": [r"typing.Callable", r"collections.abc.Callable"]}
# Patterns for Callable MUST go first
type_patterns = {**callables_patterns, **protocols_patterns}

ns = {
"typing": typing,
"collections": collections,
"bluesky": bluesky,
"NoneType": type(None),
**protocols_mapping,
}

substitutions_dict = {}

# This will work for generic types like 'typing.List[int]'
a_str = f"{annotation!r}"
# The following takes care of Python base types, such as 'int', 'float'
Expand All @@ -2947,27 +2977,28 @@ def convert_annotation_to_string(annotation):
mapping = {k.__name__: v for k, v in protocols_inv.items()}
if a_str in mapping:
a_str = mapping[a_str]
ns[a_str] = annotation # 'a_str' is __DEVICE__, __READABLE__ or similar
else:
# Replace each expression with a unique string in the form of '__CALLABLE<n>__'
while True:
pattern = _get_full_type_name(callables_patterns, a_str)
if not pattern:
break
try:
p_type = eval(pattern, ns, ns)
except Exception:
p_type = None
p_str = f"__CALLABLE{n_callables + 1}__"
a_str = re.sub(re.escape(pattern), p_str, a_str)
if p_type:
# Evaluation of the annotation will fail later and the parameter
# will have no type annotation
ns[p_str] = p_type
n_callables += 1

mapping = {f"bluesky.protocols.{k.__name__}": v for k, v in protocols_inv.items()}
for pattern, replacement in mapping.items():
a_str = re.sub(pattern, replacement, a_str)
n_patterns = 0 # Number of detected callables
for type_name, type_patterns in type_patterns.items():
while True:
pattern = _get_full_type_name(type_patterns, a_str)
if not pattern:
break
try:
p_type = eval(pattern, ns, ns)
except Exception:
p_type = None
p_str_prefix, p_str_suffix = re.findall(r".*[a-zA-Z0-9]|__$", type_name)
p_str = f"{p_str_prefix}{n_patterns + 1}{p_str_suffix}"
a_str = re.sub(re.escape(pattern), p_str, a_str)
if p_type:
# Evaluation of the annotation will fail later and the parameter
# will have no type annotation
ns[p_str] = p_type
substitutions_dict[p_str] = type_name
n_patterns += 1

# Verify if the type could be recreated by evaluating the string during validation.
try:
Expand All @@ -2976,9 +3007,8 @@ def convert_annotation_to_string(annotation):
raise Exception()

# Replace all callables ('__CALLABLE1__', '__CALLABLE2__', etc.) with '__CALLABLE__' string
for n in range(n_callables):
p_str = f"__CALLABLE{n + 1}__"
a_str = re.sub(p_str, "__CALLABLE__", a_str)
for k, v in substitutions_dict.items():
a_str = re.sub(k, v, a_str)

except Exception:
# Ignore the type if it can not be recreated.
Expand Down
116 changes: 115 additions & 1 deletion bluesky_queueserver/manager/tests/test_profile_ops.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2059,6 +2059,88 @@ def _pf2i(


def _pf2j(
val1: bluesky.protocols.Configurable,
val2: bluesky.protocols.Triggerable,
val3: bluesky.protocols.Locatable,
val4: bluesky.protocols.Stageable,
val5: bluesky.protocols.Pausable,
val6: bluesky.protocols.Stoppable,
val7: bluesky.protocols.Subscribable,
val8: bluesky.protocols.Checkable,
val9: Optional[bluesky.protocols.Configurable],
val10: typing.Union[bluesky.protocols.Triggerable, list[bluesky.protocols.Locatable]],
):
yield from [val1, val2, val3, val4, val5, val6, val7, val8]


_pf2j_processed = {
"parameters": [
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val1",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val2",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val3",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val4",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val5",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val6",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val7",
},
{
"annotation": {"type": "__DEVICE__"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val8",
},
{
"annotation": {"type": "typing.Optional[__DEVICE__]"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val9",
},
{
"annotation": {"type": "typing.Union[__DEVICE__, list[__DEVICE__]]"},
"convert_device_names": True,
"kind": {"name": "POSITIONAL_OR_KEYWORD", "value": 1},
"name": "val10",
},
],
"properties": {"is_generator": True},
}


def _pf2k(
val1: typing.Callable,
val2: typing.Callable[[int, float], str],
val3: typing.Union[
Expand All @@ -2076,7 +2158,7 @@ def _pf2j(
yield from [val1, val2, val3, val4, val5]


_pf2j_processed = {
_pf2k_processed = {
"parameters": [
{
"annotation": {"type": "__CALLABLE__"},
Expand Down Expand Up @@ -2125,6 +2207,7 @@ def _pf2j(
(_pf2h, _pf2h_processed),
(_pf2i, _pf2i_processed),
(_pf2j, _pf2j_processed),
(_pf2k, _pf2k_processed),
])
# fmt: on
def test_process_plan_2(plan_func, plan_info_expected):
Expand Down Expand Up @@ -6952,6 +7035,16 @@ def _vp3a(
yield from ["one", "two", "three"]


def _vp3b(
detectors: typing.Iterable[protocols.Readable],
motors: typing.Optional[typing.Union[protocols.Movable, typing.Iterable[protocols.Movable]]] = None,
):
"""
Test if type Iterable can be used for the detector (device) list.
"""
yield from ["one", "two", "three"]


# Error messages may be different for Pydantic 1 and 2
if pydantic_version_major == 2:
err_msg_tvp3a = "Input should be 'm2' [type=enum, input_value='m4', input_type=str]"
Expand Down Expand Up @@ -7060,6 +7153,26 @@ def _vp3a(
# Int instead of a motor name (validation should fail)
(_vp3a, {"args": [(0, "m2"), ("d1", "d2"), ("p1",), ("10.0", 20.0)], "kwargs": {}},
("m1", "m2", "d1", "d2"), False, err_msg_tvp3l),
# Parameter "detectors" is 'typing.Iterable[protocols.Readable]'
(_vp3b, {"args": [("d1", "d2")], "kwargs": {}},
("d1", "d2"), True, ""),
(_vp3b, {"args": [], "kwargs": {"detectors": ("d1", "d2")}},
("d1", "d2"), True, ""),
(_vp3b, {"args": [("d1",)], "kwargs": {}},
("d1", "d2"), True, ""),
(_vp3b, {"args": ["d1"], "kwargs": {}},
("d1", "d2"), False, "Incorrect parameter type: key='detectors'"),
(_vp3b, {"args": [], "kwargs": {"detectors": 'd1'}},
("d1", "d2"), False, "Incorrect parameter type: key='detectors'"),
(_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": ("m1", "m2")}},
("m1", "m2"), True, ""),
(_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": ("m1",)}},
("m1", "m2"), True, ""),
(_vp3b, {"args": [], "kwargs": {"detectors": [], "motors": "m1"}}, # String IS allowed !!
("m1", "m2"), True, ""),
])
# fmt: on
def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg):
Expand All @@ -7070,6 +7183,7 @@ def test_validate_plan_3(plan_func, plan, allowed_devices, success, errmsg):
plan["name"] = plan_func.__name__
allowed_plans = {
"_vp3a": _process_plan(_vp3a, existing_devices={}, existing_plans={}),
"_vp3b": _process_plan(_vp3b, existing_devices={}, existing_plans={}),
"p1": {}, # The plan is used only as a parameter value
"p2": {}, # The plan is used only as a parameter value
}
Expand Down
24 changes: 21 additions & 3 deletions docs/source/plan_annotation.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,9 +295,27 @@ The server can recognize and properly handle the following types used in the pla
* ``bluesky.protocols.Readable`` (replaced by ``__READABLE__`` built-in type);
* ``bluesky.protocols.Movable`` (replaced by ``__MOVABLE__`` built-in type);
* ``bluesky.protocols.Flyable`` (replaced by ``__FLYABLE__`` built-in type);
* ``bluesky.protocols.Configurable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Triggerable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Locatable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Stageable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Pausable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Stoppable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Subscribable`` (replaced by ``__DEVICE__`` built-in type);
* ``bluesky.protocols.Checkable`` (replaced by ``__DEVICE__`` built-in type);
* ``collections.abc.Callable`` (replaced by ``__CALLABLE__`` built-in type);
* ``typing.Callable`` (replaced by ``__CALLABLE__`` built-in type).

.. note::

Note, that ``typing.Iterable`` can be used with the types listed above with certain restrictions.
If a parameter is annotated as ``typing.Iterable[bluesky.protocols.Readable]``, then the validation
will succeed for a list of devices (names of devices), but fails if a single device name is
passed to a plan. If a parameter is expected to accept a single device or a list (iterable) of devices,
the parameter should be annotated as
``typing.Union[bluesky.protocols.Readable, typing.Iterable[bluesky.protocols.Readable]]``.
Validation will fail for a single device if the order of types in the union is reversed.

**Supported types of default values.** The default values can be objects of native Python
types and literal expressions with objects of native Python types. The default value should
be reconstructable with ``ast.literal_eval()``, i.e. for the default value ``vdefault``,
Expand Down Expand Up @@ -375,9 +393,9 @@ of plans with type hints:
# correctly processed by the Queue Server.
<code implementing the plan>
The server can process the annotations containing Bluesky protocols ``bluesky.protocols.Readable``,
``bluesky.protocols.Movable`` and ``bluesky.protocols.flyable`` and callable types
``collections.abc.Callable`` and ``typing.Callable`` with or without type parameters.
The server can process the annotations containing Bluesky protocols such as
``bluesky.protocols.Readable``, ```bluesky.protocols.Movable`` and ``bluesky.protocols.Flyable``
and callable types ``collections.abc.Callable`` and ``typing.Callable`` with or without type parameters.
Those types are replaced with ``__READABLE__``, ``__MOVABLE__``, ``__FLYABLE__``
and ``__CALLABLE__`` built-in types respectively. See the details on built-in types in
:ref:`parameter_annotation_decorator_parameter_types`.
Expand Down

0 comments on commit 7de5539

Please sign in to comment.