From 1db7c5a7c232c69a9876269229221ebf2db1ad31 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Tue, 24 Oct 2023 18:07:04 +0200 Subject: [PATCH 01/59] Convert Update and InvalidOperation to events --- link/adapters/gateway.py | 7 ++++--- link/domain/events.py | 33 ++++++++++++++++++++++++++++++ link/domain/link.py | 14 ++++++------- link/domain/state.py | 26 ++--------------------- link/service/gateway.py | 4 ++-- link/service/services.py | 9 ++++---- link/service/uow.py | 7 ++++--- tests/integration/gateway.py | 5 +++-- tests/integration/test_services.py | 5 +++-- tests/unit/entities/test_state.py | 19 +++++++++-------- 10 files changed, 73 insertions(+), 56 deletions(-) create mode 100644 link/domain/events.py diff --git a/link/adapters/gateway.py b/link/adapters/gateway.py index f219bff2..1a6658a3 100644 --- a/link/adapters/gateway.py +++ b/link/adapters/gateway.py @@ -5,9 +5,10 @@ from itertools import groupby from typing import Iterable +from link.domain import events from link.domain.custom_types import Identifier from link.domain.link import Link, create_link -from link.domain.state import Commands, Components, Processes, Update +from link.domain.state import Commands, Components, Processes from link.service.gateway import LinkGateway from .custom_types import PrimaryKey @@ -51,10 +52,10 @@ def translate_tainted_primary_keys(primary_keys: Iterable[PrimaryKey]) -> set[Id tainted_identifiers=translate_tainted_primary_keys(self.facade.get_tainted_primary_keys()), ) - def apply(self, updates: Iterable[Update]) -> None: + def apply(self, updates: Iterable[events.Update]) -> None: """Apply updates to the persistent data representing the link.""" - def keyfunc(update: Update) -> int: + def keyfunc(update: events.Update) -> int: assert update.command is not None return update.command.value diff --git a/link/domain/events.py b/link/domain/events.py new file mode 100644 index 00000000..89e43452 --- /dev/null +++ b/link/domain/events.py @@ -0,0 +1,33 @@ +"""Contains all domain events.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from .custom_types import Identifier + +if TYPE_CHECKING: + from .state import Commands, Operations, State, Transition + + +@dataclass(frozen=True) +class Event: + """Base class for all events.""" + + operation: Operations + identifier: Identifier + + +@dataclass(frozen=True) +class InvalidOperation(Event): + """Represents the result of attempting an operation that is invalid in the entity's current state.""" + + state: type[State] + + +@dataclass(frozen=True) +class Update(Event): + """Represents the persistent update needed to transition an entity.""" + + transition: Transition + command: Commands diff --git a/link/domain/link.py b/link/domain/link.py index 6485d21d..4631f2be 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -4,17 +4,15 @@ from dataclasses import dataclass from typing import Any, Iterable, Iterator, Mapping, Optional, Set, Tuple, TypeVar +from . import events from .custom_types import Identifier from .state import ( STATE_MAP, Components, Entity, - EntityOperationResult, - InvalidOperation, Operations, PersistentState, Processes, - Update, ) @@ -114,14 +112,14 @@ def operation_results(self) -> Tuple[LinkOperationResult, ...]: def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" - def create_operation_result(results: Iterable[EntityOperationResult]) -> LinkOperationResult: + def create_operation_result(results: Iterable[events.Event]) -> LinkOperationResult: """Create the result of an operation on a link from results of individual entities.""" results = set(results) operation = next(iter(results)).operation return LinkOperationResult( operation, - updates=frozenset(result for result in results if isinstance(result, Update)), - errors=frozenset(result for result in results if isinstance(result, InvalidOperation)), + updates=frozenset(result for result in results if isinstance(result, events.Update)), + errors=frozenset(result for result in results if isinstance(result, events.InvalidOperation)), ) assert requested, "No identifiers requested." @@ -151,8 +149,8 @@ class LinkOperationResult: """Represents the result of an operation on all entities of a link.""" operation: Operations - updates: frozenset[Update] - errors: frozenset[InvalidOperation] + updates: frozenset[events.Update] + errors: frozenset[events.InvalidOperation] def __post_init__(self) -> None: """Validate the result.""" diff --git a/link/domain/state.py b/link/domain/state.py index 7b453232..5679402b 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -4,9 +4,9 @@ from dataclasses import dataclass, replace from enum import Enum, auto from functools import partial -from typing import Union from .custom_types import Identifier +from .events import Event, InvalidOperation, Update class State: @@ -191,28 +191,6 @@ class Operations(Enum): PROCESS = auto() -@dataclass(frozen=True) -class Update: - """Represents the persistent update needed to transition an entity.""" - - operation: Operations - identifier: Identifier - transition: Transition - command: Commands - - -@dataclass(frozen=True) -class InvalidOperation: - """Represents the result of attempting an operation that is invalid in the entity's current state.""" - - operation: Operations - identifier: Identifier - state: type[State] - - -EntityOperationResult = Union[Update, InvalidOperation] - - class Processes(Enum): """Names for processes that pull/delete entities into/from the local side.""" @@ -290,7 +268,7 @@ class Entity: state: type[State] current_process: Processes is_tainted: bool - operation_results: tuple[EntityOperationResult, ...] + operation_results: tuple[Event, ...] def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" diff --git a/link/service/gateway.py b/link/service/gateway.py index 999b76e8..da2b5e4e 100644 --- a/link/service/gateway.py +++ b/link/service/gateway.py @@ -4,8 +4,8 @@ from abc import ABC, abstractmethod from collections.abc import Iterable +from link.domain import events from link.domain.link import Link -from link.domain.state import Update class LinkGateway(ABC): @@ -16,5 +16,5 @@ def create_link(self) -> Link: """Create a link from the persistent data.""" @abstractmethod - def apply(self, updates: Iterable[Update]) -> None: + def apply(self, updates: Iterable[events.Update]) -> None: """Apply updates to the link's persistent data.""" diff --git a/link/service/services.py b/link/service/services.py index a3237f6c..71e4f50f 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -5,8 +5,9 @@ from dataclasses import dataclass from enum import Enum, auto +from link.domain import events from link.domain.custom_types import Identifier -from link.domain.state import InvalidOperation, Operations, Update, states +from link.domain.state import Operations, states from .uow import UnitOfWork @@ -31,7 +32,7 @@ class PullResponse(Response): """Response model for the pull use-case.""" requested: frozenset[Identifier] - errors: frozenset[InvalidOperation] + errors: frozenset[events.InvalidOperation] def pull( @@ -109,8 +110,8 @@ class OperationResponse(Response): operation: Operations requested: frozenset[Identifier] - updates: frozenset[Update] - errors: frozenset[InvalidOperation] + updates: frozenset[events.Update] + errors: frozenset[events.InvalidOperation] @dataclass(frozen=True) diff --git a/link/service/uow.py b/link/service/uow.py index 34bbf0ef..c4d1d3b6 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -6,9 +6,10 @@ from types import TracebackType from typing import Callable, Iterable, Protocol +from link.domain import events from link.domain.custom_types import Identifier from link.domain.link import Link -from link.domain.state import TRANSITION_MAP, Entity, Operations, Transition, Update +from link.domain.state import TRANSITION_MAP, Entity, Operations, Transition from .gateway import LinkGateway @@ -27,7 +28,7 @@ def __init__(self, gateway: LinkGateway) -> None: """Initialize the unit of work.""" self._gateway = gateway self._link: Link | None = None - self._updates: dict[Identifier, deque[Update]] = defaultdict(deque) + self._updates: dict[Identifier, deque[events.Update]] = defaultdict(deque) def __enter__(self) -> UnitOfWork: """Enter the context in which updates to entities can be made.""" @@ -77,7 +78,7 @@ def store_update(operation: Operations, current: Entity, new: Entity) -> None: return transition = Transition(current.state, new.state) self._updates[current.identifier].append( - Update(operation, current.identifier, transition, TRANSITION_MAP[transition]) + events.Update(operation, current.identifier, transition, TRANSITION_MAP[transition]) ) self._link = self._gateway.create_link() diff --git a/tests/integration/gateway.py b/tests/integration/gateway.py index e8c1eec9..5961236a 100644 --- a/tests/integration/gateway.py +++ b/tests/integration/gateway.py @@ -3,9 +3,10 @@ from collections.abc import Mapping from typing import Iterable +from link.domain import events from link.domain.custom_types import Identifier from link.domain.link import Link, create_link -from link.domain.state import Commands, Components, Processes, Update +from link.domain.state import Commands, Components, Processes from link.service.gateway import LinkGateway @@ -27,7 +28,7 @@ def __init__( def create_link(self) -> Link: return create_link(self.assignments, tainted_identifiers=self.tainted_identifiers, processes=self.processes) - def apply(self, updates: Iterable[Update]) -> None: + def apply(self, updates: Iterable[events.Update]) -> None: for update in updates: if update.command is Commands.START_PULL_PROCESS: self.processes[Processes.PULL].add(update.identifier) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 444eab47..d1c3101b 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -6,7 +6,8 @@ import pytest -from link.domain.state import Components, InvalidOperation, Operations, Processes, State, states +from link.domain import events +from link.domain.state import Components, Operations, Processes, State, states from link.service.io import Service, make_responsive from link.service.services import ( DeleteRequest, @@ -230,7 +231,7 @@ def test_pulled_entity_ends_in_correct_state(state: EntityConfig, expected: type def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityConfig, produces_error: bool) -> None: if produces_error: errors = { - InvalidOperation( + events.InvalidOperation( operation=Operations.START_PULL, identifier=create_identifier("1"), state=states.Deprecated ) } diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index c74e7c79..4dfee5bd 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -5,17 +5,16 @@ import pytest +from link.domain import events from link.domain.custom_types import Identifier from link.domain.link import create_link from link.domain.state import ( Commands, Components, - InvalidOperation, Operations, Processes, State, Transition, - Update, states, ) from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -52,7 +51,7 @@ def test_invalid_transitions_returns_unchanged_entity( ) entity = next(entity for entity in link if entity.identifier == identifier) for operation in operations: - result = InvalidOperation(operation, identifier, state) + result = events.InvalidOperation(operation, identifier, state) assert entity.apply(operation) == replace(entity, operation_results=(result,)) @@ -64,7 +63,7 @@ def test_start_pulling_idle_entity_returns_correct_entity() -> None: state=states.Activated, current_process=Processes.PULL, operation_results=( - Update( + events.Update( Operations.START_PULL, entity.identifier, Transition(states.Idle, states.Activated), @@ -97,7 +96,7 @@ def test_processing_activated_entity_returns_correct_entity( ) entity = next(iter(link)) updated_results = entity.operation_results + ( - Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + events.Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) assert entity.apply(Operations.PROCESS) == replace( entity, state=new_state, current_process=new_process, operation_results=updated_results @@ -126,7 +125,9 @@ def test_processing_received_entity_returns_correct_entity( tainted_identifiers=tainted_identifiers, ) entity = next(iter(link)) - operation_results = (Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command),) + operation_results = ( + events.Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + ) assert entity.apply(Operations.PROCESS) == replace( entity, state=new_state, current_process=new_process, operation_results=operation_results ) @@ -139,7 +140,7 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: entity = next(iter(link)) transition = Transition(states.Pulled, states.Received) operation_results = ( - Update( + events.Update( Operations.START_DELETE, entity.identifier, transition, @@ -158,7 +159,9 @@ def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: ) entity = next(iter(link)) transition = Transition(states.Tainted, states.Received) - operation_results = (Update(Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS),) + operation_results = ( + events.Update(Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS), + ) assert entity.apply(Operations.START_DELETE) == replace( entity, state=transition.new, current_process=Processes.DELETE, operation_results=operation_results ) From 39c51db96b1414ac9a83ec80256c56508031b65e Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:14:24 +0200 Subject: [PATCH 02/59] Use update stored on entity in uow --- link/service/uow.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/link/service/uow.py b/link/service/uow.py index c4d1d3b6..75ea9cbe 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -9,7 +9,7 @@ from link.domain import events from link.domain.custom_types import Identifier from link.domain.link import Link -from link.domain.state import TRANSITION_MAP, Entity, Operations, Transition +from link.domain.state import Entity, Operations from .gateway import LinkGateway @@ -65,21 +65,17 @@ def augmented(operation: Operations) -> Entity: if current._is_expired is True: raise RuntimeError("Can not apply operation to expired entity") new = original(operation) - store_update(operation, current, new) + store_update(new) augment_entity(new) object.__setattr__(current, "_is_expired", True) return new return augmented - def store_update(operation: Operations, current: Entity, new: Entity) -> None: - assert current.identifier == new.identifier - if current.state is new.state: + def store_update(new: Entity) -> None: + if not isinstance(new.operation_results[-1], events.Update): return - transition = Transition(current.state, new.state) - self._updates[current.identifier].append( - events.Update(operation, current.identifier, transition, TRANSITION_MAP[transition]) - ) + self._updates[new.identifier].append(new.operation_results[-1]) self._link = self._gateway.create_link() augment_link(self._link) From bf243ad756c5e2e300e2f3b7157a2ae128bdb9d6 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:51:22 +0200 Subject: [PATCH 03/59] Store entities instead of updates in uow --- link/service/uow.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/link/service/uow.py b/link/service/uow.py index 75ea9cbe..178394bd 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -2,12 +2,12 @@ from __future__ import annotations from abc import ABC -from collections import defaultdict, deque +from collections import deque from types import TracebackType from typing import Callable, Iterable, Protocol -from link.domain import events from link.domain.custom_types import Identifier +from link.domain.events import Update from link.domain.link import Link from link.domain.state import Entity, Operations @@ -28,7 +28,7 @@ def __init__(self, gateway: LinkGateway) -> None: """Initialize the unit of work.""" self._gateway = gateway self._link: Link | None = None - self._updates: dict[Identifier, deque[events.Update]] = defaultdict(deque) + self._seen: dict[Identifier, Entity] = {} def __enter__(self) -> UnitOfWork: """Enter the context in which updates to entities can be made.""" @@ -56,6 +56,7 @@ def augment_entity(entity: Entity) -> None: augmented = augment_entity_apply(entity, original) object.__setattr__(entity, "apply", augmented) object.__setattr__(entity, "_is_expired", False) + self._seen[entity.identifier] = entity def augment_entity_apply( current: Entity, original: Callable[[Operations], Entity] @@ -65,18 +66,12 @@ def augmented(operation: Operations) -> Entity: if current._is_expired is True: raise RuntimeError("Can not apply operation to expired entity") new = original(operation) - store_update(new) augment_entity(new) object.__setattr__(current, "_is_expired", True) return new return augmented - def store_update(new: Entity) -> None: - if not isinstance(new.operation_results[-1], events.Update): - return - self._updates[new.identifier].append(new.operation_results[-1]) - self._link = self._gateway.create_link() augment_link(self._link) for entity in self._link: @@ -101,8 +96,8 @@ def commit(self) -> None: """Persist updates made to the link.""" if self._link is None: raise RuntimeError("Not available outside of context") - while self._updates: - identifier, updates = self._updates.popitem() + for entity in self._seen.values(): + updates = deque(event for event in entity.operation_results if isinstance(event, Update)) while updates: self._gateway.apply([updates.popleft()]) self.rollback() @@ -114,4 +109,4 @@ def rollback(self) -> None: object.__setattr__(self._link, "_is_expired", True) for entity in self._link: object.__setattr__(entity, "_is_expired", True) - self._updates.clear() + self._seen.clear() From f770fa2f657799e3fea656ef931d3963d7409902 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:44:30 +0200 Subject: [PATCH 04/59] Rename invalid operation requested event --- link/domain/events.py | 4 ++-- link/domain/link.py | 4 ++-- link/domain/state.py | 4 ++-- link/service/services.py | 4 ++-- tests/integration/test_services.py | 2 +- tests/unit/entities/test_state.py | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 89e43452..6f91ef7d 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -19,8 +19,8 @@ class Event: @dataclass(frozen=True) -class InvalidOperation(Event): - """Represents the result of attempting an operation that is invalid in the entity's current state.""" +class InvalidOperationRequested(Event): + """An operation that is invalid given the entities current state was requested.""" state: type[State] diff --git a/link/domain/link.py b/link/domain/link.py index 4631f2be..9da89a84 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -119,7 +119,7 @@ def create_operation_result(results: Iterable[events.Event]) -> LinkOperationRes return LinkOperationResult( operation, updates=frozenset(result for result in results if isinstance(result, events.Update)), - errors=frozenset(result for result in results if isinstance(result, events.InvalidOperation)), + errors=frozenset(result for result in results if isinstance(result, events.InvalidOperationRequested)), ) assert requested, "No identifiers requested." @@ -150,7 +150,7 @@ class LinkOperationResult: operation: Operations updates: frozenset[events.Update] - errors: frozenset[events.InvalidOperation] + errors: frozenset[events.InvalidOperationRequested] def __post_init__(self) -> None: """Validate the result.""" diff --git a/link/domain/state.py b/link/domain/state.py index 5679402b..63802ffa 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -6,7 +6,7 @@ from functools import partial from .custom_types import Identifier -from .events import Event, InvalidOperation, Update +from .events import Event, InvalidOperationRequested, Update class State: @@ -29,7 +29,7 @@ def process(cls, entity: Entity) -> Entity: @staticmethod def _create_invalid_operation(entity: Entity, operation: Operations) -> Entity: - updated = entity.operation_results + (InvalidOperation(operation, entity.identifier, entity.state),) + updated = entity.operation_results + (InvalidOperationRequested(operation, entity.identifier, entity.state),) return replace(entity, operation_results=updated) @classmethod diff --git a/link/service/services.py b/link/service/services.py index 71e4f50f..0a29950f 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -32,7 +32,7 @@ class PullResponse(Response): """Response model for the pull use-case.""" requested: frozenset[Identifier] - errors: frozenset[events.InvalidOperation] + errors: frozenset[events.InvalidOperationRequested] def pull( @@ -111,7 +111,7 @@ class OperationResponse(Response): operation: Operations requested: frozenset[Identifier] updates: frozenset[events.Update] - errors: frozenset[events.InvalidOperation] + errors: frozenset[events.InvalidOperationRequested] @dataclass(frozen=True) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index d1c3101b..1067f0ed 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -231,7 +231,7 @@ def test_pulled_entity_ends_in_correct_state(state: EntityConfig, expected: type def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityConfig, produces_error: bool) -> None: if produces_error: errors = { - events.InvalidOperation( + events.InvalidOperationRequested( operation=Operations.START_PULL, identifier=create_identifier("1"), state=states.Deprecated ) } diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index 4dfee5bd..33ff7e91 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -51,7 +51,7 @@ def test_invalid_transitions_returns_unchanged_entity( ) entity = next(entity for entity in link if entity.identifier == identifier) for operation in operations: - result = events.InvalidOperation(operation, identifier, state) + result = events.InvalidOperationRequested(operation, identifier, state) assert entity.apply(operation) == replace(entity, operation_results=(result,)) From fae1447e57ecf230eba9eecd46cbe662d69178b1 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:51:35 +0200 Subject: [PATCH 05/59] Rename entity state changed event --- link/adapters/gateway.py | 4 ++-- link/domain/events.py | 4 ++-- link/domain/link.py | 4 ++-- link/domain/state.py | 4 ++-- link/service/gateway.py | 2 +- link/service/services.py | 2 +- link/service/uow.py | 4 ++-- tests/integration/gateway.py | 2 +- tests/unit/entities/test_state.py | 12 +++++++----- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/link/adapters/gateway.py b/link/adapters/gateway.py index 1a6658a3..c05cf577 100644 --- a/link/adapters/gateway.py +++ b/link/adapters/gateway.py @@ -52,10 +52,10 @@ def translate_tainted_primary_keys(primary_keys: Iterable[PrimaryKey]) -> set[Id tainted_identifiers=translate_tainted_primary_keys(self.facade.get_tainted_primary_keys()), ) - def apply(self, updates: Iterable[events.Update]) -> None: + def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: """Apply updates to the persistent data representing the link.""" - def keyfunc(update: events.Update) -> int: + def keyfunc(update: events.EntityStateChanged) -> int: assert update.command is not None return update.command.value diff --git a/link/domain/events.py b/link/domain/events.py index 6f91ef7d..5fb8262d 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -26,8 +26,8 @@ class InvalidOperationRequested(Event): @dataclass(frozen=True) -class Update(Event): - """Represents the persistent update needed to transition an entity.""" +class EntityStateChanged(Event): + """The state of an entity changed during the application of an operation.""" transition: Transition command: Commands diff --git a/link/domain/link.py b/link/domain/link.py index 9da89a84..1becd9ab 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -118,7 +118,7 @@ def create_operation_result(results: Iterable[events.Event]) -> LinkOperationRes operation = next(iter(results)).operation return LinkOperationResult( operation, - updates=frozenset(result for result in results if isinstance(result, events.Update)), + updates=frozenset(result for result in results if isinstance(result, events.EntityStateChanged)), errors=frozenset(result for result in results if isinstance(result, events.InvalidOperationRequested)), ) @@ -149,7 +149,7 @@ class LinkOperationResult: """Represents the result of an operation on all entities of a link.""" operation: Operations - updates: frozenset[events.Update] + updates: frozenset[events.EntityStateChanged] errors: frozenset[events.InvalidOperationRequested] def __post_init__(self) -> None: diff --git a/link/domain/state.py b/link/domain/state.py index 63802ffa..db990b3a 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -6,7 +6,7 @@ from functools import partial from .custom_types import Identifier -from .events import Event, InvalidOperationRequested, Update +from .events import EntityStateChanged, Event, InvalidOperationRequested class State: @@ -40,7 +40,7 @@ def _transition_entity( new_process = entity.current_process transition = Transition(cls, new_state) updated_results = entity.operation_results + ( - Update(operation, entity.identifier, transition, TRANSITION_MAP[transition]), + EntityStateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), ) return replace(entity, state=transition.new, current_process=new_process, operation_results=updated_results) diff --git a/link/service/gateway.py b/link/service/gateway.py index da2b5e4e..fdc0552f 100644 --- a/link/service/gateway.py +++ b/link/service/gateway.py @@ -16,5 +16,5 @@ def create_link(self) -> Link: """Create a link from the persistent data.""" @abstractmethod - def apply(self, updates: Iterable[events.Update]) -> None: + def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: """Apply updates to the link's persistent data.""" diff --git a/link/service/services.py b/link/service/services.py index 0a29950f..73e3ba31 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -110,7 +110,7 @@ class OperationResponse(Response): operation: Operations requested: frozenset[Identifier] - updates: frozenset[events.Update] + updates: frozenset[events.EntityStateChanged] errors: frozenset[events.InvalidOperationRequested] diff --git a/link/service/uow.py b/link/service/uow.py index 178394bd..40d11103 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -7,7 +7,7 @@ from typing import Callable, Iterable, Protocol from link.domain.custom_types import Identifier -from link.domain.events import Update +from link.domain.events import EntityStateChanged from link.domain.link import Link from link.domain.state import Entity, Operations @@ -97,7 +97,7 @@ def commit(self) -> None: if self._link is None: raise RuntimeError("Not available outside of context") for entity in self._seen.values(): - updates = deque(event for event in entity.operation_results if isinstance(event, Update)) + updates = deque(event for event in entity.operation_results if isinstance(event, EntityStateChanged)) while updates: self._gateway.apply([updates.popleft()]) self.rollback() diff --git a/tests/integration/gateway.py b/tests/integration/gateway.py index 5961236a..94ce53f1 100644 --- a/tests/integration/gateway.py +++ b/tests/integration/gateway.py @@ -28,7 +28,7 @@ def __init__( def create_link(self) -> Link: return create_link(self.assignments, tainted_identifiers=self.tainted_identifiers, processes=self.processes) - def apply(self, updates: Iterable[events.Update]) -> None: + def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: for update in updates: if update.command is Commands.START_PULL_PROCESS: self.processes[Processes.PULL].add(update.identifier) diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index 33ff7e91..40312db7 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -63,7 +63,7 @@ def test_start_pulling_idle_entity_returns_correct_entity() -> None: state=states.Activated, current_process=Processes.PULL, operation_results=( - events.Update( + events.EntityStateChanged( Operations.START_PULL, entity.identifier, Transition(states.Idle, states.Activated), @@ -96,7 +96,7 @@ def test_processing_activated_entity_returns_correct_entity( ) entity = next(iter(link)) updated_results = entity.operation_results + ( - events.Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) assert entity.apply(Operations.PROCESS) == replace( entity, state=new_state, current_process=new_process, operation_results=updated_results @@ -126,7 +126,7 @@ def test_processing_received_entity_returns_correct_entity( ) entity = next(iter(link)) operation_results = ( - events.Update(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) assert entity.apply(Operations.PROCESS) == replace( entity, state=new_state, current_process=new_process, operation_results=operation_results @@ -140,7 +140,7 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: entity = next(iter(link)) transition = Transition(states.Pulled, states.Received) operation_results = ( - events.Update( + events.EntityStateChanged( Operations.START_DELETE, entity.identifier, transition, @@ -160,7 +160,9 @@ def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: entity = next(iter(link)) transition = Transition(states.Tainted, states.Received) operation_results = ( - events.Update(Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS), + events.EntityStateChanged( + Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS + ), ) assert entity.apply(Operations.START_DELETE) == replace( entity, state=transition.new, current_process=Processes.DELETE, operation_results=operation_results From 4bc5d613310c25d95a96a89429db0c7c995f4e91 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:53:54 +0200 Subject: [PATCH 06/59] Replace base event with entity operation applied --- link/domain/events.py | 8 ++++---- link/domain/link.py | 2 +- link/domain/state.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 5fb8262d..9b073a58 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -11,22 +11,22 @@ @dataclass(frozen=True) -class Event: - """Base class for all events.""" +class EntityOperationApplied: + """An operation was applied to an entity.""" operation: Operations identifier: Identifier @dataclass(frozen=True) -class InvalidOperationRequested(Event): +class InvalidOperationRequested(EntityOperationApplied): """An operation that is invalid given the entities current state was requested.""" state: type[State] @dataclass(frozen=True) -class EntityStateChanged(Event): +class EntityStateChanged(EntityOperationApplied): """The state of an entity changed during the application of an operation.""" transition: Transition diff --git a/link/domain/link.py b/link/domain/link.py index 1becd9ab..61a11036 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -112,7 +112,7 @@ def operation_results(self) -> Tuple[LinkOperationResult, ...]: def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" - def create_operation_result(results: Iterable[events.Event]) -> LinkOperationResult: + def create_operation_result(results: Iterable[events.EntityOperationApplied]) -> LinkOperationResult: """Create the result of an operation on a link from results of individual entities.""" results = set(results) operation = next(iter(results)).operation diff --git a/link/domain/state.py b/link/domain/state.py index db990b3a..fd9b50d9 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -6,7 +6,7 @@ from functools import partial from .custom_types import Identifier -from .events import EntityStateChanged, Event, InvalidOperationRequested +from .events import EntityOperationApplied, EntityStateChanged, InvalidOperationRequested class State: @@ -268,7 +268,7 @@ class Entity: state: type[State] current_process: Processes is_tainted: bool - operation_results: tuple[Event, ...] + operation_results: tuple[EntityOperationApplied, ...] def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" From 6c52cf864bd2732d9c8676c2f8bdab1864db0522 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:55:03 +0200 Subject: [PATCH 07/59] Add event base class --- link/domain/events.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/link/domain/events.py b/link/domain/events.py index 9b073a58..31d1d951 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -11,7 +11,12 @@ @dataclass(frozen=True) -class EntityOperationApplied: +class Event: + """Base class for all events.""" + + +@dataclass(frozen=True) +class EntityOperationApplied(Event): """An operation was applied to an entity.""" operation: Operations From 182a95175a4444a8404d676f8d664065af41f561 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:58:37 +0200 Subject: [PATCH 08/59] Add link state changed event --- link/domain/events.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/link/domain/events.py b/link/domain/events.py index 31d1d951..fbba8c4f 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -36,3 +36,19 @@ class EntityStateChanged(EntityOperationApplied): transition: Transition command: Commands + + +@dataclass(frozen=True) +class LinkStateChanged(Event): + """The state of a link changed during the application of an operation.""" + + operation: Operations + requested: frozenset[Identifier] + updates: frozenset[EntityStateChanged] + errors: frozenset[InvalidOperationRequested] + + def __post_init__(self) -> None: + """Validate the event.""" + assert all( + result.operation is self.operation for result in (self.updates | self.errors) + ), "Not all events have same operation." From 60579574ff057f3821c75f08311859acea3a158d Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:06:26 +0200 Subject: [PATCH 09/59] Replace link operation result with event --- link/domain/link.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 61a11036..404d2a85 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -1,7 +1,6 @@ """Contains the link class.""" from __future__ import annotations -from dataclasses import dataclass from typing import Any, Iterable, Iterator, Mapping, Optional, Set, Tuple, TypeVar from . import events @@ -93,7 +92,7 @@ class Link(Set[Entity]): """The state of a link between two databases.""" def __init__( - self, entities: Iterable[Entity], operation_results: Tuple[LinkOperationResult, ...] = tuple() + self, entities: Iterable[Entity], operation_results: Tuple[events.LinkStateChanged, ...] = tuple() ) -> None: """Initialize the link.""" self._entities = set(entities) @@ -105,19 +104,22 @@ def identifiers(self) -> frozenset[Identifier]: return frozenset(entity.identifier for entity in self) @property - def operation_results(self) -> Tuple[LinkOperationResult, ...]: + def operation_results(self) -> Tuple[events.LinkStateChanged, ...]: """Return the results of operations performed on this link.""" return self._operation_results def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" - def create_operation_result(results: Iterable[events.EntityOperationApplied]) -> LinkOperationResult: + def create_operation_result( + results: Iterable[events.EntityOperationApplied], requested: Iterable[Identifier] + ) -> events.LinkStateChanged: """Create the result of an operation on a link from results of individual entities.""" results = set(results) operation = next(iter(results)).operation - return LinkOperationResult( + return events.LinkStateChanged( operation, + requested=frozenset(requested), updates=frozenset(result for result in results if isinstance(result, events.EntityStateChanged)), errors=frozenset(result for result in results if isinstance(result, events.InvalidOperationRequested)), ) @@ -127,7 +129,7 @@ def create_operation_result(results: Iterable[events.EntityOperationApplied]) -> changed = {entity.apply(operation) for entity in self if entity.identifier in requested} unchanged = {entity for entity in self if entity.identifier not in requested} operation_results = self.operation_results + ( - create_operation_result(entity.operation_results[-1] for entity in changed), + create_operation_result((entity.operation_results[-1] for entity in changed), requested), ) return Link(changed | unchanged, operation_results) @@ -142,18 +144,3 @@ def __iter__(self) -> Iterator[Entity]: def __len__(self) -> int: """Return the number of entities in the link.""" return len(self._entities) - - -@dataclass(frozen=True) -class LinkOperationResult: - """Represents the result of an operation on all entities of a link.""" - - operation: Operations - updates: frozenset[events.EntityStateChanged] - errors: frozenset[events.InvalidOperationRequested] - - def __post_init__(self) -> None: - """Validate the result.""" - assert all( - result.operation is self.operation for result in (self.updates | self.errors) - ), "Not all results have same operation." From 2894d136114279bce65f293fd45c777b87d3b62c Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:09:05 +0200 Subject: [PATCH 10/59] Rename operation results attribute on link --- link/domain/link.py | 14 +++++----- link/service/services.py | 6 ++--- .../integration/test_datajoint_persistence.py | 26 +++++++++---------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 404d2a85..82191240 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -91,12 +91,10 @@ def assign_to_component(component: Components) -> set[Entity]: class Link(Set[Entity]): """The state of a link between two databases.""" - def __init__( - self, entities: Iterable[Entity], operation_results: Tuple[events.LinkStateChanged, ...] = tuple() - ) -> None: + def __init__(self, entities: Iterable[Entity], events: Tuple[events.LinkStateChanged, ...] = tuple()) -> None: """Initialize the link.""" self._entities = set(entities) - self._operation_results = operation_results + self._events = events @property def identifiers(self) -> frozenset[Identifier]: @@ -104,9 +102,9 @@ def identifiers(self) -> frozenset[Identifier]: return frozenset(entity.identifier for entity in self) @property - def operation_results(self) -> Tuple[events.LinkStateChanged, ...]: - """Return the results of operations performed on this link.""" - return self._operation_results + def events(self) -> Tuple[events.LinkStateChanged, ...]: + """Return the events that happened to the link.""" + return self._events def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" @@ -128,7 +126,7 @@ def create_operation_result( assert set(requested) <= self.identifiers, "Requested identifiers not present in link." changed = {entity.apply(operation) for entity in self if entity.identifier in requested} unchanged = {entity for entity in self if entity.identifier not in requested} - operation_results = self.operation_results + ( + operation_results = self.events + ( create_operation_result((entity.operation_results[-1] for entity in changed), requested), ) return Link(changed | unchanged, operation_results) diff --git a/link/service/services.py b/link/service/services.py index 73e3ba31..7165fb7f 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -129,7 +129,7 @@ def start_pull_process( ) -> None: """Start the pull process for the requested entities.""" with uow: - result = uow.link.apply(Operations.START_PULL, requested=request.requested).operation_results[0] + result = uow.link.apply(Operations.START_PULL, requested=request.requested).events[0] uow.commit() output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) @@ -149,7 +149,7 @@ def start_delete_process( ) -> None: """Start the delete process for the requested entities.""" with uow: - result = uow.link.apply(Operations.START_DELETE, requested=request.requested).operation_results[0] + result = uow.link.apply(Operations.START_DELETE, requested=request.requested).events[0] uow.commit() output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) @@ -164,7 +164,7 @@ class ProcessRequest(Request): def process(request: ProcessRequest, *, uow: UnitOfWork, output_port: Callable[[OperationResponse], None]) -> None: """Process entities.""" with uow: - result = uow.link.apply(Operations.PROCESS, requested=request.requested).operation_results[0] + result = uow.link.apply(Operations.PROCESS, requested=request.requested).events[0] uow.commit() output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) diff --git a/tests/integration/test_datajoint_persistence.py b/tests/integration/test_datajoint_persistence.py index 0de89752..89445517 100644 --- a/tests/integration/test_datajoint_persistence.py +++ b/tests/integration/test_datajoint_persistence.py @@ -373,7 +373,7 @@ def test_add_to_local_command() -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -416,7 +416,7 @@ def test_add_to_local_command_with_error() -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) except RuntimeError: @@ -438,7 +438,7 @@ def test_add_to_local_command_with_external_file(tmpdir: Path) -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) fetch_filepath = Path(tables["local"].fetch(as_dict=True, download_path=str(tmpdir))[0]["external"]) @@ -462,7 +462,7 @@ def test_remove_from_local_command() -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -483,7 +483,7 @@ def test_start_pull_process() -> None: gateway.apply( gateway.create_link() .apply(Operations.START_PULL, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -513,7 +513,7 @@ def test_state_after_command(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -535,7 +535,7 @@ def test_rollback_on_error(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) except RuntimeError: @@ -561,7 +561,7 @@ def test_state_after_command(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.START_DELETE, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -583,7 +583,7 @@ def test_rollback_on_error(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.START_DELETE, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) except RuntimeError: @@ -606,7 +606,7 @@ def test_finish_delete_process_command() -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -629,7 +629,7 @@ def test_state_after_command(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) @@ -650,7 +650,7 @@ def test_rollback_on_error(initial_state: State) -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .operation_results[0] + .events[0] .updates ) except RuntimeError: @@ -680,7 +680,7 @@ def test_applying_multiple_commands() -> None: gateway.apply( gateway.create_link() .apply(Operations.PROCESS, requested=gateway.translator.to_identifiers([{"a": 0}, {"a": 1}])) - .operation_results[0] + .events[0] .updates ) From f71ed5b28e78eb058d251e91ca375137c84dff4d Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:31:14 +0200 Subject: [PATCH 11/59] Rename entity operation results attribute to events --- link/domain/link.py | 4 ++-- link/domain/state.py | 10 +++++----- link/service/uow.py | 2 +- tests/unit/entities/test_state.py | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 82191240..4e9250f2 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -67,7 +67,7 @@ def create_entity(identifier: Identifier) -> Entity: state=state, current_process=processes_map.get(identifier, Processes.NONE), is_tainted=is_tainted(identifier), - operation_results=tuple(), + events=tuple(), ) return {create_entity(identifier) for identifier in assignments[Components.SOURCE]} @@ -127,7 +127,7 @@ def create_operation_result( changed = {entity.apply(operation) for entity in self if entity.identifier in requested} unchanged = {entity for entity in self if entity.identifier not in requested} operation_results = self.events + ( - create_operation_result((entity.operation_results[-1] for entity in changed), requested), + create_operation_result((entity.events[-1] for entity in changed), requested), ) return Link(changed | unchanged, operation_results) diff --git a/link/domain/state.py b/link/domain/state.py index fd9b50d9..5fcadc96 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -29,8 +29,8 @@ def process(cls, entity: Entity) -> Entity: @staticmethod def _create_invalid_operation(entity: Entity, operation: Operations) -> Entity: - updated = entity.operation_results + (InvalidOperationRequested(operation, entity.identifier, entity.state),) - return replace(entity, operation_results=updated) + updated = entity.events + (InvalidOperationRequested(operation, entity.identifier, entity.state),) + return replace(entity, events=updated) @classmethod def _transition_entity( @@ -39,10 +39,10 @@ def _transition_entity( if new_process is None: new_process = entity.current_process transition = Transition(cls, new_state) - updated_results = entity.operation_results + ( + updated_events = entity.events + ( EntityStateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), ) - return replace(entity, state=transition.new, current_process=new_process, operation_results=updated_results) + return replace(entity, state=transition.new, current_process=new_process, events=updated_events) class States: @@ -268,7 +268,7 @@ class Entity: state: type[State] current_process: Processes is_tainted: bool - operation_results: tuple[EntityOperationApplied, ...] + events: tuple[EntityOperationApplied, ...] def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" diff --git a/link/service/uow.py b/link/service/uow.py index 40d11103..38855818 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -97,7 +97,7 @@ def commit(self) -> None: if self._link is None: raise RuntimeError("Not available outside of context") for entity in self._seen.values(): - updates = deque(event for event in entity.operation_results if isinstance(event, EntityStateChanged)) + updates = deque(event for event in entity.events if isinstance(event, EntityStateChanged)) while updates: self._gateway.apply([updates.popleft()]) self.rollback() diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index 40312db7..fe63e2d8 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -52,7 +52,7 @@ def test_invalid_transitions_returns_unchanged_entity( entity = next(entity for entity in link if entity.identifier == identifier) for operation in operations: result = events.InvalidOperationRequested(operation, identifier, state) - assert entity.apply(operation) == replace(entity, operation_results=(result,)) + assert entity.apply(operation) == replace(entity, events=(result,)) def test_start_pulling_idle_entity_returns_correct_entity() -> None: @@ -62,7 +62,7 @@ def test_start_pulling_idle_entity_returns_correct_entity() -> None: entity, state=states.Activated, current_process=Processes.PULL, - operation_results=( + events=( events.EntityStateChanged( Operations.START_PULL, entity.identifier, @@ -95,11 +95,11 @@ def test_processing_activated_entity_returns_correct_entity( tainted_identifiers=tainted_identifiers, ) entity = next(iter(link)) - updated_results = entity.operation_results + ( + updated_results = entity.events + ( events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) assert entity.apply(Operations.PROCESS) == replace( - entity, state=new_state, current_process=new_process, operation_results=updated_results + entity, state=new_state, current_process=new_process, events=updated_results ) @@ -125,11 +125,11 @@ def test_processing_received_entity_returns_correct_entity( tainted_identifiers=tainted_identifiers, ) entity = next(iter(link)) - operation_results = ( + expected_events = ( events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) assert entity.apply(Operations.PROCESS) == replace( - entity, state=new_state, current_process=new_process, operation_results=operation_results + entity, state=new_state, current_process=new_process, events=expected_events ) @@ -139,7 +139,7 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: ) entity = next(iter(link)) transition = Transition(states.Pulled, states.Received) - operation_results = ( + expected_events = ( events.EntityStateChanged( Operations.START_DELETE, entity.identifier, @@ -148,7 +148,7 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: ), ) assert entity.apply(Operations.START_DELETE) == replace( - entity, state=transition.new, current_process=Processes.DELETE, operation_results=operation_results + entity, state=transition.new, current_process=Processes.DELETE, events=expected_events ) @@ -159,11 +159,11 @@ def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: ) entity = next(iter(link)) transition = Transition(states.Tainted, states.Received) - operation_results = ( + expected_events = ( events.EntityStateChanged( Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS ), ) assert entity.apply(Operations.START_DELETE) == replace( - entity, state=transition.new, current_process=Processes.DELETE, operation_results=operation_results + entity, state=transition.new, current_process=Processes.DELETE, events=expected_events ) From 7ae93de8e9e67a3bb4ddf5293406bf76f36113c3 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:37:59 +0200 Subject: [PATCH 12/59] Use link state changed event in place of result --- link/adapters/present.py | 10 ++++------ link/service/services.py | 30 +++++++++++------------------- tests/integration/test_services.py | 9 ++++----- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/link/adapters/present.py b/link/adapters/present.py index 28c4e47e..729b1950 100644 --- a/link/adapters/present.py +++ b/link/adapters/present.py @@ -4,10 +4,8 @@ from dataclasses import dataclass from typing import Callable, Iterable -from link.service.services import ( - ListIdleEntitiesResponse, - OperationResponse, -) +from link.domain import events +from link.service.services import ListIdleEntitiesResponse from .custom_types import PrimaryKey from .identification import IdentificationTranslator @@ -58,13 +56,13 @@ class Failure: def create_operation_response_presenter( translator: IdentificationTranslator, show: Callable[[OperationRecord], None] -) -> Callable[[OperationResponse], None]: +) -> Callable[[events.LinkStateChanged], None]: """Create a callable that when called presents information about a finished operation.""" def get_class_name(obj: type) -> str: return obj.__name__ - def present_operation_response(response: OperationResponse) -> None: + def present_operation_response(response: events.LinkStateChanged) -> None: show( OperationRecord( [ diff --git a/link/service/services.py b/link/service/services.py index 7165fb7f..37cee768 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -39,7 +39,7 @@ def pull( request: PullRequest, *, process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], - start_pull_process_service: Callable[[StartPullProcessRequest], OperationResponse], + start_pull_process_service: Callable[[StartPullProcessRequest], events.LinkStateChanged], output_port: Callable[[PullResponse], None], ) -> None: """Pull entities across the link.""" @@ -68,7 +68,7 @@ def delete( request: DeleteRequest, *, process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], - start_delete_process_service: Callable[[StartDeleteProcessRequest], OperationResponse], + start_delete_process_service: Callable[[StartDeleteProcessRequest], events.LinkStateChanged], output_port: Callable[[DeleteResponse], None], ) -> None: """Delete pulled entities.""" @@ -95,7 +95,7 @@ class ProcessToCompletionResponse(Response): def process_to_completion( request: ProcessToCompletionRequest, *, - process_service: Callable[[ProcessRequest], OperationResponse], + process_service: Callable[[ProcessRequest], events.LinkStateChanged], output_port: Callable[[ProcessToCompletionResponse], None], ) -> None: """Process entities until their processes are complete.""" @@ -104,16 +104,6 @@ def process_to_completion( output_port(ProcessToCompletionResponse(request.requested)) -@dataclass(frozen=True) -class OperationResponse(Response): - """Response model for all use-cases that operate on entities.""" - - operation: Operations - requested: frozenset[Identifier] - updates: frozenset[events.EntityStateChanged] - errors: frozenset[events.InvalidOperationRequested] - - @dataclass(frozen=True) class StartPullProcessRequest(Request): """Request model for the start-pull-process service.""" @@ -125,13 +115,13 @@ def start_pull_process( request: StartPullProcessRequest, *, uow: UnitOfWork, - output_port: Callable[[OperationResponse], None], + output_port: Callable[[events.LinkStateChanged], None], ) -> None: """Start the pull process for the requested entities.""" with uow: result = uow.link.apply(Operations.START_PULL, requested=request.requested).events[0] uow.commit() - output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) + output_port(result) @dataclass(frozen=True) @@ -145,13 +135,13 @@ def start_delete_process( request: StartDeleteProcessRequest, *, uow: UnitOfWork, - output_port: Callable[[OperationResponse], None], + output_port: Callable[[events.LinkStateChanged], None], ) -> None: """Start the delete process for the requested entities.""" with uow: result = uow.link.apply(Operations.START_DELETE, requested=request.requested).events[0] uow.commit() - output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) + output_port(result) @dataclass(frozen=True) @@ -161,12 +151,14 @@ class ProcessRequest(Request): requested: frozenset[Identifier] -def process(request: ProcessRequest, *, uow: UnitOfWork, output_port: Callable[[OperationResponse], None]) -> None: +def process( + request: ProcessRequest, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None] +) -> None: """Process entities.""" with uow: result = uow.link.apply(Operations.PROCESS, requested=request.requested).events[0] uow.commit() - output_port(OperationResponse(result.operation, request.requested, result.updates, result.errors)) + output_port(result) @dataclass(frozen=True) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 1067f0ed..90ab23c2 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -2,7 +2,7 @@ from collections.abc import Callable from functools import partial -from typing import Generic, TypedDict, TypeVar +from typing import Generic, TypedDict, TypeVar, Union import pytest @@ -14,7 +14,6 @@ DeleteResponse, ListIdleEntitiesRequest, ListIdleEntitiesResponse, - OperationResponse, ProcessRequest, ProcessToCompletionRequest, PullRequest, @@ -33,7 +32,7 @@ from .gateway import FakeLinkGateway -T = TypeVar("T", bound=Response) +T = TypeVar("T", bound=Union[Response, events.Event]) class FakeOutputPort(Generic[T]): @@ -257,7 +256,7 @@ def test_entity_undergoing_process_gets_processed() -> None: process( ProcessRequest(frozenset(create_identifiers("1"))), uow=uow, - output_port=FakeOutputPort[OperationResponse](), + output_port=FakeOutputPort[events.LinkStateChanged](), ) with uow: entity = next(entity for entity in uow.link if entity.identifier == create_identifier("1")) @@ -271,7 +270,7 @@ def test_correct_response_model_gets_passed_to_process_output_port() -> None: processes={Processes.PULL: create_identifiers("1")}, ) ) - output_port = FakeOutputPort[OperationResponse]() + output_port = FakeOutputPort[events.LinkStateChanged]() process( ProcessRequest(frozenset(create_identifiers("1"))), uow=uow, From f1acb7d4de09f2421e74f252a606ee277dacb6c1 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:43:42 +0200 Subject: [PATCH 13/59] Add commands module --- link/domain/commands.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 link/domain/commands.py diff --git a/link/domain/commands.py b/link/domain/commands.py new file mode 100644 index 00000000..96786b72 --- /dev/null +++ b/link/domain/commands.py @@ -0,0 +1,18 @@ +"""Contains all domain commands.""" +from __future__ import annotations + +from dataclasses import dataclass + +from .custom_types import Identifier + + +@dataclass(frozen=True) +class Command: + """Base class for all commands.""" + + +@dataclass(frozen=True) +class ProcessLink(Command): + """Process the requested entities in the link.""" + + requested: frozenset[Identifier] From 6dead2a148129a37e44579897d2daa01f2deb96e Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:45:48 +0200 Subject: [PATCH 14/59] Replace process request with command --- link/service/services.py | 17 +++++------------ tests/integration/test_services.py | 7 +++---- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/link/service/services.py b/link/service/services.py index 37cee768..c8acd796 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from enum import Enum, auto -from link.domain import events +from link.domain import commands, events from link.domain.custom_types import Identifier from link.domain.state import Operations, states @@ -95,11 +95,11 @@ class ProcessToCompletionResponse(Response): def process_to_completion( request: ProcessToCompletionRequest, *, - process_service: Callable[[ProcessRequest], events.LinkStateChanged], + process_service: Callable[[commands.ProcessLink], events.LinkStateChanged], output_port: Callable[[ProcessToCompletionResponse], None], ) -> None: """Process entities until their processes are complete.""" - while process_service(ProcessRequest(request.requested)).updates: + while process_service(commands.ProcessLink(request.requested)).updates: pass output_port(ProcessToCompletionResponse(request.requested)) @@ -144,19 +144,12 @@ def start_delete_process( output_port(result) -@dataclass(frozen=True) -class ProcessRequest(Request): - """Request model for the process use-case.""" - - requested: frozenset[Identifier] - - def process( - request: ProcessRequest, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None] + command: commands.ProcessLink, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None] ) -> None: """Process entities.""" with uow: - result = uow.link.apply(Operations.PROCESS, requested=request.requested).events[0] + result = uow.link.apply(Operations.PROCESS, requested=command.requested).events[0] uow.commit() output_port(result) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 90ab23c2..614b0520 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -6,7 +6,7 @@ import pytest -from link.domain import events +from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states from link.service.io import Service, make_responsive from link.service.services import ( @@ -14,7 +14,6 @@ DeleteResponse, ListIdleEntitiesRequest, ListIdleEntitiesResponse, - ProcessRequest, ProcessToCompletionRequest, PullRequest, PullResponse, @@ -254,7 +253,7 @@ def test_entity_undergoing_process_gets_processed() -> None: ) ) process( - ProcessRequest(frozenset(create_identifiers("1"))), + commands.ProcessLink(frozenset(create_identifiers("1"))), uow=uow, output_port=FakeOutputPort[events.LinkStateChanged](), ) @@ -272,7 +271,7 @@ def test_correct_response_model_gets_passed_to_process_output_port() -> None: ) output_port = FakeOutputPort[events.LinkStateChanged]() process( - ProcessRequest(frozenset(create_identifiers("1"))), + commands.ProcessLink(frozenset(create_identifiers("1"))), uow=uow, output_port=output_port, ) From 7e8d778d5b4e92b7888ee8a0e20409fd0e11bae3 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:58:46 +0200 Subject: [PATCH 15/59] Add commands for starting processes --- link/domain/commands.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/link/domain/commands.py b/link/domain/commands.py index 96786b72..7856b55e 100644 --- a/link/domain/commands.py +++ b/link/domain/commands.py @@ -10,9 +10,19 @@ class Command: """Base class for all commands.""" + requested: frozenset[Identifier] + @dataclass(frozen=True) class ProcessLink(Command): """Process the requested entities in the link.""" - requested: frozenset[Identifier] + +@dataclass(frozen=True) +class StartPullProcess(Command): + """Start the pull process for the requested entities.""" + + +@dataclass(frozen=True) +class StartDeleteProcess(Command): + """Start the delete process for the requested entities.""" From 28360d074e22b8cf78dc77d0f7e85e36f8124458 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:04:10 +0200 Subject: [PATCH 16/59] Use process starting commands in place of requests --- link/service/services.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/link/service/services.py b/link/service/services.py index c8acd796..42f2ea3a 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -39,12 +39,12 @@ def pull( request: PullRequest, *, process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], - start_pull_process_service: Callable[[StartPullProcessRequest], events.LinkStateChanged], + start_pull_process_service: Callable[[commands.StartPullProcess], events.LinkStateChanged], output_port: Callable[[PullResponse], None], ) -> None: """Pull entities across the link.""" process_to_completion_service(ProcessToCompletionRequest(request.requested)) - response = start_pull_process_service(StartPullProcessRequest(request.requested)) + response = start_pull_process_service(commands.StartPullProcess(request.requested)) errors = (error for error in response.errors if error.state is states.Deprecated) process_to_completion_service(ProcessToCompletionRequest(request.requested)) output_port(PullResponse(request.requested, errors=frozenset(errors))) @@ -68,12 +68,12 @@ def delete( request: DeleteRequest, *, process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], - start_delete_process_service: Callable[[StartDeleteProcessRequest], events.LinkStateChanged], + start_delete_process_service: Callable[[commands.StartDeleteProcess], events.LinkStateChanged], output_port: Callable[[DeleteResponse], None], ) -> None: """Delete pulled entities.""" process_to_completion_service(ProcessToCompletionRequest(request.requested)) - start_delete_process_service(StartDeleteProcessRequest(request.requested)) + start_delete_process_service(commands.StartDeleteProcess(request.requested)) process_to_completion_service(ProcessToCompletionRequest(request.requested)) output_port(DeleteResponse(request.requested)) @@ -104,42 +104,28 @@ def process_to_completion( output_port(ProcessToCompletionResponse(request.requested)) -@dataclass(frozen=True) -class StartPullProcessRequest(Request): - """Request model for the start-pull-process service.""" - - requested: frozenset[Identifier] - - def start_pull_process( - request: StartPullProcessRequest, + command: commands.StartPullProcess, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None], ) -> None: """Start the pull process for the requested entities.""" with uow: - result = uow.link.apply(Operations.START_PULL, requested=request.requested).events[0] + result = uow.link.apply(Operations.START_PULL, requested=command.requested).events[0] uow.commit() output_port(result) -@dataclass(frozen=True) -class StartDeleteProcessRequest(Request): - """Request model for the start-delete-process service.""" - - requested: frozenset[Identifier] - - def start_delete_process( - request: StartDeleteProcessRequest, + command: commands.StartDeleteProcess, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None], ) -> None: """Start the delete process for the requested entities.""" with uow: - result = uow.link.apply(Operations.START_DELETE, requested=request.requested).events[0] + result = uow.link.apply(Operations.START_DELETE, requested=command.requested).events[0] uow.commit() output_port(result) From 3a2e58f445364133eeddfe64a07f62803b338de7 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:14:41 +0200 Subject: [PATCH 17/59] Add list idle entities command and event --- link/domain/commands.py | 13 +++++++++++-- link/domain/events.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/link/domain/commands.py b/link/domain/commands.py index 7856b55e..d495ebd7 100644 --- a/link/domain/commands.py +++ b/link/domain/commands.py @@ -10,19 +10,28 @@ class Command: """Base class for all commands.""" - requested: frozenset[Identifier] - @dataclass(frozen=True) class ProcessLink(Command): """Process the requested entities in the link.""" + requested: frozenset[Identifier] + @dataclass(frozen=True) class StartPullProcess(Command): """Start the pull process for the requested entities.""" + requested: frozenset[Identifier] + @dataclass(frozen=True) class StartDeleteProcess(Command): """Start the delete process for the requested entities.""" + + requested: frozenset[Identifier] + + +@dataclass(frozen=True) +class ListIdleEntities(Command): + """Start the delete process for the requested entities.""" diff --git a/link/domain/events.py b/link/domain/events.py index fbba8c4f..181141d6 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -52,3 +52,10 @@ def __post_init__(self) -> None: assert all( result.operation is self.operation for result in (self.updates | self.errors) ), "Not all events have same operation." + + +@dataclass(frozen=True) +class IdleEntitiesListed(Event): + """Idle entities in a link have been listed.""" + + identifiers: frozenset[Identifier] From 1118b996a5ed705a037c2ec2589476aedfc4e87d Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:08:27 +0200 Subject: [PATCH 18/59] Make events attribute on link mutable --- link/domain/link.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 4e9250f2..4b3a9aa2 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -91,10 +91,10 @@ def assign_to_component(component: Components) -> set[Entity]: class Link(Set[Entity]): """The state of a link between two databases.""" - def __init__(self, entities: Iterable[Entity], events: Tuple[events.LinkStateChanged, ...] = tuple()) -> None: + def __init__(self, entities: Iterable[Entity], events: Iterable[events.LinkStateChanged] | None = None) -> None: """Initialize the link.""" self._entities = set(entities) - self._events = events + self._events = list(events) if events is not None else [] @property def identifiers(self) -> frozenset[Identifier]: @@ -104,7 +104,7 @@ def identifiers(self) -> frozenset[Identifier]: @property def events(self) -> Tuple[events.LinkStateChanged, ...]: """Return the events that happened to the link.""" - return self._events + return tuple(self._events) def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" From 176ddc8e7177335e12874d79de3b144fd01542ce Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:48:45 +0200 Subject: [PATCH 19/59] Add list idle entities domain service --- link/adapters/controller.py | 6 +- link/adapters/present.py | 5 +- link/domain/link.py | 11 +- link/service/services.py | 26 ++--- .../integration/test_datajoint_persistence.py | 102 +++++------------- tests/integration/test_services.py | 6 +- 6 files changed, 49 insertions(+), 107 deletions(-) diff --git a/link/adapters/controller.py b/link/adapters/controller.py index 4d2289e6..11872ecc 100644 --- a/link/adapters/controller.py +++ b/link/adapters/controller.py @@ -3,9 +3,9 @@ from typing import Callable, Iterable, Mapping +from link.domain import commands from link.service.services import ( DeleteRequest, - ListIdleEntitiesRequest, PullRequest, Request, Services, @@ -20,7 +20,7 @@ class DJController: def __init__( self, - handlers: Mapping[Services, Callable[[Request], None]], + handlers: Mapping[Services, Callable[[Request | commands.Command], None]], translator: IdentificationTranslator, ) -> None: """Initialize the translator.""" @@ -37,4 +37,4 @@ def delete(self, primary_keys: Iterable[PrimaryKey]) -> None: def list_idle_entities(self) -> None: """Execute the use-case that lists idle entities.""" - self.__handlers[Services.LIST_IDLE_ENTITIES](ListIdleEntitiesRequest()) + self.__handlers[Services.LIST_IDLE_ENTITIES](commands.ListIdleEntities()) diff --git a/link/adapters/present.py b/link/adapters/present.py index 729b1950..f308e696 100644 --- a/link/adapters/present.py +++ b/link/adapters/present.py @@ -5,7 +5,6 @@ from typing import Callable, Iterable from link.domain import events -from link.service.services import ListIdleEntitiesResponse from .custom_types import PrimaryKey from .identification import IdentificationTranslator @@ -96,10 +95,10 @@ def present_operation_response(response: events.LinkStateChanged) -> None: def create_idle_entities_updater( translator: IdentificationTranslator, update: Callable[[Iterable[PrimaryKey]], None] -) -> Callable[[ListIdleEntitiesResponse], None]: +) -> Callable[[events.IdleEntitiesListed], None]: """Create a callable that when called updates the list of idle entities.""" - def update_idle_entities(response: ListIdleEntitiesResponse) -> None: + def update_idle_entities(response: events.IdleEntitiesListed) -> None: update(translator.to_primary_key(identifier) for identifier in response.identifiers) return update_idle_entities diff --git a/link/domain/link.py b/link/domain/link.py index 4b3a9aa2..9153e86c 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -9,6 +9,7 @@ STATE_MAP, Components, Entity, + Idle, Operations, PersistentState, Processes, @@ -91,7 +92,7 @@ def assign_to_component(component: Components) -> set[Entity]: class Link(Set[Entity]): """The state of a link between two databases.""" - def __init__(self, entities: Iterable[Entity], events: Iterable[events.LinkStateChanged] | None = None) -> None: + def __init__(self, entities: Iterable[Entity], events: Iterable[events.Event] | None = None) -> None: """Initialize the link.""" self._entities = set(entities) self._events = list(events) if events is not None else [] @@ -102,7 +103,7 @@ def identifiers(self) -> frozenset[Identifier]: return frozenset(entity.identifier for entity in self) @property - def events(self) -> Tuple[events.LinkStateChanged, ...]: + def events(self) -> Tuple[events.Event, ...]: """Return the events that happened to the link.""" return tuple(self._events) @@ -131,6 +132,12 @@ def create_operation_result( ) return Link(changed | unchanged, operation_results) + def list_idle_entities(self) -> None: + """List the identifiers of all idle entities in the link.""" + self._events.append( + events.IdleEntitiesListed(frozenset(entity.identifier for entity in self._entities if entity.state is Idle)) + ) + def __contains__(self, entity: object) -> bool: """Check if the link contains the given entity.""" return entity in self._entities diff --git a/link/service/services.py b/link/service/services.py index 42f2ea3a..ea2c023c 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -114,6 +114,7 @@ def start_pull_process( with uow: result = uow.link.apply(Operations.START_PULL, requested=command.requested).events[0] uow.commit() + assert isinstance(result, events.LinkStateChanged) output_port(result) @@ -127,6 +128,7 @@ def start_delete_process( with uow: result = uow.link.apply(Operations.START_DELETE, requested=command.requested).events[0] uow.commit() + assert isinstance(result, events.LinkStateChanged) output_port(result) @@ -137,32 +139,22 @@ def process( with uow: result = uow.link.apply(Operations.PROCESS, requested=command.requested).events[0] uow.commit() + assert isinstance(result, events.LinkStateChanged) output_port(result) -@dataclass(frozen=True) -class ListIdleEntitiesRequest(Request): - """Request model for the use-case that lists idle entities.""" - - -@dataclass(frozen=True) -class ListIdleEntitiesResponse(Response): - """Response model for the use-case that lists idle entities.""" - - identifiers: frozenset[Identifier] - - def list_idle_entities( - request: ListIdleEntitiesRequest, + command: commands.ListIdleEntities, *, uow: UnitOfWork, - output_port: Callable[[ListIdleEntitiesResponse], None], + output_port: Callable[[events.IdleEntitiesListed], None], ) -> None: """List all idle entities.""" with uow: - output_port( - ListIdleEntitiesResponse(frozenset(entity.identifier for entity in uow.link if entity.state is states.Idle)) - ) + uow.link.list_idle_entities() + event = uow.link.events[-1] + assert isinstance(event, events.IdleEntitiesListed) + output_port(event) class Services(Enum): diff --git a/tests/integration/test_datajoint_persistence.py b/tests/integration/test_datajoint_persistence.py index 89445517..f8de5f06 100644 --- a/tests/integration/test_datajoint_persistence.py +++ b/tests/integration/test_datajoint_persistence.py @@ -17,6 +17,7 @@ from link.adapters import PrimaryKey from link.adapters.gateway import DJLinkGateway from link.adapters.identification import IdentificationTranslator +from link.domain import events from link.domain.link import create_link from link.domain.state import Components, Operations, Processes from link.infrastructure.facade import DJLinkFacade, Table @@ -340,6 +341,16 @@ def test_link_creation() -> None: ) +def apply_update(gateway: DJLinkGateway, operation: Operations, requested: Iterable[PrimaryKey]) -> None: + event = ( + gateway.create_link() + .apply(operation, requested={gateway.translator.to_identifier(key) for key in requested}) + .events[0] + ) + assert isinstance(event, events.LinkStateChanged) + gateway.apply(event.updates) + + def test_add_to_local_command() -> None: tables = create_tables( "link", @@ -370,12 +381,7 @@ def test_add_to_local_command() -> None: ), ) - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) assert has_state( tables, @@ -413,12 +419,7 @@ def test_add_to_local_command_with_error() -> None: tables["local"].children(as_objects=True)[0].error_on_insert = RuntimeError try: - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) except RuntimeError: pass @@ -435,12 +436,7 @@ def test_add_to_local_command_with_external_file(tmpdir: Path) -> None: tables["source"].insert([{"a": 0, "external": insert_filepath}]) os.remove(insert_filepath) tables["outbound"].insert([{"a": 0, "process": "PULL", "is_flagged": "FALSE", "is_deprecated": "FALSE"}]) - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) fetch_filepath = Path(tables["local"].fetch(as_dict=True, download_path=str(tmpdir))[0]["external"]) with fetch_filepath.open(mode="rb") as file: assert file.read() == data @@ -459,12 +455,7 @@ def test_remove_from_local_command() -> None: ) with as_stdin(StringIO("y")): - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) assert has_state( tables, @@ -480,12 +471,7 @@ def test_start_pull_process() -> None: "link", primary={"a"}, non_primary={"b"}, initial=State(source=TableState([{"a": 0, "b": 1}])) ) - gateway.apply( - gateway.create_link() - .apply(Operations.START_PULL, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.START_PULL, [{"a": 0}]) assert has_state( tables, @@ -510,12 +496,7 @@ def initial_state() -> State: def test_state_after_command(initial_state: State) -> None: tables, gateway = initialize("link", primary={"a"}, non_primary={"b"}, initial=initial_state) - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) assert has_state( tables, @@ -532,12 +513,7 @@ def test_rollback_on_error(initial_state: State) -> None: tables["outbound"].error_on_insert = RuntimeError try: - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) except RuntimeError: pass @@ -558,12 +534,7 @@ def initial_state() -> State: def test_state_after_command(initial_state: State) -> None: tables, gateway = initialize("link", primary={"a"}, non_primary={"b"}, initial=initial_state) - gateway.apply( - gateway.create_link() - .apply(Operations.START_DELETE, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.START_DELETE, [{"a": 0}]) assert has_state( tables, @@ -580,12 +551,7 @@ def test_rollback_on_error(initial_state: State) -> None: tables["outbound"].error_on_insert = RuntimeError try: - gateway.apply( - gateway.create_link() - .apply(Operations.START_DELETE, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.START_DELETE, [{"a": 0}]) except RuntimeError: pass @@ -603,12 +569,7 @@ def test_finish_delete_process_command() -> None: ), ) - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) assert has_state(tables, State(source=TableState([{"a": 0, "b": 1}]))) @@ -626,12 +587,7 @@ def initial_state() -> State: def test_state_after_command(initial_state: State) -> None: tables, gateway = initialize("link", primary={"a"}, non_primary={"b"}, initial=initial_state) - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) assert has_state( tables, @@ -647,12 +603,7 @@ def test_rollback_on_error(initial_state: State) -> None: tables["outbound"].error_on_insert = RuntimeError try: - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested={gateway.translator.to_identifier({"a": 0})}) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}]) except RuntimeError: pass @@ -677,12 +628,7 @@ def test_applying_multiple_commands() -> None: ) with as_stdin(StringIO("y")): - gateway.apply( - gateway.create_link() - .apply(Operations.PROCESS, requested=gateway.translator.to_identifiers([{"a": 0}, {"a": 1}])) - .events[0] - .updates - ) + apply_update(gateway, Operations.PROCESS, [{"a": 0}, {"a": 1}]) assert has_state( tables, diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 614b0520..f3026a31 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -12,8 +12,6 @@ from link.service.services import ( DeleteRequest, DeleteResponse, - ListIdleEntitiesRequest, - ListIdleEntitiesResponse, ProcessToCompletionRequest, PullRequest, PullResponse, @@ -285,6 +283,6 @@ def test_correct_response_model_gets_passed_to_list_idle_entities_output_port() create_assignments({Components.SOURCE: {"1", "2"}, Components.OUTBOUND: {"2"}, Components.LOCAL: {"2"}}) ) ) - output_port = FakeOutputPort[ListIdleEntitiesResponse]() - list_idle_entities(ListIdleEntitiesRequest(), uow=uow, output_port=output_port) + output_port = FakeOutputPort[events.IdleEntitiesListed]() + list_idle_entities(commands.ListIdleEntities(), uow=uow, output_port=output_port) assert set(output_port.response.identifiers) == create_identifiers("1") From 0d21115f16833e0e0e6d1b2101ddc6deb24c41f7 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:54:13 +0200 Subject: [PATCH 20/59] Use fully process link command instead of request --- link/domain/commands.py | 7 +++++++ link/service/services.py | 25 +++++++++---------------- tests/integration/test_services.py | 3 +-- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/link/domain/commands.py b/link/domain/commands.py index d495ebd7..c72bb46c 100644 --- a/link/domain/commands.py +++ b/link/domain/commands.py @@ -11,6 +11,13 @@ class Command: """Base class for all commands.""" +@dataclass(frozen=True) +class FullyProcessLink(Command): + """Process the requested entities in the link until their processes are completed.""" + + requested: frozenset[Identifier] + + @dataclass(frozen=True) class ProcessLink(Command): """Process the requested entities in the link.""" diff --git a/link/service/services.py b/link/service/services.py index ea2c023c..e8eadee9 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -38,15 +38,15 @@ class PullResponse(Response): def pull( request: PullRequest, *, - process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], + process_to_completion_service: Callable[[commands.FullyProcessLink], ProcessToCompletionResponse], start_pull_process_service: Callable[[commands.StartPullProcess], events.LinkStateChanged], output_port: Callable[[PullResponse], None], ) -> None: """Pull entities across the link.""" - process_to_completion_service(ProcessToCompletionRequest(request.requested)) + process_to_completion_service(commands.FullyProcessLink(request.requested)) response = start_pull_process_service(commands.StartPullProcess(request.requested)) errors = (error for error in response.errors if error.state is states.Deprecated) - process_to_completion_service(ProcessToCompletionRequest(request.requested)) + process_to_completion_service(commands.FullyProcessLink(request.requested)) output_port(PullResponse(request.requested, errors=frozenset(errors))) @@ -67,24 +67,17 @@ class DeleteResponse(Response): def delete( request: DeleteRequest, *, - process_to_completion_service: Callable[[ProcessToCompletionRequest], ProcessToCompletionResponse], + process_to_completion_service: Callable[[commands.FullyProcessLink], ProcessToCompletionResponse], start_delete_process_service: Callable[[commands.StartDeleteProcess], events.LinkStateChanged], output_port: Callable[[DeleteResponse], None], ) -> None: """Delete pulled entities.""" - process_to_completion_service(ProcessToCompletionRequest(request.requested)) + process_to_completion_service(commands.FullyProcessLink(request.requested)) start_delete_process_service(commands.StartDeleteProcess(request.requested)) - process_to_completion_service(ProcessToCompletionRequest(request.requested)) + process_to_completion_service(commands.FullyProcessLink(request.requested)) output_port(DeleteResponse(request.requested)) -@dataclass(frozen=True) -class ProcessToCompletionRequest(Request): - """Request model for the process to completion use-case.""" - - requested: frozenset[Identifier] - - @dataclass(frozen=True) class ProcessToCompletionResponse(Response): """Response model for the process to completion use-case.""" @@ -93,15 +86,15 @@ class ProcessToCompletionResponse(Response): def process_to_completion( - request: ProcessToCompletionRequest, + command: commands.FullyProcessLink, *, process_service: Callable[[commands.ProcessLink], events.LinkStateChanged], output_port: Callable[[ProcessToCompletionResponse], None], ) -> None: """Process entities until their processes are complete.""" - while process_service(commands.ProcessLink(request.requested)).updates: + while process_service(commands.ProcessLink(command.requested)).updates: pass - output_port(ProcessToCompletionResponse(request.requested)) + output_port(ProcessToCompletionResponse(command.requested)) def start_pull_process( diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index f3026a31..e520d9a3 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -12,7 +12,6 @@ from link.service.services import ( DeleteRequest, DeleteResponse, - ProcessToCompletionRequest, PullRequest, PullResponse, Response, @@ -83,7 +82,7 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: ) -def create_process_to_completion_service(uow: UnitOfWork) -> Callable[[ProcessToCompletionRequest], None]: +def create_process_to_completion_service(uow: UnitOfWork) -> Callable[[commands.FullyProcessLink], None]: process_service = partial(make_responsive(partial(process, uow=uow)), output_port=lambda x: None) return partial( make_responsive( From 417d8947168047940f9a518c7d69a6ec4722dfee Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:37:54 +0200 Subject: [PATCH 21/59] Add pull method to link --- link/domain/link.py | 16 ++++++++++++++++ link/infrastructure/link.py | 11 ++--------- link/service/services.py | 20 ++++++++------------ tests/integration/test_services.py | 11 +---------- 4 files changed, 27 insertions(+), 31 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 9153e86c..92074946 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -132,6 +132,22 @@ def create_operation_result( ) return Link(changed | unchanged, operation_results) + def pull(self, requested: Iterable[Identifier]) -> Link: + """Pull the requested entities.""" + + def complete_all_processes(link: Link) -> Link: + while True: + link = link.apply(Operations.PROCESS, requested=requested) + latest_event = link.events[-1] + assert hasattr(latest_event, "updates") + if not latest_event.updates: + break + return link + + link = complete_all_processes(self) + link = link.apply(Operations.START_PULL, requested=requested) + return complete_all_processes(link) + def list_idle_entities(self) -> None: """List the identifiers of all idle entities in the link.""" self._events.append( diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index 53b727a1..72cf05b4 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -60,9 +60,7 @@ def inner(obj: type) -> Any: idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) operation_presenter = create_operation_response_presenter(translator, create_operation_logger()) process_service = partial(make_responsive(partial(process, uow=uow)), output_port=operation_presenter) - start_pull_process_service = partial( - make_responsive(partial(start_pull_process, uow=uow)), output_port=operation_presenter - ) + partial(make_responsive(partial(start_pull_process, uow=uow)), output_port=operation_presenter) start_delete_process_service = partial( make_responsive(partial(start_delete_process, uow=uow)), output_port=operation_presenter ) @@ -70,12 +68,7 @@ def inner(obj: type) -> Any: make_responsive(partial(process_to_completion, process_service=process_service)), output_port=lambda x: None ) handlers = { - Services.PULL: partial( - pull, - process_to_completion_service=process_to_completion_service, - start_pull_process_service=start_pull_process_service, - output_port=lambda x: None, - ), + Services.PULL: partial(pull, uow=uow, output_port=lambda x: None), Services.DELETE: partial( delete, process_to_completion_service=process_to_completion_service, diff --git a/link/service/services.py b/link/service/services.py index e8eadee9..15bfac1b 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -35,19 +35,15 @@ class PullResponse(Response): errors: frozenset[events.InvalidOperationRequested] -def pull( - request: PullRequest, - *, - process_to_completion_service: Callable[[commands.FullyProcessLink], ProcessToCompletionResponse], - start_pull_process_service: Callable[[commands.StartPullProcess], events.LinkStateChanged], - output_port: Callable[[PullResponse], None], -) -> None: +def pull(request: PullRequest, *, uow: UnitOfWork, output_port: Callable[[PullResponse], None]) -> None: """Pull entities across the link.""" - process_to_completion_service(commands.FullyProcessLink(request.requested)) - response = start_pull_process_service(commands.StartPullProcess(request.requested)) - errors = (error for error in response.errors if error.state is states.Deprecated) - process_to_completion_service(commands.FullyProcessLink(request.requested)) - output_port(PullResponse(request.requested, errors=frozenset(errors))) + with uow: + link = uow.link.pull(request.requested) + uow.commit() + state_changed_events = (event for event in link.events if isinstance(event, events.LinkStateChanged)) + start_pull_event = next(event for event in state_changed_events if event.operation is Operations.START_PULL) + errors = (error for error in start_pull_event.errors if error.state is states.Deprecated) + output_port(PullResponse(request.requested, frozenset(errors))) @dataclass(frozen=True) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index e520d9a3..bebb2a9a 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -21,7 +21,6 @@ process_to_completion, pull, start_delete_process, - start_pull_process, ) from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -96,15 +95,7 @@ def create_process_to_completion_service(uow: UnitOfWork) -> Callable[[commands. def create_pull_service(uow: UnitOfWork) -> Service[PullRequest, PullResponse]: - process_to_completion_service = create_process_to_completion_service(uow) - start_pull_process_service = partial( - make_responsive(partial(start_pull_process, uow=uow)), output_port=lambda x: None - ) - return partial( - pull, - process_to_completion_service=process_to_completion_service, - start_pull_process_service=start_pull_process_service, - ) + return partial(pull, uow=uow) def create_delete_service(uow: UnitOfWork) -> Service[DeleteRequest, DeleteResponse]: From c8d74d9b9d95971d6942faaa4e41be318c853f41 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:49:34 +0200 Subject: [PATCH 22/59] Add delete method to link --- link/domain/link.py | 30 +++++++++++++++---------- link/infrastructure/link.py | 35 +++--------------------------- link/service/services.py | 14 ++++-------- tests/integration/test_services.py | 11 +--------- 4 files changed, 26 insertions(+), 64 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 92074946..f4164b5a 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -134,19 +134,15 @@ def create_operation_result( def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" - - def complete_all_processes(link: Link) -> Link: - while True: - link = link.apply(Operations.PROCESS, requested=requested) - latest_event = link.events[-1] - assert hasattr(latest_event, "updates") - if not latest_event.updates: - break - return link - - link = complete_all_processes(self) + link = _complete_all_processes(self, requested) link = link.apply(Operations.START_PULL, requested=requested) - return complete_all_processes(link) + return _complete_all_processes(link, requested) + + def delete(self, requested: Iterable[Identifier]) -> Link: + """Delete the requested entities.""" + link = _complete_all_processes(self, requested) + link = link.apply(Operations.START_DELETE, requested=requested) + return _complete_all_processes(link, requested) def list_idle_entities(self) -> None: """List the identifiers of all idle entities in the link.""" @@ -165,3 +161,13 @@ def __iter__(self) -> Iterator[Entity]: def __len__(self) -> int: """Return the number of entities in the link.""" return len(self._entities) + + +def _complete_all_processes(link: Link, requested: Iterable[Identifier]) -> Link: + while True: + link = link.apply(Operations.PROCESS, requested=requested) + latest_event = link.events[-1] + assert hasattr(latest_event, "updates") + if not latest_event.updates: + break + return link diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index 72cf05b4..af22682b 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -9,26 +9,12 @@ from link.adapters.custom_types import PrimaryKey from link.adapters.gateway import DJLinkGateway from link.adapters.identification import IdentificationTranslator -from link.adapters.present import ( - create_idle_entities_updater, - create_operation_response_presenter, -) -from link.service.io import make_responsive -from link.service.services import ( - Services, - delete, - list_idle_entities, - process, - process_to_completion, - pull, - start_delete_process, - start_pull_process, -) +from link.adapters.present import create_idle_entities_updater +from link.service.services import Services, delete, list_idle_entities, pull from link.service.uow import UnitOfWork from . import DJConfiguration, create_tables from .facade import DJLinkFacade -from .log import create_operation_logger from .mixin import create_local_endpoint from .sequence import IterationCallbackList, create_content_replacer @@ -58,24 +44,9 @@ def inner(obj: type) -> Any: uow = UnitOfWork(gateway) source_restriction: IterationCallbackList[PrimaryKey] = IterationCallbackList() idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) - operation_presenter = create_operation_response_presenter(translator, create_operation_logger()) - process_service = partial(make_responsive(partial(process, uow=uow)), output_port=operation_presenter) - partial(make_responsive(partial(start_pull_process, uow=uow)), output_port=operation_presenter) - start_delete_process_service = partial( - make_responsive(partial(start_delete_process, uow=uow)), output_port=operation_presenter - ) - process_to_completion_service = partial( - make_responsive(partial(process_to_completion, process_service=process_service)), output_port=lambda x: None - ) handlers = { Services.PULL: partial(pull, uow=uow, output_port=lambda x: None), - Services.DELETE: partial( - delete, - process_to_completion_service=process_to_completion_service, - start_delete_process_service=start_delete_process_service, - output_port=lambda x: None, - ), - Services.PROCESS: partial(process, uow=uow, output_port=operation_presenter), + Services.DELETE: partial(delete, uow=uow, output_port=lambda x: None), Services.LIST_IDLE_ENTITIES: partial(list_idle_entities, uow=uow, output_port=idle_entities_updater), } controller = DJController(handlers, translator) diff --git a/link/service/services.py b/link/service/services.py index 15bfac1b..f8a3a1bc 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -60,17 +60,11 @@ class DeleteResponse(Response): requested: frozenset[Identifier] -def delete( - request: DeleteRequest, - *, - process_to_completion_service: Callable[[commands.FullyProcessLink], ProcessToCompletionResponse], - start_delete_process_service: Callable[[commands.StartDeleteProcess], events.LinkStateChanged], - output_port: Callable[[DeleteResponse], None], -) -> None: +def delete(request: DeleteRequest, *, uow: UnitOfWork, output_port: Callable[[DeleteResponse], None]) -> None: """Delete pulled entities.""" - process_to_completion_service(commands.FullyProcessLink(request.requested)) - start_delete_process_service(commands.StartDeleteProcess(request.requested)) - process_to_completion_service(commands.FullyProcessLink(request.requested)) + with uow: + uow.link.delete(request.requested) + uow.commit() output_port(DeleteResponse(request.requested)) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index bebb2a9a..ebb47544 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -20,7 +20,6 @@ process, process_to_completion, pull, - start_delete_process, ) from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -99,15 +98,7 @@ def create_pull_service(uow: UnitOfWork) -> Service[PullRequest, PullResponse]: def create_delete_service(uow: UnitOfWork) -> Service[DeleteRequest, DeleteResponse]: - process_to_completion_service = create_process_to_completion_service(uow) - start_delete_process_service = partial( - make_responsive(partial(start_delete_process, uow=uow)), output_port=lambda x: None - ) - return partial( - delete, - process_to_completion_service=process_to_completion_service, - start_delete_process_service=start_delete_process_service, - ) + return partial(delete, uow=uow) class EntityConfig(TypedDict): From 20f8462b4f87ff994311f7addb1a7fc784b3d574 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:20:26 +0200 Subject: [PATCH 23/59] Add __eq__ to link --- link/domain/link.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index f4164b5a..8b2dc8c2 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -13,6 +13,7 @@ Operations, PersistentState, Processes, + State, ) @@ -162,12 +163,19 @@ def __len__(self) -> int: """Return the number of entities in the link.""" return len(self._entities) + def __eq__(self, other: object) -> bool: + """Return True if both links have entities with the same identifiers and states.""" + if not isinstance(other, type(self)): + raise NotImplementedError -def _complete_all_processes(link: Link, requested: Iterable[Identifier]) -> Link: - while True: - link = link.apply(Operations.PROCESS, requested=requested) - latest_event = link.events[-1] - assert hasattr(latest_event, "updates") - if not latest_event.updates: - break - return link + def create_identifier_state_pairs(link: Link) -> set[tuple[Identifier, type[State]]]: + return {(entity.identifier, entity.state) for entity in link} + + return create_identifier_state_pairs(self) == create_identifier_state_pairs(other) + + +def _complete_all_processes(current: Link, requested: Iterable[Identifier]) -> Link: + new = current.apply(Operations.PROCESS, requested=requested) + if new == current: + return new + return _complete_all_processes(new, requested) From 2a6ca9840be6d333e04d378d2ff7cd26af76da5c Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:33:27 +0200 Subject: [PATCH 24/59] Remove unused services --- link/domain/commands.py | 30 ---------------- link/service/io.py | 47 +----------------------- link/service/services.py | 58 ------------------------------ tests/integration/test_services.py | 52 +-------------------------- 4 files changed, 2 insertions(+), 185 deletions(-) diff --git a/link/domain/commands.py b/link/domain/commands.py index c72bb46c..c952f39b 100644 --- a/link/domain/commands.py +++ b/link/domain/commands.py @@ -3,42 +3,12 @@ from dataclasses import dataclass -from .custom_types import Identifier - @dataclass(frozen=True) class Command: """Base class for all commands.""" -@dataclass(frozen=True) -class FullyProcessLink(Command): - """Process the requested entities in the link until their processes are completed.""" - - requested: frozenset[Identifier] - - -@dataclass(frozen=True) -class ProcessLink(Command): - """Process the requested entities in the link.""" - - requested: frozenset[Identifier] - - -@dataclass(frozen=True) -class StartPullProcess(Command): - """Start the pull process for the requested entities.""" - - requested: frozenset[Identifier] - - -@dataclass(frozen=True) -class StartDeleteProcess(Command): - """Start the delete process for the requested entities.""" - - requested: frozenset[Identifier] - - @dataclass(frozen=True) class ListIdleEntities(Command): """Start the delete process for the requested entities.""" diff --git a/link/service/io.py b/link/service/io.py index cef47d47..b78f3796 100644 --- a/link/service/io.py +++ b/link/service/io.py @@ -1,33 +1,10 @@ """Contains logic related to input/output handling to/from services.""" from __future__ import annotations -from functools import partial -from typing import Any, Callable, Generic, Protocol, TypeVar +from typing import Any, Callable, Protocol, TypeVar from .services import Request, Response -_Response = TypeVar("_Response", bound=Response) - - -class ResponseRelay(Generic[_Response]): - """A relay that makes the response of one service available to another.""" - - def __init__(self) -> None: - """Initialize the relay.""" - self._response: _Response | None = None - - def get_response(self) -> _Response: - """Return the response of the relayed service.""" - assert self._response is not None - return self._response - - def __call__(self, response: _Response) -> None: - """Store the response of the relayed service.""" - self._response = response - - -_Request = TypeVar("_Request", bound=Request) - _Response_co = TypeVar("_Response_co", bound=Response, covariant=True) _Request_contra = TypeVar("_Request_contra", bound=Request, contravariant=True) @@ -38,25 +15,3 @@ class Service(Protocol[_Request_contra, _Response_co]): def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response_co], None], **kwargs: Any) -> None: """Execute the service.""" - - -class ReturningService(Protocol[_Request_contra, _Response_co]): - """Protocol for services that return their response.""" - - def __call__( - self, request: _Request_contra, *, output_port: Callable[[_Response_co], None], **kwargs: Any - ) -> _Response_co: - """Execute the service.""" - - -def make_responsive(service: Service[_Request, _Response]) -> ReturningService[_Request, _Response]: - """Create a version of the service that returns its response in addition to sending it to the output port.""" - relay: ResponseRelay[_Response] = ResponseRelay() - service = partial(service, output_port=relay) - - def returning_service(request: _Request, *, output_port: Callable[[_Response], None], **kwargs: Any) -> _Response: - service(request, **kwargs) - output_port(relay.get_response()) - return relay.get_response() - - return returning_service diff --git a/link/service/services.py b/link/service/services.py index f8a3a1bc..e39b2832 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -68,64 +68,6 @@ def delete(request: DeleteRequest, *, uow: UnitOfWork, output_port: Callable[[De output_port(DeleteResponse(request.requested)) -@dataclass(frozen=True) -class ProcessToCompletionResponse(Response): - """Response model for the process to completion use-case.""" - - requested: frozenset[Identifier] - - -def process_to_completion( - command: commands.FullyProcessLink, - *, - process_service: Callable[[commands.ProcessLink], events.LinkStateChanged], - output_port: Callable[[ProcessToCompletionResponse], None], -) -> None: - """Process entities until their processes are complete.""" - while process_service(commands.ProcessLink(command.requested)).updates: - pass - output_port(ProcessToCompletionResponse(command.requested)) - - -def start_pull_process( - command: commands.StartPullProcess, - *, - uow: UnitOfWork, - output_port: Callable[[events.LinkStateChanged], None], -) -> None: - """Start the pull process for the requested entities.""" - with uow: - result = uow.link.apply(Operations.START_PULL, requested=command.requested).events[0] - uow.commit() - assert isinstance(result, events.LinkStateChanged) - output_port(result) - - -def start_delete_process( - command: commands.StartDeleteProcess, - *, - uow: UnitOfWork, - output_port: Callable[[events.LinkStateChanged], None], -) -> None: - """Start the delete process for the requested entities.""" - with uow: - result = uow.link.apply(Operations.START_DELETE, requested=command.requested).events[0] - uow.commit() - assert isinstance(result, events.LinkStateChanged) - output_port(result) - - -def process( - command: commands.ProcessLink, *, uow: UnitOfWork, output_port: Callable[[events.LinkStateChanged], None] -) -> None: - """Process entities.""" - with uow: - result = uow.link.apply(Operations.PROCESS, requested=command.requested).events[0] - uow.commit() - assert isinstance(result, events.LinkStateChanged) - output_port(result) - - def list_idle_entities( command: commands.ListIdleEntities, *, diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index ebb47544..69b26074 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Callable from functools import partial from typing import Generic, TypedDict, TypeVar, Union @@ -8,7 +7,7 @@ from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states -from link.service.io import Service, make_responsive +from link.service.io import Service from link.service.services import ( DeleteRequest, DeleteResponse, @@ -17,8 +16,6 @@ Response, delete, list_idle_entities, - process, - process_to_completion, pull, ) from link.service.uow import UnitOfWork @@ -80,19 +77,6 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: ) -def create_process_to_completion_service(uow: UnitOfWork) -> Callable[[commands.FullyProcessLink], None]: - process_service = partial(make_responsive(partial(process, uow=uow)), output_port=lambda x: None) - return partial( - make_responsive( - partial( - process_to_completion, - process_service=process_service, - ), - ), - output_port=lambda x: None, - ) - - def create_pull_service(uow: UnitOfWork) -> Service[PullRequest, PullResponse]: return partial(pull, uow=uow) @@ -224,40 +208,6 @@ def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityCon assert output_port.response == PullResponse(requested=frozenset(create_identifiers("1")), errors=frozenset(errors)) -def test_entity_undergoing_process_gets_processed() -> None: - uow = UnitOfWork( - FakeLinkGateway( - create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}}), - processes={Processes.PULL: create_identifiers("1")}, - ) - ) - process( - commands.ProcessLink(frozenset(create_identifiers("1"))), - uow=uow, - output_port=FakeOutputPort[events.LinkStateChanged](), - ) - with uow: - entity = next(entity for entity in uow.link if entity.identifier == create_identifier("1")) - assert entity.state is states.Received - - -def test_correct_response_model_gets_passed_to_process_output_port() -> None: - uow = UnitOfWork( - FakeLinkGateway( - create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}}), - processes={Processes.PULL: create_identifiers("1")}, - ) - ) - output_port = FakeOutputPort[events.LinkStateChanged]() - process( - commands.ProcessLink(frozenset(create_identifiers("1"))), - uow=uow, - output_port=output_port, - ) - assert output_port.response.requested == create_identifiers("1") - assert output_port.response.operation is Operations.PROCESS - - def test_correct_response_model_gets_passed_to_list_idle_entities_output_port() -> None: uow = UnitOfWork( FakeLinkGateway( From 4ba2411c6f1a0f42faad6fb47d0c4662b9ca0973 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:42:40 +0200 Subject: [PATCH 25/59] Merge io and service tests modules --- link/service/io.py | 17 ----------------- tests/integration/test_services.py | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 19 deletions(-) delete mode 100644 link/service/io.py diff --git a/link/service/io.py b/link/service/io.py deleted file mode 100644 index b78f3796..00000000 --- a/link/service/io.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Contains logic related to input/output handling to/from services.""" -from __future__ import annotations - -from typing import Any, Callable, Protocol, TypeVar - -from .services import Request, Response - -_Response_co = TypeVar("_Response_co", bound=Response, covariant=True) - -_Request_contra = TypeVar("_Request_contra", bound=Request, contravariant=True) - - -class Service(Protocol[_Request_contra, _Response_co]): - """Protocol for services.""" - - def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response_co], None], **kwargs: Any) -> None: - """Execute the service.""" diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 69b26074..617deeef 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -1,18 +1,18 @@ from __future__ import annotations from functools import partial -from typing import Generic, TypedDict, TypeVar, Union +from typing import Any, Callable, Generic, Protocol, TypedDict, TypeVar, Union import pytest from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states -from link.service.io import Service from link.service.services import ( DeleteRequest, DeleteResponse, PullRequest, PullResponse, + Request, Response, delete, list_idle_entities, @@ -77,6 +77,18 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: ) +_Response_co = TypeVar("_Response_co", bound=Response, covariant=True) + +_Request_contra = TypeVar("_Request_contra", bound=Request, contravariant=True) + + +class Service(Protocol[_Request_contra, _Response_co]): + """Protocol for services.""" + + def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response_co], None], **kwargs: Any) -> None: + """Execute the service.""" + + def create_pull_service(uow: UnitOfWork) -> Service[PullRequest, PullResponse]: return partial(pull, uow=uow) From 044bc47ec5cab96e5786e80b31e5d2ebe0a00392 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:52:26 +0200 Subject: [PATCH 26/59] Replace requests with commands --- link/adapters/controller.py | 15 ++++++--------- link/domain/commands.py | 16 ++++++++++++++++ link/service/services.py | 30 ++++++------------------------ tests/integration/test_services.py | 17 +++++++---------- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/link/adapters/controller.py b/link/adapters/controller.py index 11872ecc..70047000 100644 --- a/link/adapters/controller.py +++ b/link/adapters/controller.py @@ -4,12 +4,7 @@ from typing import Callable, Iterable, Mapping from link.domain import commands -from link.service.services import ( - DeleteRequest, - PullRequest, - Request, - Services, -) +from link.service.services import Services from .custom_types import PrimaryKey from .identification import IdentificationTranslator @@ -20,7 +15,7 @@ class DJController: def __init__( self, - handlers: Mapping[Services, Callable[[Request | commands.Command], None]], + handlers: Mapping[Services, Callable[[commands.Command], None]], translator: IdentificationTranslator, ) -> None: """Initialize the translator.""" @@ -29,11 +24,13 @@ def __init__( def pull(self, primary_keys: Iterable[PrimaryKey]) -> None: """Execute the pull use-case.""" - self.__handlers[Services.PULL](PullRequest(frozenset(self.__translator.to_identifiers(primary_keys)))) + self.__handlers[Services.PULL](commands.PullEntities(frozenset(self.__translator.to_identifiers(primary_keys)))) def delete(self, primary_keys: Iterable[PrimaryKey]) -> None: """Execute the delete use-case.""" - self.__handlers[Services.DELETE](DeleteRequest(frozenset(self.__translator.to_identifiers(primary_keys)))) + self.__handlers[Services.DELETE]( + commands.DeleteEntities(frozenset(self.__translator.to_identifiers(primary_keys))) + ) def list_idle_entities(self) -> None: """Execute the use-case that lists idle entities.""" diff --git a/link/domain/commands.py b/link/domain/commands.py index c952f39b..d6598def 100644 --- a/link/domain/commands.py +++ b/link/domain/commands.py @@ -3,12 +3,28 @@ from dataclasses import dataclass +from .custom_types import Identifier + @dataclass(frozen=True) class Command: """Base class for all commands.""" +@dataclass(frozen=True) +class PullEntities(Command): + """Pull the requested entities.""" + + requested: frozenset[Identifier] + + +@dataclass(frozen=True) +class DeleteEntities(Command): + """Delete the requested entities.""" + + requested: frozenset[Identifier] + + @dataclass(frozen=True) class ListIdleEntities(Command): """Start the delete process for the requested entities.""" diff --git a/link/service/services.py b/link/service/services.py index e39b2832..ac8c32d7 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -12,21 +12,10 @@ from .uow import UnitOfWork -class Request: - """Base class for all request models.""" - - class Response: """Base class for all response models.""" -@dataclass(frozen=True) -class PullRequest(Request): - """Request model for the pull use-case.""" - - requested: frozenset[Identifier] - - @dataclass(frozen=True) class PullResponse(Response): """Response model for the pull use-case.""" @@ -35,22 +24,15 @@ class PullResponse(Response): errors: frozenset[events.InvalidOperationRequested] -def pull(request: PullRequest, *, uow: UnitOfWork, output_port: Callable[[PullResponse], None]) -> None: +def pull(command: commands.PullEntities, *, uow: UnitOfWork, output_port: Callable[[PullResponse], None]) -> None: """Pull entities across the link.""" with uow: - link = uow.link.pull(request.requested) + link = uow.link.pull(command.requested) uow.commit() state_changed_events = (event for event in link.events if isinstance(event, events.LinkStateChanged)) start_pull_event = next(event for event in state_changed_events if event.operation is Operations.START_PULL) errors = (error for error in start_pull_event.errors if error.state is states.Deprecated) - output_port(PullResponse(request.requested, frozenset(errors))) - - -@dataclass(frozen=True) -class DeleteRequest(Request): - """Request model for the delete use-case.""" - - requested: frozenset[Identifier] + output_port(PullResponse(command.requested, frozenset(errors))) @dataclass(frozen=True) @@ -60,12 +42,12 @@ class DeleteResponse(Response): requested: frozenset[Identifier] -def delete(request: DeleteRequest, *, uow: UnitOfWork, output_port: Callable[[DeleteResponse], None]) -> None: +def delete(command: commands.DeleteEntities, *, uow: UnitOfWork, output_port: Callable[[DeleteResponse], None]) -> None: """Delete pulled entities.""" with uow: - uow.link.delete(request.requested) + uow.link.delete(command.requested) uow.commit() - output_port(DeleteResponse(request.requested)) + output_port(DeleteResponse(command.requested)) def list_idle_entities( diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 617deeef..0705b42b 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -8,11 +8,8 @@ from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states from link.service.services import ( - DeleteRequest, DeleteResponse, - PullRequest, PullResponse, - Request, Response, delete, list_idle_entities, @@ -79,7 +76,7 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: _Response_co = TypeVar("_Response_co", bound=Response, covariant=True) -_Request_contra = TypeVar("_Request_contra", bound=Request, contravariant=True) +_Request_contra = TypeVar("_Request_contra", bound=commands.Command, contravariant=True) class Service(Protocol[_Request_contra, _Response_co]): @@ -89,11 +86,11 @@ def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response """Execute the service.""" -def create_pull_service(uow: UnitOfWork) -> Service[PullRequest, PullResponse]: +def create_pull_service(uow: UnitOfWork) -> Service[commands.PullEntities, PullResponse]: return partial(pull, uow=uow) -def create_delete_service(uow: UnitOfWork) -> Service[DeleteRequest, DeleteResponse]: +def create_delete_service(uow: UnitOfWork) -> Service[commands.DeleteEntities, DeleteResponse]: return partial(delete, uow=uow) @@ -139,7 +136,7 @@ class EntityConfig(TypedDict): def test_deleted_entity_ends_in_correct_state(state: EntityConfig, expected: type[State]) -> None: uow = create_uow(**state) delete_service = create_delete_service(uow) - delete_service(DeleteRequest(frozenset(create_identifiers("1"))), output_port=lambda x: None) + delete_service(commands.DeleteEntities(frozenset(create_identifiers("1"))), output_port=lambda x: None) with uow: assert next(iter(uow.link)).state is expected @@ -152,7 +149,7 @@ def test_correct_response_model_gets_passed_to_delete_output_port() -> None: ) output_port = FakeOutputPort[DeleteResponse]() delete_service = create_delete_service(uow) - delete_service(DeleteRequest(frozenset(create_identifiers("1"))), output_port=output_port) + delete_service(commands.DeleteEntities(frozenset(create_identifiers("1"))), output_port=output_port) assert output_port.response.requested == create_identifiers("1") @@ -177,7 +174,7 @@ def test_pulled_entity_ends_in_correct_state(state: EntityConfig, expected: type uow = create_uow(**state) pull_service = create_pull_service(uow) pull_service( - PullRequest(frozenset(create_identifiers("1"))), + commands.PullEntities(frozenset(create_identifiers("1"))), output_port=lambda x: None, ) with uow: @@ -214,7 +211,7 @@ def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityCon output_port = FakeOutputPort[PullResponse]() pull_service = create_pull_service(gateway) pull_service( - PullRequest(frozenset(create_identifiers("1"))), + commands.PullEntities(frozenset(create_identifiers("1"))), output_port=output_port, ) assert output_port.response == PullResponse(requested=frozenset(create_identifiers("1")), errors=frozenset(errors)) From 987f31c0528ee5cdb64f95a578e05eefa52cb291 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:28:49 +0200 Subject: [PATCH 27/59] Use entities pulled event in place of response --- link/domain/events.py | 8 ++++++++ link/domain/link.py | 9 ++++++++- link/service/services.py | 20 ++++++-------------- tests/integration/test_services.py | 19 +++++++------------ 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 181141d6..413d7ebd 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -59,3 +59,11 @@ class IdleEntitiesListed(Event): """Idle entities in a link have been listed.""" identifiers: frozenset[Identifier] + + +@dataclass(frozen=True) +class EntitiesPulled(Event): + """The requested entities have been pulled.""" + + requested: frozenset[Identifier] + errors: frozenset[InvalidOperationRequested] diff --git a/link/domain/link.py b/link/domain/link.py index 8b2dc8c2..59cbf296 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -14,6 +14,7 @@ PersistentState, Processes, State, + states, ) @@ -135,9 +136,15 @@ def create_operation_result( def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" + requested = frozenset(requested) link = _complete_all_processes(self, requested) link = link.apply(Operations.START_PULL, requested=requested) - return _complete_all_processes(link, requested) + start_pull_event = link.events[-1] + assert isinstance(start_pull_event, events.LinkStateChanged) + link = _complete_all_processes(link, requested) + errors = frozenset(error for error in start_pull_event.errors if error.state is states.Deprecated) + link._events.append(events.EntitiesPulled(requested, errors)) + return link def delete(self, requested: Iterable[Identifier]) -> Link: """Delete the requested entities.""" diff --git a/link/service/services.py b/link/service/services.py index ac8c32d7..eec3285d 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -7,7 +7,6 @@ from link.domain import commands, events from link.domain.custom_types import Identifier -from link.domain.state import Operations, states from .uow import UnitOfWork @@ -16,23 +15,16 @@ class Response: """Base class for all response models.""" -@dataclass(frozen=True) -class PullResponse(Response): - """Response model for the pull use-case.""" - - requested: frozenset[Identifier] - errors: frozenset[events.InvalidOperationRequested] - - -def pull(command: commands.PullEntities, *, uow: UnitOfWork, output_port: Callable[[PullResponse], None]) -> None: +def pull( + command: commands.PullEntities, *, uow: UnitOfWork, output_port: Callable[[events.EntitiesPulled], None] +) -> None: """Pull entities across the link.""" with uow: link = uow.link.pull(command.requested) uow.commit() - state_changed_events = (event for event in link.events if isinstance(event, events.LinkStateChanged)) - start_pull_event = next(event for event in state_changed_events if event.operation is Operations.START_PULL) - errors = (error for error in start_pull_event.errors if error.state is states.Deprecated) - output_port(PullResponse(command.requested, frozenset(errors))) + event = link.events[-1] + assert isinstance(event, events.EntitiesPulled) + output_port(event) @dataclass(frozen=True) diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 0705b42b..e5a150c1 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -7,14 +7,7 @@ from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states -from link.service.services import ( - DeleteResponse, - PullResponse, - Response, - delete, - list_idle_entities, - pull, -) +from link.service.services import DeleteResponse, Response, delete, list_idle_entities, pull from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -74,7 +67,7 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: ) -_Response_co = TypeVar("_Response_co", bound=Response, covariant=True) +_Response_co = TypeVar("_Response_co", bound=Union[Response, events.Event], covariant=True) _Request_contra = TypeVar("_Request_contra", bound=commands.Command, contravariant=True) @@ -86,7 +79,7 @@ def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response """Execute the service.""" -def create_pull_service(uow: UnitOfWork) -> Service[commands.PullEntities, PullResponse]: +def create_pull_service(uow: UnitOfWork) -> Service[commands.PullEntities, events.EntitiesPulled]: return partial(pull, uow=uow) @@ -208,13 +201,15 @@ def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityCon else: errors = set() gateway = create_uow(**state) - output_port = FakeOutputPort[PullResponse]() + output_port = FakeOutputPort[events.EntitiesPulled]() pull_service = create_pull_service(gateway) pull_service( commands.PullEntities(frozenset(create_identifiers("1"))), output_port=output_port, ) - assert output_port.response == PullResponse(requested=frozenset(create_identifiers("1")), errors=frozenset(errors)) + assert output_port.response == events.EntitiesPulled( + requested=frozenset(create_identifiers("1")), errors=frozenset(errors) + ) def test_correct_response_model_gets_passed_to_list_idle_entities_output_port() -> None: From 6c5c5b099f417d46a9e921487a5530010a63ce70 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:38:56 +0200 Subject: [PATCH 28/59] Use entities deleted event in place of response --- link/domain/events.py | 7 +++++++ link/domain/link.py | 5 ++++- link/service/services.py | 23 +++++++---------------- tests/integration/test_services.py | 18 +++++++++--------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 413d7ebd..0fb07b2b 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -67,3 +67,10 @@ class EntitiesPulled(Event): requested: frozenset[Identifier] errors: frozenset[InvalidOperationRequested] + + +@dataclass(frozen=True) +class EntitiesDeleted(Event): + """The requested entities have been deleted.""" + + requested: frozenset[Identifier] diff --git a/link/domain/link.py b/link/domain/link.py index 59cbf296..3a1fdfc0 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -148,9 +148,12 @@ def pull(self, requested: Iterable[Identifier]) -> Link: def delete(self, requested: Iterable[Identifier]) -> Link: """Delete the requested entities.""" + requested = frozenset(requested) link = _complete_all_processes(self, requested) link = link.apply(Operations.START_DELETE, requested=requested) - return _complete_all_processes(link, requested) + link = _complete_all_processes(link, requested) + link._events.append(events.EntitiesDeleted(requested)) + return link def list_idle_entities(self) -> None: """List the identifiers of all idle entities in the link.""" diff --git a/link/service/services.py b/link/service/services.py index eec3285d..d9d329b2 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -2,19 +2,13 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass from enum import Enum, auto from link.domain import commands, events -from link.domain.custom_types import Identifier from .uow import UnitOfWork -class Response: - """Base class for all response models.""" - - def pull( command: commands.PullEntities, *, uow: UnitOfWork, output_port: Callable[[events.EntitiesPulled], None] ) -> None: @@ -27,19 +21,16 @@ def pull( output_port(event) -@dataclass(frozen=True) -class DeleteResponse(Response): - """Response model for the delete use-case.""" - - requested: frozenset[Identifier] - - -def delete(command: commands.DeleteEntities, *, uow: UnitOfWork, output_port: Callable[[DeleteResponse], None]) -> None: +def delete( + command: commands.DeleteEntities, *, uow: UnitOfWork, output_port: Callable[[events.EntitiesDeleted], None] +) -> None: """Delete pulled entities.""" with uow: - uow.link.delete(command.requested) + link = uow.link.delete(command.requested) uow.commit() - output_port(DeleteResponse(command.requested)) + event = link.events[-1] + assert isinstance(event, events.EntitiesDeleted) + output_port(event) def list_idle_entities( diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index e5a150c1..d41c27fc 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -1,19 +1,19 @@ from __future__ import annotations from functools import partial -from typing import Any, Callable, Generic, Protocol, TypedDict, TypeVar, Union +from typing import Any, Callable, Generic, Protocol, TypedDict, TypeVar import pytest from link.domain import commands, events from link.domain.state import Components, Operations, Processes, State, states -from link.service.services import DeleteResponse, Response, delete, list_idle_entities, pull +from link.service.services import delete, list_idle_entities, pull from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifier, create_identifiers from .gateway import FakeLinkGateway -T = TypeVar("T", bound=Union[Response, events.Event]) +T = TypeVar("T", bound=events.Event) class FakeOutputPort(Generic[T]): @@ -67,15 +67,15 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: ) -_Response_co = TypeVar("_Response_co", bound=Union[Response, events.Event], covariant=True) +_Event_co = TypeVar("_Event_co", bound=events.Event, covariant=True) -_Request_contra = TypeVar("_Request_contra", bound=commands.Command, contravariant=True) +_Command_contra = TypeVar("_Command_contra", bound=commands.Command, contravariant=True) -class Service(Protocol[_Request_contra, _Response_co]): +class Service(Protocol[_Command_contra, _Event_co]): """Protocol for services.""" - def __call__(self, request: _Request_contra, *, output_port: Callable[[_Response_co], None], **kwargs: Any) -> None: + def __call__(self, command: _Command_contra, *, output_port: Callable[[_Event_co], None], **kwargs: Any) -> None: """Execute the service.""" @@ -83,7 +83,7 @@ def create_pull_service(uow: UnitOfWork) -> Service[commands.PullEntities, event return partial(pull, uow=uow) -def create_delete_service(uow: UnitOfWork) -> Service[commands.DeleteEntities, DeleteResponse]: +def create_delete_service(uow: UnitOfWork) -> Service[commands.DeleteEntities, events.EntitiesDeleted]: return partial(delete, uow=uow) @@ -140,7 +140,7 @@ def test_correct_response_model_gets_passed_to_delete_output_port() -> None: create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}, Components.LOCAL: {"1"}}) ) ) - output_port = FakeOutputPort[DeleteResponse]() + output_port = FakeOutputPort[events.EntitiesDeleted]() delete_service = create_delete_service(uow) delete_service(commands.DeleteEntities(frozenset(create_identifiers("1"))), output_port=output_port) assert output_port.response.requested == create_identifiers("1") From bd840f6459d3ddb2dd6e31f3d7869922ce693abc Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:07:50 +0200 Subject: [PATCH 29/59] Remove non-existing service from enum --- link/service/services.py | 1 - 1 file changed, 1 deletion(-) diff --git a/link/service/services.py b/link/service/services.py index d9d329b2..591cbaef 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -52,5 +52,4 @@ class Services(Enum): PULL = auto() DELETE = auto() - PROCESS = auto() LIST_IDLE_ENTITIES = auto() From 1424cc4e04454c36ec397d9884b32a89ce31566f Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 26 Oct 2023 17:09:57 +0200 Subject: [PATCH 30/59] Add message bus --- link/service/messagebus.py | 73 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 link/service/messagebus.py diff --git a/link/service/messagebus.py b/link/service/messagebus.py new file mode 100644 index 00000000..b0d9d372 --- /dev/null +++ b/link/service/messagebus.py @@ -0,0 +1,73 @@ +"""Contains the message bus.""" +from __future__ import annotations + +import logging +from collections import deque +from typing import Callable, Iterable, Protocol, TypeVar, Union + +from link.domain.commands import Command +from link.domain.events import Event + +from .uow import UnitOfWork + +Message = Union[Command, Event] + + +logger = logging.getLogger() + +T = TypeVar("T", bound=Command) + + +class CommandHandlers(Protocol): + """A mapping of command types to handlers.""" + + def __getitem__(self, command_type: type[T]) -> Callable[[T], None]: + """Get the appropriate handler for the given command.""" + + +V = TypeVar("V", bound=Event) + + +class EventHandlers(Protocol): + """A mapping of event types to handlers.""" + + def __getitem__(self, event_type: type[V]) -> Iterable[Callable[[V], None]]: + """Get the appropriate handlers for the given event.""" + + +class Messagebus: + """A message bus that dispatches domain messages to their appropriate handlers.""" + + def __init__(self, uow: UnitOfWork, command_handlers: CommandHandlers, event_handlers: EventHandlers) -> None: + """Initialize the bus.""" + self._uow = uow + self._queue: deque[Message] = deque() + self._command_handlers = command_handlers + self._event_handlers = event_handlers + + def handle(self, message: Message) -> None: + """Handle the message.""" + self._queue.append(message) + while self._queue: + message = self._queue.popleft() + if isinstance(message, Command): + self._handle_command(message) + if isinstance(message, Event): + self._handle_event(message) + else: + raise TypeError(f"Unknown message type {type(message)!r}") + + def _handle_command(self, command: Command) -> None: + handler = self._command_handlers[type(command)] + try: + handler(command) + except Exception: + logger.exception(f"Error handling command {command!r} with handler {handler!r}") + raise + + def _handle_event(self, event: Event) -> None: + for handler in self._event_handlers[type(event)]: + try: + handler(event) + except Exception: + logger.exception(f"Error handling event {event!r} with handler {handler!r}") From ee11bd420b6c1f27cfdb384f2d04b53dbc2e781d Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:55:50 +0100 Subject: [PATCH 31/59] Replace object.__setattr__ with setattr --- link/service/uow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/link/service/uow.py b/link/service/uow.py index 38855818..e7ede334 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -36,8 +36,8 @@ def __enter__(self) -> UnitOfWork: def augment_link(link: Link) -> None: original = getattr(link, "apply") augmented = augment_link_apply(link, original) - object.__setattr__(link, "apply", augmented) - object.__setattr__(link, "_is_expired", False) + setattr(link, "apply", augmented) + setattr(link, "_is_expired", False) def augment_link_apply(current: Link, original: SupportsLinkApply) -> SupportsLinkApply: def augmented(operation: Operations, *, requested: Iterable[Identifier]) -> Link: @@ -46,7 +46,7 @@ def augmented(operation: Operations, *, requested: Iterable[Identifier]) -> Link raise RuntimeError("Can not apply operation to expired link") self._link = original(operation, requested=requested) augment_link(self._link) - object.__setattr__(current, "_is_expired", True) + setattr(current, "_is_expired", True) return self._link return augmented @@ -106,7 +106,7 @@ def rollback(self) -> None: """Throw away any not yet persisted updates.""" if self._link is None: raise RuntimeError("Not available outside of context") - object.__setattr__(self._link, "_is_expired", True) + setattr(self._link, "_is_expired", True) for entity in self._link: object.__setattr__(entity, "_is_expired", True) self._seen.clear() From 5e846aee67c607bf540557cf61d98540c6c581f7 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:34:47 +0100 Subject: [PATCH 32/59] Fix TypeError being raised for known message type --- link/service/messagebus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/link/service/messagebus.py b/link/service/messagebus.py index b0d9d372..f8c0b030 100644 --- a/link/service/messagebus.py +++ b/link/service/messagebus.py @@ -35,7 +35,7 @@ def __getitem__(self, event_type: type[V]) -> Iterable[Callable[[V], None]]: """Get the appropriate handlers for the given event.""" -class Messagebus: +class MessageBus: """A message bus that dispatches domain messages to their appropriate handlers.""" def __init__(self, uow: UnitOfWork, command_handlers: CommandHandlers, event_handlers: EventHandlers) -> None: @@ -52,7 +52,7 @@ def handle(self, message: Message) -> None: message = self._queue.popleft() if isinstance(message, Command): self._handle_command(message) - if isinstance(message, Event): + elif isinstance(message, Event): self._handle_event(message) else: raise TypeError(f"Unknown message type {type(message)!r}") From 154cb0179044531dd62a3867672fffa97336338a Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:38:10 +0100 Subject: [PATCH 33/59] Add ability to add items to handler mappings --- link/service/messagebus.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/link/service/messagebus.py b/link/service/messagebus.py index f8c0b030..b0355730 100644 --- a/link/service/messagebus.py +++ b/link/service/messagebus.py @@ -22,7 +22,10 @@ class CommandHandlers(Protocol): """A mapping of command types to handlers.""" def __getitem__(self, command_type: type[T]) -> Callable[[T], None]: - """Get the appropriate handler for the given command.""" + """Get the appropriate handler for the given type of command.""" + + def __setitem__(self, command_type: type[T], handler: Callable[[T], None]) -> None: + """Set the appropriate handler for the given type of command.""" V = TypeVar("V", bound=Event) @@ -32,7 +35,10 @@ class EventHandlers(Protocol): """A mapping of event types to handlers.""" def __getitem__(self, event_type: type[V]) -> Iterable[Callable[[V], None]]: - """Get the appropriate handlers for the given event.""" + """Get the appropriate handlers for the given type of event.""" + + def __setitem__(selG, event_type: type[V], handlers: Iterable[Callable[[V], None]]) -> None: + """Set the appropriate handlers for the given type of event.""" class MessageBus: From 044f58507a1d8d38fc08a8b57c7cb3ee9d5ae889 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:46:45 +0100 Subject: [PATCH 34/59] Add custom eq und hash methods to entity --- link/domain/state.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/link/domain/state.py b/link/domain/state.py index 5fcadc96..dc4c61b9 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -290,3 +290,13 @@ def _start_delete(self) -> Entity: def _process(self) -> Entity: """Process the entity.""" return self.state.process(self) + + def __hash__(self) -> int: + """Return the hash of this entity.""" + return hash(self.identifier) + + def __eq__(self, other: object) -> bool: + """Return True if both entities are equal.""" + if not isinstance(other, type(self)): + raise NotImplementedError + return hash(self) == hash(other) From f82421cbb3c537c22ca3596e30da2999f8c97278 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:57:55 +0100 Subject: [PATCH 35/59] Unfreeze entity --- link/domain/state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/link/domain/state.py b/link/domain/state.py index dc4c61b9..b30b5317 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -260,7 +260,7 @@ class PersistentState: } -@dataclass(frozen=True) +@dataclass class Entity: """An entity in a link.""" From ecb0492b5b3981aff67b325e82bd1142d1eb5e10 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:58:46 +0100 Subject: [PATCH 36/59] Use deque to store events on entity --- link/domain/link.py | 3 ++- link/domain/state.py | 11 ++++++----- tests/unit/entities/test_state.py | 6 ++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 3a1fdfc0..c16a29dc 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -1,6 +1,7 @@ """Contains the link class.""" from __future__ import annotations +from collections import deque from typing import Any, Iterable, Iterator, Mapping, Optional, Set, Tuple, TypeVar from . import events @@ -70,7 +71,7 @@ def create_entity(identifier: Identifier) -> Entity: state=state, current_process=processes_map.get(identifier, Processes.NONE), is_tainted=is_tainted(identifier), - events=tuple(), + events=deque(), ) return {create_entity(identifier) for identifier in assignments[Components.SOURCE]} diff --git a/link/domain/state.py b/link/domain/state.py index b30b5317..fbe6f1bb 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -1,6 +1,7 @@ """Contains everything state related.""" from __future__ import annotations +from collections import deque from dataclasses import dataclass, replace from enum import Enum, auto from functools import partial @@ -29,8 +30,8 @@ def process(cls, entity: Entity) -> Entity: @staticmethod def _create_invalid_operation(entity: Entity, operation: Operations) -> Entity: - updated = entity.events + (InvalidOperationRequested(operation, entity.identifier, entity.state),) - return replace(entity, events=updated) + entity.events.append(InvalidOperationRequested(operation, entity.identifier, entity.state)) + return replace(entity) @classmethod def _transition_entity( @@ -39,10 +40,10 @@ def _transition_entity( if new_process is None: new_process = entity.current_process transition = Transition(cls, new_state) - updated_events = entity.events + ( + entity.events.append( EntityStateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), ) - return replace(entity, state=transition.new, current_process=new_process, events=updated_events) + return replace(entity, state=transition.new, current_process=new_process) class States: @@ -268,7 +269,7 @@ class Entity: state: type[State] current_process: Processes is_tainted: bool - events: tuple[EntityOperationApplied, ...] + events: deque[EntityOperationApplied] def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index fe63e2d8..5254e1ec 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -95,12 +95,10 @@ def test_processing_activated_entity_returns_correct_entity( tainted_identifiers=tainted_identifiers, ) entity = next(iter(link)) - updated_results = entity.events + ( + entity.events.append( events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) - assert entity.apply(Operations.PROCESS) == replace( - entity, state=new_state, current_process=new_process, events=updated_results - ) + assert entity.apply(Operations.PROCESS) == replace(entity, state=new_state, current_process=new_process) @pytest.mark.parametrize( From 2f88920b8f2af5589c4583e73443c419b899914c Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:11:19 +0100 Subject: [PATCH 37/59] Remove entities pulled event --- link/domain/events.py | 8 ----- link/domain/link.py | 5 --- link/infrastructure/link.py | 2 +- link/service/services.py | 9 ++---- tests/integration/test_services.py | 49 +++--------------------------- 5 files changed, 7 insertions(+), 66 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 0fb07b2b..7c31cf21 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -61,14 +61,6 @@ class IdleEntitiesListed(Event): identifiers: frozenset[Identifier] -@dataclass(frozen=True) -class EntitiesPulled(Event): - """The requested entities have been pulled.""" - - requested: frozenset[Identifier] - errors: frozenset[InvalidOperationRequested] - - @dataclass(frozen=True) class EntitiesDeleted(Event): """The requested entities have been deleted.""" diff --git a/link/domain/link.py b/link/domain/link.py index c16a29dc..dbbf3ef8 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -15,7 +15,6 @@ PersistentState, Processes, State, - states, ) @@ -140,11 +139,7 @@ def pull(self, requested: Iterable[Identifier]) -> Link: requested = frozenset(requested) link = _complete_all_processes(self, requested) link = link.apply(Operations.START_PULL, requested=requested) - start_pull_event = link.events[-1] - assert isinstance(start_pull_event, events.LinkStateChanged) link = _complete_all_processes(link, requested) - errors = frozenset(error for error in start_pull_event.errors if error.state is states.Deprecated) - link._events.append(events.EntitiesPulled(requested, errors)) return link def delete(self, requested: Iterable[Identifier]) -> Link: diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index af22682b..1db21155 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -45,7 +45,7 @@ def inner(obj: type) -> Any: source_restriction: IterationCallbackList[PrimaryKey] = IterationCallbackList() idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) handlers = { - Services.PULL: partial(pull, uow=uow, output_port=lambda x: None), + Services.PULL: partial(pull, uow=uow), Services.DELETE: partial(delete, uow=uow, output_port=lambda x: None), Services.LIST_IDLE_ENTITIES: partial(list_idle_entities, uow=uow, output_port=idle_entities_updater), } diff --git a/link/service/services.py b/link/service/services.py index 591cbaef..9b7d182e 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -9,16 +9,11 @@ from .uow import UnitOfWork -def pull( - command: commands.PullEntities, *, uow: UnitOfWork, output_port: Callable[[events.EntitiesPulled], None] -) -> None: +def pull(command: commands.PullEntities, *, uow: UnitOfWork) -> None: """Pull entities across the link.""" with uow: - link = uow.link.pull(command.requested) + uow.link.pull(command.requested) uow.commit() - event = link.events[-1] - assert isinstance(event, events.EntitiesPulled) - output_port(event) def delete( diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index d41c27fc..935d50db 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -6,10 +6,10 @@ import pytest from link.domain import commands, events -from link.domain.state import Components, Operations, Processes, State, states +from link.domain.state import Components, Processes, State, states from link.service.services import delete, list_idle_entities, pull from link.service.uow import UnitOfWork -from tests.assignments import create_assignments, create_identifier, create_identifiers +from tests.assignments import create_assignments, create_identifiers from .gateway import FakeLinkGateway @@ -79,7 +79,7 @@ def __call__(self, command: _Command_contra, *, output_port: Callable[[_Event_co """Execute the service.""" -def create_pull_service(uow: UnitOfWork) -> Service[commands.PullEntities, events.EntitiesPulled]: +def create_pull_service(uow: UnitOfWork) -> Callable[[commands.PullEntities], None]: return partial(pull, uow=uow) @@ -166,52 +166,11 @@ def test_correct_response_model_gets_passed_to_delete_output_port() -> None: def test_pulled_entity_ends_in_correct_state(state: EntityConfig, expected: type[State]) -> None: uow = create_uow(**state) pull_service = create_pull_service(uow) - pull_service( - commands.PullEntities(frozenset(create_identifiers("1"))), - output_port=lambda x: None, - ) + pull_service(commands.PullEntities(frozenset(create_identifiers("1")))) with uow: assert next(iter(uow.link)).state is expected -@pytest.mark.parametrize( - ("state", "produces_error"), - [ - (STATES[0], False), - (STATES[1], False), - (STATES[2], False), - (STATES[3], True), - (STATES[4], True), - (STATES[5], False), - (STATES[6], False), - (STATES[7], False), - (STATES[8], True), - (STATES[9], False), - (STATES[10], False), - (STATES[11], True), - ], -) -def test_correct_response_model_gets_passed_to_pull_output_port(state: EntityConfig, produces_error: bool) -> None: - if produces_error: - errors = { - events.InvalidOperationRequested( - operation=Operations.START_PULL, identifier=create_identifier("1"), state=states.Deprecated - ) - } - else: - errors = set() - gateway = create_uow(**state) - output_port = FakeOutputPort[events.EntitiesPulled]() - pull_service = create_pull_service(gateway) - pull_service( - commands.PullEntities(frozenset(create_identifiers("1"))), - output_port=output_port, - ) - assert output_port.response == events.EntitiesPulled( - requested=frozenset(create_identifiers("1")), errors=frozenset(errors) - ) - - def test_correct_response_model_gets_passed_to_list_idle_entities_output_port() -> None: uow = UnitOfWork( FakeLinkGateway( From a65b81587b19d5628d37a13600b5e940d3fa63bd Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:30:23 +0100 Subject: [PATCH 38/59] Remove entities deleted event --- link/domain/events.py | 7 ------- link/domain/link.py | 1 - link/infrastructure/link.py | 2 +- link/service/services.py | 9 ++------- tests/integration/test_services.py | 25 +++---------------------- 5 files changed, 6 insertions(+), 38 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 7c31cf21..181141d6 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -59,10 +59,3 @@ class IdleEntitiesListed(Event): """Idle entities in a link have been listed.""" identifiers: frozenset[Identifier] - - -@dataclass(frozen=True) -class EntitiesDeleted(Event): - """The requested entities have been deleted.""" - - requested: frozenset[Identifier] diff --git a/link/domain/link.py b/link/domain/link.py index dbbf3ef8..3caaf80e 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -148,7 +148,6 @@ def delete(self, requested: Iterable[Identifier]) -> Link: link = _complete_all_processes(self, requested) link = link.apply(Operations.START_DELETE, requested=requested) link = _complete_all_processes(link, requested) - link._events.append(events.EntitiesDeleted(requested)) return link def list_idle_entities(self) -> None: diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index 1db21155..9509250a 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -46,7 +46,7 @@ def inner(obj: type) -> Any: idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) handlers = { Services.PULL: partial(pull, uow=uow), - Services.DELETE: partial(delete, uow=uow, output_port=lambda x: None), + Services.DELETE: partial(delete, uow=uow), Services.LIST_IDLE_ENTITIES: partial(list_idle_entities, uow=uow, output_port=idle_entities_updater), } controller = DJController(handlers, translator) diff --git a/link/service/services.py b/link/service/services.py index 9b7d182e..96169d72 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -16,16 +16,11 @@ def pull(command: commands.PullEntities, *, uow: UnitOfWork) -> None: uow.commit() -def delete( - command: commands.DeleteEntities, *, uow: UnitOfWork, output_port: Callable[[events.EntitiesDeleted], None] -) -> None: +def delete(command: commands.DeleteEntities, *, uow: UnitOfWork) -> None: """Delete pulled entities.""" with uow: - link = uow.link.delete(command.requested) + uow.link.delete(command.requested) uow.commit() - event = link.events[-1] - assert isinstance(event, events.EntitiesDeleted) - output_port(event) def list_idle_entities( diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 935d50db..72521a7e 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -1,7 +1,7 @@ from __future__ import annotations from functools import partial -from typing import Any, Callable, Generic, Protocol, TypedDict, TypeVar +from typing import Callable, Generic, TypedDict, TypeVar import pytest @@ -72,18 +72,11 @@ def create_uow(state: type[State], process: Processes | None = None, is_tainted: _Command_contra = TypeVar("_Command_contra", bound=commands.Command, contravariant=True) -class Service(Protocol[_Command_contra, _Event_co]): - """Protocol for services.""" - - def __call__(self, command: _Command_contra, *, output_port: Callable[[_Event_co], None], **kwargs: Any) -> None: - """Execute the service.""" - - def create_pull_service(uow: UnitOfWork) -> Callable[[commands.PullEntities], None]: return partial(pull, uow=uow) -def create_delete_service(uow: UnitOfWork) -> Service[commands.DeleteEntities, events.EntitiesDeleted]: +def create_delete_service(uow: UnitOfWork) -> Callable[[commands.DeleteEntities], None]: return partial(delete, uow=uow) @@ -129,23 +122,11 @@ class EntityConfig(TypedDict): def test_deleted_entity_ends_in_correct_state(state: EntityConfig, expected: type[State]) -> None: uow = create_uow(**state) delete_service = create_delete_service(uow) - delete_service(commands.DeleteEntities(frozenset(create_identifiers("1"))), output_port=lambda x: None) + delete_service(commands.DeleteEntities(frozenset(create_identifiers("1")))) with uow: assert next(iter(uow.link)).state is expected -def test_correct_response_model_gets_passed_to_delete_output_port() -> None: - uow = UnitOfWork( - FakeLinkGateway( - create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}, Components.LOCAL: {"1"}}) - ) - ) - output_port = FakeOutputPort[events.EntitiesDeleted]() - delete_service = create_delete_service(uow) - delete_service(commands.DeleteEntities(frozenset(create_identifiers("1"))), output_port=output_port) - assert output_port.response.requested == create_identifiers("1") - - @pytest.mark.parametrize( ("state", "expected"), [ From 88807530f8aabefa0c48ea9fec840671498d2f1a Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:37:50 +0100 Subject: [PATCH 39/59] Remove operations presenter --- link/adapters/present.py | 84 -------------------------------------- link/infrastructure/log.py | 21 ---------- 2 files changed, 105 deletions(-) delete mode 100644 link/infrastructure/log.py diff --git a/link/adapters/present.py b/link/adapters/present.py index f308e696..09c13bfc 100644 --- a/link/adapters/present.py +++ b/link/adapters/present.py @@ -1,7 +1,6 @@ """Logic associated with presenting information about finished use-cases.""" from __future__ import annotations -from dataclasses import dataclass from typing import Callable, Iterable from link.domain import events @@ -10,89 +9,6 @@ from .identification import IdentificationTranslator -@dataclass(frozen=True) -class OperationRecord: - """Record of a finished operation.""" - - requests: list[Request] - successes: list[Sucess] - failures: list[Failure] - - -@dataclass(frozen=True) -class Request: - """Record of a request to perform a certain operation on a particular entity.""" - - primary_key: PrimaryKey - operation: str - - -@dataclass(frozen=True) -class Sucess: - """Record of a successful operation on a particular entity.""" - - primary_key: PrimaryKey - operation: str - transition: Transition - - -@dataclass(frozen=True) -class Transition: - """Record of a transition between two states.""" - - old: str - new: str - - -@dataclass(frozen=True) -class Failure: - """Record of a failed operation on a particular entity.""" - - primary_key: PrimaryKey - operation: str - state: str - - -def create_operation_response_presenter( - translator: IdentificationTranslator, show: Callable[[OperationRecord], None] -) -> Callable[[events.LinkStateChanged], None]: - """Create a callable that when called presents information about a finished operation.""" - - def get_class_name(obj: type) -> str: - return obj.__name__ - - def present_operation_response(response: events.LinkStateChanged) -> None: - show( - OperationRecord( - [ - Request(translator.to_primary_key(identifier), response.operation.name) - for identifier in response.requested - ], - [ - Sucess( - translator.to_primary_key(update.identifier), - operation=response.operation.name, - transition=Transition( - get_class_name(update.transition.current).upper(), - get_class_name(update.transition.new).upper(), - ), - ) - for update in response.updates - ], - [ - Failure( - translator.to_primary_key(error.identifier), - error.operation.name, - get_class_name(error.state).upper(), - ) - for error in response.errors - ], - ) - ) - - return present_operation_response - - def create_idle_entities_updater( translator: IdentificationTranslator, update: Callable[[Iterable[PrimaryKey]], None] ) -> Callable[[events.IdleEntitiesListed], None]: diff --git a/link/infrastructure/log.py b/link/infrastructure/log.py deleted file mode 100644 index b7f4248e..00000000 --- a/link/infrastructure/log.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Logging functionality.""" -import logging -from dataclasses import asdict -from typing import Callable - -from link.adapters.present import OperationRecord - - -def create_operation_logger() -> Callable[[OperationRecord], None]: - """Create a function that logs information about finished operations.""" - logger = logging.getLogger("link[operations]") - - def log(record: OperationRecord) -> None: - for request in record.requests: - logger.info(f"Operation requested {asdict(request)}") - for success in record.successes: - logger.info(f"Operation succeeded {asdict(success)}") - for failure in record.failures: - logger.info(f"Operation failed {asdict(failure)}") - - return log From d75f0921a601b4b8aa884ae6855fd3a537199a4e Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:41:55 +0100 Subject: [PATCH 40/59] Remove link state changed event --- link/domain/link.py | 19 +------------------ .../integration/test_datajoint_persistence.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 3caaf80e..85637841 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -111,28 +111,11 @@ def events(self) -> Tuple[events.Event, ...]: def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" - - def create_operation_result( - results: Iterable[events.EntityOperationApplied], requested: Iterable[Identifier] - ) -> events.LinkStateChanged: - """Create the result of an operation on a link from results of individual entities.""" - results = set(results) - operation = next(iter(results)).operation - return events.LinkStateChanged( - operation, - requested=frozenset(requested), - updates=frozenset(result for result in results if isinstance(result, events.EntityStateChanged)), - errors=frozenset(result for result in results if isinstance(result, events.InvalidOperationRequested)), - ) - assert requested, "No identifiers requested." assert set(requested) <= self.identifiers, "Requested identifiers not present in link." changed = {entity.apply(operation) for entity in self if entity.identifier in requested} unchanged = {entity for entity in self if entity.identifier not in requested} - operation_results = self.events + ( - create_operation_result((entity.events[-1] for entity in changed), requested), - ) - return Link(changed | unchanged, operation_results) + return Link(changed | unchanged) def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" diff --git a/tests/integration/test_datajoint_persistence.py b/tests/integration/test_datajoint_persistence.py index f8de5f06..0dcd98f7 100644 --- a/tests/integration/test_datajoint_persistence.py +++ b/tests/integration/test_datajoint_persistence.py @@ -342,13 +342,15 @@ def test_link_creation() -> None: def apply_update(gateway: DJLinkGateway, operation: Operations, requested: Iterable[PrimaryKey]) -> None: - event = ( - gateway.create_link() - .apply(operation, requested={gateway.translator.to_identifier(key) for key in requested}) - .events[0] + link = gateway.create_link().apply( + operation, requested={gateway.translator.to_identifier(key) for key in requested} ) - assert isinstance(event, events.LinkStateChanged) - gateway.apply(event.updates) + for entity in link: + while entity.events: + event = entity.events.popleft() + if not isinstance(event, events.EntityStateChanged): + continue + gateway.apply([event]) def test_add_to_local_command() -> None: From 2e4ce7e6093a2d55391ff79da87770633a7a6fdc Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:44:26 +0100 Subject: [PATCH 41/59] Produce idle entities listed event in service --- link/domain/link.py | 14 +++----------- link/service/services.py | 6 ++---- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 85637841..9d35cd1b 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections import deque -from typing import Any, Iterable, Iterator, Mapping, Optional, Set, Tuple, TypeVar +from typing import Any, Iterable, Iterator, Mapping, Optional, Set, TypeVar from . import events from .custom_types import Identifier @@ -97,18 +97,12 @@ class Link(Set[Entity]): def __init__(self, entities: Iterable[Entity], events: Iterable[events.Event] | None = None) -> None: """Initialize the link.""" self._entities = set(entities) - self._events = list(events) if events is not None else [] @property def identifiers(self) -> frozenset[Identifier]: """Return the identifiers of all entities in the link.""" return frozenset(entity.identifier for entity in self) - @property - def events(self) -> Tuple[events.Event, ...]: - """Return the events that happened to the link.""" - return tuple(self._events) - def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: """Apply an operation to the requested entities.""" assert requested, "No identifiers requested." @@ -133,11 +127,9 @@ def delete(self, requested: Iterable[Identifier]) -> Link: link = _complete_all_processes(link, requested) return link - def list_idle_entities(self) -> None: + def list_idle_entities(self) -> frozenset[Identifier]: """List the identifiers of all idle entities in the link.""" - self._events.append( - events.IdleEntitiesListed(frozenset(entity.identifier for entity in self._entities if entity.state is Idle)) - ) + return frozenset(entity.identifier for entity in self if entity.state is Idle) def __contains__(self, entity: object) -> bool: """Check if the link contains the given entity.""" diff --git a/link/service/services.py b/link/service/services.py index 96169d72..b2be348a 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -31,10 +31,8 @@ def list_idle_entities( ) -> None: """List all idle entities.""" with uow: - uow.link.list_idle_entities() - event = uow.link.events[-1] - assert isinstance(event, events.IdleEntitiesListed) - output_port(event) + idle = uow.link.list_idle_entities() + output_port(events.IdleEntitiesListed(idle)) class Services(Enum): From 7a3fde6f6e44a6feac01e58c24fdc7469c0d5354 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:59:02 +0100 Subject: [PATCH 42/59] Add pull method to entity --- link/domain/link.py | 10 +++++----- link/domain/state.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 9d35cd1b..4f110a7f 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -113,11 +113,11 @@ def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Li def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" - requested = frozenset(requested) - link = _complete_all_processes(self, requested) - link = link.apply(Operations.START_PULL, requested=requested) - link = _complete_all_processes(link, requested) - return link + changed = set() + for entity in (entity for entity in self if entity.identifier in requested): + changed.add(entity.pull()) + unchanged = self - changed + return type(self)(changed | unchanged) def delete(self, requested: Iterable[Identifier]) -> Link: """Delete the requested entities.""" diff --git a/link/domain/state.py b/link/domain/state.py index fbe6f1bb..3cd1bcef 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -271,6 +271,12 @@ class Entity: is_tainted: bool events: deque[EntityOperationApplied] + def pull(self) -> Entity: + """Pull the entity.""" + entity = self._finish_process(self) + entity = entity.apply(Operations.START_PULL) + return self._finish_process(entity) + def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" if operation is Operations.START_PULL: @@ -292,6 +298,12 @@ def _process(self) -> Entity: """Process the entity.""" return self.state.process(self) + @staticmethod + def _finish_process(entity: Entity) -> Entity: + while entity.current_process is not Processes.NONE: + entity = entity.apply(Operations.PROCESS) + return entity + def __hash__(self) -> int: """Return the hash of this entity.""" return hash(self.identifier) From 35f92933ef63758f903a5769442dfc8abf6587b4 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Fri, 3 Nov 2023 18:03:37 +0100 Subject: [PATCH 43/59] Add delete method to entity --- link/domain/link.py | 10 +++++----- link/domain/state.py | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 4f110a7f..bd358573 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -121,11 +121,11 @@ def pull(self, requested: Iterable[Identifier]) -> Link: def delete(self, requested: Iterable[Identifier]) -> Link: """Delete the requested entities.""" - requested = frozenset(requested) - link = _complete_all_processes(self, requested) - link = link.apply(Operations.START_DELETE, requested=requested) - link = _complete_all_processes(link, requested) - return link + changed = set() + for entity in (entity for entity in self if entity.identifier in requested): + changed.add(entity.delete()) + unchanged = self - changed + return type(self)(changed | unchanged) def list_idle_entities(self) -> frozenset[Identifier]: """List the identifiers of all idle entities in the link.""" diff --git a/link/domain/state.py b/link/domain/state.py index 3cd1bcef..8e1a05d6 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -277,6 +277,12 @@ def pull(self) -> Entity: entity = entity.apply(Operations.START_PULL) return self._finish_process(entity) + def delete(self) -> Entity: + """Delete the entity.""" + entity = self._finish_process(self) + entity = entity.apply(Operations.START_DELETE) + return self._finish_process(entity) + def apply(self, operation: Operations) -> Entity: """Apply an operation to the entity.""" if operation is Operations.START_PULL: From 96f99a5028657622821086c79dc0b58e7c29a60d Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:46:23 +0100 Subject: [PATCH 44/59] Remove events parameter from link init --- link/domain/link.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index bd358573..018573c2 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -4,7 +4,6 @@ from collections import deque from typing import Any, Iterable, Iterator, Mapping, Optional, Set, TypeVar -from . import events from .custom_types import Identifier from .state import ( STATE_MAP, @@ -94,7 +93,7 @@ def assign_to_component(component: Components) -> set[Entity]: class Link(Set[Entity]): """The state of a link between two databases.""" - def __init__(self, entities: Iterable[Entity], events: Iterable[events.Event] | None = None) -> None: + def __init__(self, entities: Iterable[Entity]) -> None: """Initialize the link.""" self._entities = set(entities) From 39efed4ab85bbc63da83d76a8a72863c03f6165a Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:55:07 +0100 Subject: [PATCH 45/59] Do not use apply method on link in uow tests --- tests/integration/test_uow.py | 36 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 651e8e44..18fcb37d 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -19,10 +19,8 @@ def initialize(assignments: Mapping[Components, Iterable[str]]) -> tuple[FakeLin def test_updates_are_applied_to_gateway_on_commit() -> None: gateway, uow = initialize({Components.SOURCE: {"1", "2"}, Components.OUTBOUND: {"2"}, Components.LOCAL: {"2"}}) with uow: - uow.link.apply(Operations.START_PULL, requested=create_identifiers("1")) - uow.link.apply(Operations.START_DELETE, requested=create_identifiers("2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) + uow.link.pull(create_identifiers("1")) + uow.link.delete(create_identifiers("2")) uow.commit() actual = {(entity.identifier, entity.state) for entity in gateway.create_link()} expected = {(create_identifier("1"), states.Pulled), (create_identifier("2"), states.Idle)} @@ -32,10 +30,8 @@ def test_updates_are_applied_to_gateway_on_commit() -> None: def test_updates_are_discarded_on_context_exit() -> None: gateway, uow = initialize({Components.SOURCE: {"1", "2"}, Components.OUTBOUND: {"2"}, Components.LOCAL: {"2"}}) with uow: - uow.link.apply(Operations.START_PULL, requested=create_identifiers("1")) - uow.link.apply(Operations.START_DELETE, requested=create_identifiers("2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) + uow.link.pull(create_identifiers("1")) + uow.link.delete(create_identifiers("2")) actual = {(entity.identifier, entity.state) for entity in gateway.create_link()} expected = {(create_identifier("1"), states.Idle), (create_identifier("2"), states.Pulled)} assert actual == expected @@ -44,10 +40,8 @@ def test_updates_are_discarded_on_context_exit() -> None: def test_updates_are_discarded_on_rollback() -> None: gateway, uow = initialize({Components.SOURCE: {"1", "2"}, Components.OUTBOUND: {"2"}, Components.LOCAL: {"2"}}) with uow: - uow.link.apply(Operations.START_PULL, requested=create_identifiers("1")) - uow.link.apply(Operations.START_DELETE, requested=create_identifiers("2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) - uow.link.apply(Operations.PROCESS, requested=create_identifiers("1", "2")) + uow.link.pull(create_identifiers("1")) + uow.link.delete(create_identifiers("2")) uow.rollback() actual = {(entity.identifier, entity.state) for entity in gateway.create_link()} expected = {(create_identifier("1"), states.Idle), (create_identifier("2"), states.Pulled)} @@ -114,8 +108,8 @@ def test_link_expires_when_committing() -> None: with uow: link = uow.link uow.commit() - with pytest.raises(RuntimeError, match="expired link"): - link.apply(Operations.START_PULL, requested=create_identifiers("1")) + with pytest.raises(RuntimeError, match="expired entity"): + link.pull(create_identifiers("1")) def test_link_expires_when_rolling_back() -> None: @@ -123,22 +117,22 @@ def test_link_expires_when_rolling_back() -> None: with uow: link = uow.link uow.rollback() - with pytest.raises(RuntimeError, match="expired link"): - link.apply(Operations.START_PULL, requested=create_identifiers("1")) + with pytest.raises(RuntimeError, match="expired entity"): + link.pull(create_identifiers("1")) def test_link_expires_when_exiting_context() -> None: _, uow = initialize({Components.SOURCE: {"1"}}) with uow: link = uow.link - with pytest.raises(RuntimeError, match="expired link"): - link.apply(Operations.START_PULL, requested=create_identifiers("1")) + with pytest.raises(RuntimeError, match="expired entity"): + link.pull(create_identifiers("1")) def test_link_expires_when_applying_operation() -> None: _, uow = initialize({Components.SOURCE: {"1"}}) with uow: link = uow.link - link.apply(Operations.START_PULL, requested=create_identifiers("1")) - with pytest.raises(RuntimeError, match="expired link"): - link.apply(Operations.PROCESS, requested=create_identifiers("1")) + link.pull(create_identifiers("1")) + with pytest.raises(RuntimeError, match="expired entity"): + link.pull(create_identifiers("1")) From 844606bca346f2dce999fa72df34c2ddd431cf5b Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 10:56:13 +0100 Subject: [PATCH 46/59] Remove unused function from link module --- link/domain/link.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 018573c2..d5924ab7 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -151,10 +151,3 @@ def create_identifier_state_pairs(link: Link) -> set[tuple[Identifier, type[Stat return {(entity.identifier, entity.state) for entity in link} return create_identifier_state_pairs(self) == create_identifier_state_pairs(other) - - -def _complete_all_processes(current: Link, requested: Iterable[Identifier]) -> Link: - new = current.apply(Operations.PROCESS, requested=requested) - if new == current: - return new - return _complete_all_processes(new, requested) From 917e0ceb8ec94a3b68a0139d07db401fc70db66e Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:00:45 +0100 Subject: [PATCH 47/59] Do not use link apply method in persistence tests --- tests/integration/test_datajoint_persistence.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_datajoint_persistence.py b/tests/integration/test_datajoint_persistence.py index 0dcd98f7..caa5bd0e 100644 --- a/tests/integration/test_datajoint_persistence.py +++ b/tests/integration/test_datajoint_persistence.py @@ -342,10 +342,11 @@ def test_link_creation() -> None: def apply_update(gateway: DJLinkGateway, operation: Operations, requested: Iterable[PrimaryKey]) -> None: - link = gateway.create_link().apply( - operation, requested={gateway.translator.to_identifier(key) for key in requested} - ) + link = gateway.create_link() for entity in link: + if entity.identifier not in {gateway.translator.to_identifier(key) for key in requested}: + continue + entity.apply(operation) while entity.events: event = entity.events.popleft() if not isinstance(event, events.EntityStateChanged): From 4d1b3bc32840469d64732c6494f963b344f42394 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:18:23 +0100 Subject: [PATCH 48/59] Do not use link apply in link tests --- link/domain/link.py | 7 +++ tests/unit/entities/test_link.py | 84 ++++---------------------------- 2 files changed, 17 insertions(+), 74 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index d5924ab7..289b1f92 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -112,6 +112,8 @@ def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Li def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" + requested = set(requested) + self._validate_requested(requested) changed = set() for entity in (entity for entity in self if entity.identifier in requested): changed.add(entity.pull()) @@ -120,6 +122,8 @@ def pull(self, requested: Iterable[Identifier]) -> Link: def delete(self, requested: Iterable[Identifier]) -> Link: """Delete the requested entities.""" + requested = set(requested) + self._validate_requested(requested) changed = set() for entity in (entity for entity in self if entity.identifier in requested): changed.add(entity.delete()) @@ -130,6 +134,9 @@ def list_idle_entities(self) -> frozenset[Identifier]: """List the identifiers of all idle entities in the link.""" return frozenset(entity.identifier for entity in self if entity.state is Idle) + def _validate_requested(self, requested: Iterable[Identifier]) -> None: + assert set(requested) <= self.identifiers, "Requested identifiers not present in link." + def __contains__(self, entity: object) -> bool: """Check if the link contains the given entity.""" return entity in self._entities diff --git a/tests/unit/entities/test_link.py b/tests/unit/entities/test_link.py index 9df845a3..b4bfd4aa 100644 --- a/tests/unit/entities/test_link.py +++ b/tests/unit/entities/test_link.py @@ -6,8 +6,8 @@ import pytest from link.domain.custom_types import Identifier -from link.domain.link import Link, create_link -from link.domain.state import Components, Operations, Processes, State, states +from link.domain.link import create_link +from link.domain.state import Components, Processes, State, states from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -166,80 +166,16 @@ def test_can_get_identifiers_of_entities_in_component( link = create_link(assignments) assert set(link.identifiers) == create_identifiers("1", "2") - -def test_link_is_processed_correctly() -> None: - link = create_link( - create_assignments( - { - Components.SOURCE: {"1", "2", "3", "4", "5"}, - Components.OUTBOUND: {"1", "2", "3", "4", "5"}, - Components.LOCAL: {"2", "4", "5"}, - } - ), - processes={ - Processes.PULL: create_identifiers("1", "2"), - Processes.DELETE: create_identifiers("3", "4", "5"), - }, - ) - actual = { - (entity.identifier, entity.state) - for entity in link.apply(Operations.PROCESS, requested=create_identifiers("1", "2", "3", "4")) - } - expected = { - (create_identifier("1"), states.Received), - (create_identifier("2"), states.Pulled), - (create_identifier("3"), states.Idle), - (create_identifier("4"), states.Activated), - (create_identifier("5"), states.Received), - } - assert actual == expected - - -class TestStartPull: - @staticmethod - @pytest.fixture() - def link() -> Link: - return create_link(create_assignments({Components.SOURCE: {"1"}})) - - @staticmethod - def test_idle_entity_becomes_activated(link: Link) -> None: - link = link.apply(Operations.START_PULL, requested=create_identifiers("1")) - entity = next(iter(link)) - assert entity.identifier == create_identifier("1") - assert entity.state is states.Activated - - @staticmethod - def test_not_specifying_requested_identifiers_raises_error(link: Link) -> None: - with pytest.raises(AssertionError, match="No identifiers requested."): - link.apply(Operations.START_PULL, requested={}) - @staticmethod - def test_specifying_identifiers_not_present_in_link_raises_error(link: Link) -> None: + def test_specifying_identifiers_not_present_in_link_raises_error_when_pulling() -> None: + link = create_link(create_assignments({Components.SOURCE: {"1"}})) with pytest.raises(AssertionError, match="Requested identifiers not present in link."): - link.apply(Operations.START_PULL, requested=create_identifiers("2")) - - -@pytest.fixture() -def link() -> Link: - return create_link( - create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}, Components.LOCAL: {"1"}}) - ) - - -class TestStartDelete: - @staticmethod - def test_pulled_entity_becomes_received(link: Link) -> None: - link = link.apply(Operations.START_DELETE, requested=create_identifiers("1")) - entity = next(iter(link)) - assert entity.identifier == create_identifier("1") - assert entity.state is states.Received - - @staticmethod - def test_not_specifying_requested_identifiers_raises_error(link: Link) -> None: - with pytest.raises(AssertionError, match="No identifiers requested."): - link.apply(Operations.START_DELETE, requested={}) + link.pull(create_identifiers("2")) @staticmethod - def test_specifying_identifiers_not_present_in_link_raises_error(link: Link) -> None: + def test_specifying_identifiers_not_present_in_link_raises_error_when_deleting() -> None: + link = create_link( + create_assignments({Components.SOURCE: {"1"}, Components.OUTBOUND: {"1"}, Components.LOCAL: {"1"}}) + ) with pytest.raises(AssertionError, match="Requested identifiers not present in link."): - link.apply(Operations.START_DELETE, requested=create_identifiers("2")) + link.delete(create_identifiers("2")) From 0382a07aa82f9fd0b390bd63e76be9db3529ca94 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:22:11 +0100 Subject: [PATCH 49/59] Remove apply method from link --- link/domain/link.py | 19 +------------------ link/service/uow.py | 20 -------------------- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 289b1f92..0962c777 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -5,16 +5,7 @@ from typing import Any, Iterable, Iterator, Mapping, Optional, Set, TypeVar from .custom_types import Identifier -from .state import ( - STATE_MAP, - Components, - Entity, - Idle, - Operations, - PersistentState, - Processes, - State, -) +from .state import STATE_MAP, Components, Entity, Idle, PersistentState, Processes, State def create_link( @@ -102,14 +93,6 @@ def identifiers(self) -> frozenset[Identifier]: """Return the identifiers of all entities in the link.""" return frozenset(entity.identifier for entity in self) - def apply(self, operation: Operations, *, requested: Iterable[Identifier]) -> Link: - """Apply an operation to the requested entities.""" - assert requested, "No identifiers requested." - assert set(requested) <= self.identifiers, "Requested identifiers not present in link." - changed = {entity.apply(operation) for entity in self if entity.identifier in requested} - unchanged = {entity for entity in self if entity.identifier not in requested} - return Link(changed | unchanged) - def pull(self, requested: Iterable[Identifier]) -> Link: """Pull the requested entities.""" requested = set(requested) diff --git a/link/service/uow.py b/link/service/uow.py index e7ede334..e96eaf68 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -33,24 +33,6 @@ def __init__(self, gateway: LinkGateway) -> None: def __enter__(self) -> UnitOfWork: """Enter the context in which updates to entities can be made.""" - def augment_link(link: Link) -> None: - original = getattr(link, "apply") - augmented = augment_link_apply(link, original) - setattr(link, "apply", augmented) - setattr(link, "_is_expired", False) - - def augment_link_apply(current: Link, original: SupportsLinkApply) -> SupportsLinkApply: - def augmented(operation: Operations, *, requested: Iterable[Identifier]) -> Link: - assert hasattr(current, "_is_expired") - if current._is_expired: - raise RuntimeError("Can not apply operation to expired link") - self._link = original(operation, requested=requested) - augment_link(self._link) - setattr(current, "_is_expired", True) - return self._link - - return augmented - def augment_entity(entity: Entity) -> None: original = getattr(entity, "apply") augmented = augment_entity_apply(entity, original) @@ -73,7 +55,6 @@ def augmented(operation: Operations) -> Entity: return augmented self._link = self._gateway.create_link() - augment_link(self._link) for entity in self._link: augment_entity(entity) return self @@ -106,7 +87,6 @@ def rollback(self) -> None: """Throw away any not yet persisted updates.""" if self._link is None: raise RuntimeError("Not available outside of context") - setattr(self._link, "_is_expired", True) for entity in self._link: object.__setattr__(entity, "_is_expired", True) self._seen.clear() From 6719c97d5672536e65739b3e171ed49d5fa57cce Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:35:14 +0100 Subject: [PATCH 50/59] Modify entities instead of creating new ones --- link/domain/link.py | 14 ++----- link/domain/state.py | 58 +++++++++++++-------------- link/service/uow.py | 44 +++++++++++---------- tests/integration/test_uow.py | 18 --------- tests/unit/entities/test_state.py | 65 ++++++++++++++++--------------- 5 files changed, 89 insertions(+), 110 deletions(-) diff --git a/link/domain/link.py b/link/domain/link.py index 0962c777..942d5422 100644 --- a/link/domain/link.py +++ b/link/domain/link.py @@ -93,25 +93,19 @@ def identifiers(self) -> frozenset[Identifier]: """Return the identifiers of all entities in the link.""" return frozenset(entity.identifier for entity in self) - def pull(self, requested: Iterable[Identifier]) -> Link: + def pull(self, requested: Iterable[Identifier]) -> None: """Pull the requested entities.""" requested = set(requested) self._validate_requested(requested) - changed = set() for entity in (entity for entity in self if entity.identifier in requested): - changed.add(entity.pull()) - unchanged = self - changed - return type(self)(changed | unchanged) + entity.pull() - def delete(self, requested: Iterable[Identifier]) -> Link: + def delete(self, requested: Iterable[Identifier]) -> None: """Delete the requested entities.""" requested = set(requested) self._validate_requested(requested) - changed = set() for entity in (entity for entity in self if entity.identifier in requested): - changed.add(entity.delete()) - unchanged = self - changed - return type(self)(changed | unchanged) + entity.delete() def list_idle_entities(self) -> frozenset[Identifier]: """List the identifiers of all idle entities in the link.""" diff --git a/link/domain/state.py b/link/domain/state.py index 8e1a05d6..272cd68f 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections import deque -from dataclasses import dataclass, replace +from dataclasses import dataclass from enum import Enum, auto from functools import partial @@ -14,36 +14,36 @@ class State: """An entity's state.""" @classmethod - def start_pull(cls, entity: Entity) -> Entity: + def start_pull(cls, entity: Entity) -> None: """Return the command needed to start the pull process for the entity.""" return cls._create_invalid_operation(entity, Operations.START_PULL) @classmethod - def start_delete(cls, entity: Entity) -> Entity: + def start_delete(cls, entity: Entity) -> None: """Return the commands needed to start the delete process for the entity.""" return cls._create_invalid_operation(entity, Operations.START_DELETE) @classmethod - def process(cls, entity: Entity) -> Entity: + def process(cls, entity: Entity) -> None: """Return the commands needed to process the entity.""" return cls._create_invalid_operation(entity, Operations.PROCESS) @staticmethod - def _create_invalid_operation(entity: Entity, operation: Operations) -> Entity: + def _create_invalid_operation(entity: Entity, operation: Operations) -> None: entity.events.append(InvalidOperationRequested(operation, entity.identifier, entity.state)) - return replace(entity) @classmethod def _transition_entity( cls, entity: Entity, operation: Operations, new_state: type[State], *, new_process: Processes | None = None - ) -> Entity: + ) -> None: if new_process is None: new_process = entity.current_process transition = Transition(cls, new_state) + entity.state = transition.new + entity.current_process = new_process entity.events.append( EntityStateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), ) - return replace(entity, state=transition.new, current_process=new_process) class States: @@ -69,7 +69,7 @@ class Idle(State): """The default state of an entity.""" @classmethod - def start_pull(cls, entity: Entity) -> Entity: + def start_pull(cls, entity: Entity) -> None: """Return the command needed to start the pull process for an entity.""" return cls._transition_entity(entity, Operations.START_PULL, Activated, new_process=Processes.PULL) @@ -81,7 +81,7 @@ class Activated(State): """The state of an activated entity.""" @classmethod - def process(cls, entity: Entity) -> Entity: + def process(cls, entity: Entity) -> None: """Return the commands needed to process an activated entity.""" transition_entity = partial(cls._transition_entity, entity, Operations.PROCESS) if entity.is_tainted: @@ -100,7 +100,7 @@ class Received(State): """The state of an received entity.""" @classmethod - def process(cls, entity: Entity) -> Entity: + def process(cls, entity: Entity) -> None: """Return the commands needed to process a received entity.""" transition_entity = partial(cls._transition_entity, entity, Operations.PROCESS) if entity.current_process is Processes.PULL: @@ -120,7 +120,7 @@ class Pulled(State): """The state of an entity that has been copied to the local side.""" @classmethod - def start_delete(cls, entity: Entity) -> Entity: + def start_delete(cls, entity: Entity) -> None: """Return the commands needed to start the delete process for the entity.""" return cls._transition_entity(entity, Operations.START_DELETE, Received, new_process=Processes.DELETE) @@ -132,7 +132,7 @@ class Tainted(State): """The state of an entity that has been flagged as faulty by the source side.""" @classmethod - def start_delete(cls, entity: Entity) -> Entity: + def start_delete(cls, entity: Entity) -> None: """Return the commands needed to start the delete process for the entity.""" return cls._transition_entity(entity, Operations.START_DELETE, Received, new_process=Processes.DELETE) @@ -271,19 +271,19 @@ class Entity: is_tainted: bool events: deque[EntityOperationApplied] - def pull(self) -> Entity: + def pull(self) -> None: """Pull the entity.""" - entity = self._finish_process(self) - entity = entity.apply(Operations.START_PULL) - return self._finish_process(entity) + self._finish_process() + self.apply(Operations.START_PULL) + self._finish_process() - def delete(self) -> Entity: + def delete(self) -> None: """Delete the entity.""" - entity = self._finish_process(self) - entity = entity.apply(Operations.START_DELETE) - return self._finish_process(entity) + self._finish_process() + self.apply(Operations.START_DELETE) + self._finish_process() - def apply(self, operation: Operations) -> Entity: + def apply(self, operation: Operations) -> None: """Apply an operation to the entity.""" if operation is Operations.START_PULL: return self._start_pull() @@ -292,23 +292,21 @@ def apply(self, operation: Operations) -> Entity: if operation is Operations.PROCESS: return self._process() - def _start_pull(self) -> Entity: + def _start_pull(self) -> None: """Start the pull process for the entity.""" return self.state.start_pull(self) - def _start_delete(self) -> Entity: + def _start_delete(self) -> None: """Start the delete process for the entity.""" return self.state.start_delete(self) - def _process(self) -> Entity: + def _process(self) -> None: """Process the entity.""" return self.state.process(self) - @staticmethod - def _finish_process(entity: Entity) -> Entity: - while entity.current_process is not Processes.NONE: - entity = entity.apply(Operations.PROCESS) - return entity + def _finish_process(self) -> None: + while self.current_process is not Processes.NONE: + self.apply(Operations.PROCESS) def __hash__(self) -> int: """Return the hash of this entity.""" diff --git a/link/service/uow.py b/link/service/uow.py index e96eaf68..8454c8af 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -6,10 +6,11 @@ from types import TracebackType from typing import Callable, Iterable, Protocol +from link.domain import events from link.domain.custom_types import Identifier from link.domain.events import EntityStateChanged from link.domain.link import Link -from link.domain.state import Entity, Operations +from link.domain.state import TRANSITION_MAP, Entity, Operations, Transition from .gateway import LinkGateway @@ -28,7 +29,7 @@ def __init__(self, gateway: LinkGateway) -> None: """Initialize the unit of work.""" self._gateway = gateway self._link: Link | None = None - self._seen: dict[Identifier, Entity] = {} + self._updates: deque[EntityStateChanged] = deque() def __enter__(self) -> UnitOfWork: """Enter the context in which updates to entities can be made.""" @@ -36,21 +37,24 @@ def __enter__(self) -> UnitOfWork: def augment_entity(entity: Entity) -> None: original = getattr(entity, "apply") augmented = augment_entity_apply(entity, original) - object.__setattr__(entity, "apply", augmented) - object.__setattr__(entity, "_is_expired", False) - self._seen[entity.identifier] = entity + setattr(entity, "apply", augmented) + setattr(entity, "_is_expired", False) def augment_entity_apply( - current: Entity, original: Callable[[Operations], Entity] - ) -> Callable[[Operations], Entity]: - def augmented(operation: Operations) -> Entity: - assert hasattr(current, "_is_expired") - if current._is_expired is True: - raise RuntimeError("Can not apply operation to expired entity") - new = original(operation) - augment_entity(new) - object.__setattr__(current, "_is_expired", True) - return new + entity: Entity, original: Callable[[Operations], None] + ) -> Callable[[Operations], None]: + def augmented(operation: Operations) -> None: + assert hasattr(entity, "_is_expired") + if entity._is_expired: + raise RuntimeError("Can not apply operation to expired entity.") + current_state = entity.state + original(operation) + new_state = entity.state + if current_state is new_state: + return + transition = Transition(current_state, new_state) + command = TRANSITION_MAP[transition] + self._updates.append(events.EntityStateChanged(operation, entity.identifier, transition, command)) return augmented @@ -77,10 +81,8 @@ def commit(self) -> None: """Persist updates made to the link.""" if self._link is None: raise RuntimeError("Not available outside of context") - for entity in self._seen.values(): - updates = deque(event for event in entity.events if isinstance(event, EntityStateChanged)) - while updates: - self._gateway.apply([updates.popleft()]) + while self._updates: + self._gateway.apply([self._updates.popleft()]) self.rollback() def rollback(self) -> None: @@ -88,5 +90,5 @@ def rollback(self) -> None: if self._link is None: raise RuntimeError("Not available outside of context") for entity in self._link: - object.__setattr__(entity, "_is_expired", True) - self._seen.clear() + setattr(entity, "_is_expired", True) + self._updates.clear() diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 18fcb37d..439fec55 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -94,15 +94,6 @@ def test_entity_expires_when_leaving_context() -> None: entity.apply(Operations.START_PULL) -def test_entity_expires_when_applying_operation() -> None: - _, uow = initialize({Components.SOURCE: {"1"}}) - with uow: - entity = next(entity for entity in uow.link if entity.identifier == create_identifier("1")) - entity.apply(Operations.START_PULL) - with pytest.raises(RuntimeError, match="expired entity"): - entity.apply(Operations.PROCESS) - - def test_link_expires_when_committing() -> None: _, uow = initialize({Components.SOURCE: {"1"}}) with uow: @@ -127,12 +118,3 @@ def test_link_expires_when_exiting_context() -> None: link = uow.link with pytest.raises(RuntimeError, match="expired entity"): link.pull(create_identifiers("1")) - - -def test_link_expires_when_applying_operation() -> None: - _, uow = initialize({Components.SOURCE: {"1"}}) - with uow: - link = uow.link - link.pull(create_identifiers("1")) - with pytest.raises(RuntimeError, match="expired entity"): - link.pull(create_identifiers("1")) diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index 5254e1ec..7e0f3da4 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -1,6 +1,5 @@ from __future__ import annotations -from dataclasses import replace from typing import Iterable import pytest @@ -52,25 +51,24 @@ def test_invalid_transitions_returns_unchanged_entity( entity = next(entity for entity in link if entity.identifier == identifier) for operation in operations: result = events.InvalidOperationRequested(operation, identifier, state) - assert entity.apply(operation) == replace(entity, events=(result,)) + entity.apply(operation) + assert entity.events.pop() == result def test_start_pulling_idle_entity_returns_correct_entity() -> None: link = create_link(create_assignments({Components.SOURCE: {"1"}})) entity = next(iter(link)) - assert entity.apply(Operations.START_PULL) == replace( - entity, - state=states.Activated, - current_process=Processes.PULL, - events=( - events.EntityStateChanged( - Operations.START_PULL, - entity.identifier, - Transition(states.Idle, states.Activated), - Commands.START_PULL_PROCESS, - ), - ), - ) + entity.apply(Operations.START_PULL) + assert entity.state is states.Activated + assert entity.current_process is Processes.PULL + assert list(entity.events) == [ + events.EntityStateChanged( + Operations.START_PULL, + entity.identifier, + Transition(states.Idle, states.Activated), + Commands.START_PULL_PROCESS, + ) + ] @pytest.mark.parametrize( @@ -98,7 +96,9 @@ def test_processing_activated_entity_returns_correct_entity( entity.events.append( events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) - assert entity.apply(Operations.PROCESS) == replace(entity, state=new_state, current_process=new_process) + entity.apply(Operations.PROCESS) + assert entity.state == new_state + assert entity.current_process == new_process @pytest.mark.parametrize( @@ -123,12 +123,13 @@ def test_processing_received_entity_returns_correct_entity( tainted_identifiers=tainted_identifiers, ) entity = next(iter(link)) - expected_events = ( + expected_events = [ events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), - ) - assert entity.apply(Operations.PROCESS) == replace( - entity, state=new_state, current_process=new_process, events=expected_events - ) + ] + entity.apply(Operations.PROCESS) + assert entity.state == new_state + assert entity.current_process == new_process + assert list(entity.events) == expected_events def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: @@ -137,17 +138,18 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: ) entity = next(iter(link)) transition = Transition(states.Pulled, states.Received) - expected_events = ( + expected_events = [ events.EntityStateChanged( Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS, ), - ) - assert entity.apply(Operations.START_DELETE) == replace( - entity, state=transition.new, current_process=Processes.DELETE, events=expected_events - ) + ] + entity.apply(Operations.START_DELETE) + assert entity.state == transition.new + assert entity.current_process == Processes.DELETE + assert list(entity.events) == expected_events def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: @@ -157,11 +159,12 @@ def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: ) entity = next(iter(link)) transition = Transition(states.Tainted, states.Received) - expected_events = ( + expected_events = [ events.EntityStateChanged( Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS ), - ) - assert entity.apply(Operations.START_DELETE) == replace( - entity, state=transition.new, current_process=Processes.DELETE, events=expected_events - ) + ] + entity.apply(Operations.START_DELETE) + assert entity.state == transition.new + assert entity.current_process == Processes.DELETE + assert list(entity.events) == expected_events From 1452fed654a73b05964b7f25c49bb4bf122e6611 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:26:32 +0100 Subject: [PATCH 51/59] Collect new events in uow --- link/service/uow.py | 17 +++++++- tests/integration/test_uow.py | 76 ++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/link/service/uow.py b/link/service/uow.py index 8454c8af..4efd9f78 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -4,7 +4,7 @@ from abc import ABC from collections import deque from types import TracebackType -from typing import Callable, Iterable, Protocol +from typing import Callable, Iterable, Iterator, Protocol from link.domain import events from link.domain.custom_types import Identifier @@ -30,6 +30,8 @@ def __init__(self, gateway: LinkGateway) -> None: self._gateway = gateway self._link: Link | None = None self._updates: deque[EntityStateChanged] = deque() + self._events: deque[events.Event] = deque() + self._seen: list[Entity] = [] def __enter__(self) -> UnitOfWork: """Enter the context in which updates to entities can be made.""" @@ -47,6 +49,8 @@ def augmented(operation: Operations) -> None: assert hasattr(entity, "_is_expired") if entity._is_expired: raise RuntimeError("Can not apply operation to expired entity.") + if entity not in self._seen: + self._seen.append(entity) current_state = entity.state original(operation) new_state = entity.state @@ -83,6 +87,9 @@ def commit(self) -> None: raise RuntimeError("Not available outside of context") while self._updates: self._gateway.apply([self._updates.popleft()]) + for entity in self._seen: + while entity.events: + self._events.append(entity.events.popleft()) self.rollback() def rollback(self) -> None: @@ -92,3 +99,11 @@ def rollback(self) -> None: for entity in self._link: setattr(entity, "_is_expired", True) self._updates.clear() + self._seen.clear() + + def collect_new_events(self) -> Iterator[events.Event]: + """Collect new events from entities.""" + if self._link is not None: + raise RuntimeError("New events can not be collected when inside context") + while self._events: + yield self._events.popleft() diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 439fec55..12473f25 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -4,7 +4,8 @@ import pytest -from link.domain.state import Components, Operations, states +from link.domain import events +from link.domain.state import Commands, Components, Operations, Transition, states from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifier, create_identifiers @@ -118,3 +119,76 @@ def test_link_expires_when_exiting_context() -> None: link = uow.link with pytest.raises(RuntimeError, match="expired entity"): link.pull(create_identifiers("1")) + + +def test_correct_events_are_collected() -> None: + _, uow = initialize({Components.SOURCE: {"1", "2"}, Components.OUTBOUND: {"2"}, Components.LOCAL: {"2"}}) + with uow: + uow.link.pull(create_identifiers("1")) + uow.link.delete(create_identifiers("2")) + uow.commit() + expected = [ + events.EntityStateChanged( + Operations.START_PULL, + create_identifier("1"), + Transition(states.Idle, states.Activated), + Commands.START_PULL_PROCESS, + ), + events.EntityStateChanged( + Operations.PROCESS, + create_identifier("1"), + Transition(states.Activated, states.Received), + Commands.ADD_TO_LOCAL, + ), + events.EntityStateChanged( + Operations.PROCESS, + create_identifier("1"), + Transition(states.Received, states.Pulled), + Commands.FINISH_PULL_PROCESS, + ), + events.EntityStateChanged( + Operations.START_DELETE, + create_identifier("2"), + Transition(states.Pulled, states.Received), + Commands.START_DELETE_PROCESS, + ), + events.EntityStateChanged( + Operations.PROCESS, + create_identifier("2"), + Transition(states.Received, states.Activated), + Commands.REMOVE_FROM_LOCAL, + ), + events.EntityStateChanged( + Operations.PROCESS, + create_identifier("2"), + Transition(states.Activated, states.Idle), + Commands.FINISH_DELETE_PROCESS, + ), + ] + actual = list(uow.collect_new_events()) + assert actual == expected + + +def test_unit_must_be_committed_to_collect_events() -> None: + _, uow = initialize({Components.SOURCE: {"1"}}) + with uow: + uow.link.pull(create_identifiers("1")) + assert list(uow.collect_new_events()) == [] + + +def test_events_can_only_be_collected_once() -> None: + _, uow = initialize({Components.SOURCE: {"1"}}) + with uow: + uow.link.pull(create_identifiers("1")) + uow.commit() + list(uow.collect_new_events()) + assert list(uow.collect_new_events()) == [] + + +def test_events_can_only_be_collected_outside_of_context() -> None: + _, uow = initialize({Components.SOURCE: {"1"}}) + with uow: + uow.link.pull(create_identifiers("1")) + uow.commit() + with pytest.raises(RuntimeError, match="inside context"): + list(uow.collect_new_events()) From aa0f4bf0ed9ae9c00bea8a3c2500c15ceae778ce Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:27:59 +0100 Subject: [PATCH 52/59] Add collected events to messagebus --- link/service/messagebus.py | 1 + 1 file changed, 1 insertion(+) diff --git a/link/service/messagebus.py b/link/service/messagebus.py index b0355730..3e981805 100644 --- a/link/service/messagebus.py +++ b/link/service/messagebus.py @@ -62,6 +62,7 @@ def handle(self, message: Message) -> None: self._handle_event(message) else: raise TypeError(f"Unknown message type {type(message)!r}") + self._queue.extend(self._uow.collect_new_events()) def _handle_command(self, command: Command) -> None: handler = self._command_handlers[type(command)] From 3b22f4da79ea40e97bd5e9d51c70384c507f4ec8 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:42:19 +0100 Subject: [PATCH 53/59] Remove unused event --- link/domain/events.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/link/domain/events.py b/link/domain/events.py index 181141d6..c9d61dd4 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -38,22 +38,6 @@ class EntityStateChanged(EntityOperationApplied): command: Commands -@dataclass(frozen=True) -class LinkStateChanged(Event): - """The state of a link changed during the application of an operation.""" - - operation: Operations - requested: frozenset[Identifier] - updates: frozenset[EntityStateChanged] - errors: frozenset[InvalidOperationRequested] - - def __post_init__(self) -> None: - """Validate the event.""" - assert all( - result.operation is self.operation for result in (self.updates | self.errors) - ), "Not all events have same operation." - - @dataclass(frozen=True) class IdleEntitiesListed(Event): """Idle entities in a link have been listed.""" From 50450e033d83acdcea6c7e52072995e22c3748b5 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:17:11 +0100 Subject: [PATCH 54/59] Use message bus --- link/adapters/controller.py | 18 ++++++++---------- link/infrastructure/link.py | 21 ++++++++++++++------- link/service/services.py | 9 --------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/link/adapters/controller.py b/link/adapters/controller.py index 70047000..f3bab2a4 100644 --- a/link/adapters/controller.py +++ b/link/adapters/controller.py @@ -1,10 +1,10 @@ """Contains code controlling the execution of use-cases.""" from __future__ import annotations -from typing import Callable, Iterable, Mapping +from typing import Iterable from link.domain import commands -from link.service.services import Services +from link.service.messagebus import MessageBus from .custom_types import PrimaryKey from .identification import IdentificationTranslator @@ -15,23 +15,21 @@ class DJController: def __init__( self, - handlers: Mapping[Services, Callable[[commands.Command], None]], + message_bus: MessageBus, translator: IdentificationTranslator, ) -> None: """Initialize the translator.""" - self.__handlers = handlers - self.__translator = translator + self._message_bus = message_bus + self._translator = translator def pull(self, primary_keys: Iterable[PrimaryKey]) -> None: """Execute the pull use-case.""" - self.__handlers[Services.PULL](commands.PullEntities(frozenset(self.__translator.to_identifiers(primary_keys)))) + self._message_bus.handle(commands.PullEntities(frozenset(self._translator.to_identifiers(primary_keys)))) def delete(self, primary_keys: Iterable[PrimaryKey]) -> None: """Execute the delete use-case.""" - self.__handlers[Services.DELETE]( - commands.DeleteEntities(frozenset(self.__translator.to_identifiers(primary_keys))) - ) + self._message_bus.handle(commands.DeleteEntities(frozenset(self._translator.to_identifiers(primary_keys)))) def list_idle_entities(self) -> None: """Execute the use-case that lists idle entities.""" - self.__handlers[Services.LIST_IDLE_ENTITIES](commands.ListIdleEntities()) + self._message_bus.handle(commands.ListIdleEntities()) diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index 9509250a..3c6312d1 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -10,7 +10,9 @@ from link.adapters.gateway import DJLinkGateway from link.adapters.identification import IdentificationTranslator from link.adapters.present import create_idle_entities_updater -from link.service.services import Services, delete, list_idle_entities, pull +from link.domain import commands, events +from link.service.messagebus import CommandHandlers, EventHandlers, MessageBus +from link.service.services import delete, list_idle_entities, pull from link.service.uow import UnitOfWork from . import DJConfiguration, create_tables @@ -44,12 +46,17 @@ def inner(obj: type) -> Any: uow = UnitOfWork(gateway) source_restriction: IterationCallbackList[PrimaryKey] = IterationCallbackList() idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) - handlers = { - Services.PULL: partial(pull, uow=uow), - Services.DELETE: partial(delete, uow=uow), - Services.LIST_IDLE_ENTITIES: partial(list_idle_entities, uow=uow, output_port=idle_entities_updater), - } - controller = DJController(handlers, translator) + command_handlers: CommandHandlers = {} + command_handlers[commands.PullEntities] = partial(pull, uow=uow) + command_handlers[commands.DeleteEntities] = partial(delete, uow=uow) + command_handlers[commands.ListIdleEntities] = partial( + list_idle_entities, uow=uow, output_port=idle_entities_updater + ) + event_handlers: EventHandlers = {} + event_handlers[events.EntityStateChanged] = [lambda event: None] + event_handlers[events.InvalidOperationRequested] = [lambda event: None] + bus = MessageBus(uow, command_handlers, event_handlers) + controller = DJController(bus, translator) source_restriction.callback = controller.list_idle_entities return create_local_endpoint(controller, tables, source_restriction) diff --git a/link/service/services.py b/link/service/services.py index b2be348a..0ca11e40 100644 --- a/link/service/services.py +++ b/link/service/services.py @@ -2,7 +2,6 @@ from __future__ import annotations from collections.abc import Callable -from enum import Enum, auto from link.domain import commands, events @@ -33,11 +32,3 @@ def list_idle_entities( with uow: idle = uow.link.list_idle_entities() output_port(events.IdleEntitiesListed(idle)) - - -class Services(Enum): - """Names for all available services.""" - - PULL = auto() - DELETE = auto() - LIST_IDLE_ENTITIES = auto() From 6e9b04f86c69b08184e64c5665948d307ab0378c Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:21:35 +0100 Subject: [PATCH 55/59] Clean up uow imports --- link/service/uow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/link/service/uow.py b/link/service/uow.py index 4efd9f78..fd10d52d 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -8,7 +8,6 @@ from link.domain import events from link.domain.custom_types import Identifier -from link.domain.events import EntityStateChanged from link.domain.link import Link from link.domain.state import TRANSITION_MAP, Entity, Operations, Transition @@ -29,7 +28,7 @@ def __init__(self, gateway: LinkGateway) -> None: """Initialize the unit of work.""" self._gateway = gateway self._link: Link | None = None - self._updates: deque[EntityStateChanged] = deque() + self._updates: deque[events.EntityStateChanged] = deque() self._events: deque[events.Event] = deque() self._seen: list[Entity] = [] From bd531b9ae5521a1f96c35d1ec1cd1a6d99e1304a Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:24:35 +0100 Subject: [PATCH 56/59] Rename services module to handlers --- link/infrastructure/link.py | 2 +- link/service/{services.py => handlers.py} | 2 +- tests/integration/test_services.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename link/service/{services.py => handlers.py} (93%) diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index 3c6312d1..e9b9f5db 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -11,8 +11,8 @@ from link.adapters.identification import IdentificationTranslator from link.adapters.present import create_idle_entities_updater from link.domain import commands, events +from link.service.handlers import delete, list_idle_entities, pull from link.service.messagebus import CommandHandlers, EventHandlers, MessageBus -from link.service.services import delete, list_idle_entities, pull from link.service.uow import UnitOfWork from . import DJConfiguration, create_tables diff --git a/link/service/services.py b/link/service/handlers.py similarity index 93% rename from link/service/services.py rename to link/service/handlers.py index 0ca11e40..d3bdfe53 100644 --- a/link/service/services.py +++ b/link/service/handlers.py @@ -1,4 +1,4 @@ -"""Contains all the services.""" +"""Contains code handling domain commands and events.""" from __future__ import annotations from collections.abc import Callable diff --git a/tests/integration/test_services.py b/tests/integration/test_services.py index 72521a7e..ee1630b6 100644 --- a/tests/integration/test_services.py +++ b/tests/integration/test_services.py @@ -7,7 +7,7 @@ from link.domain import commands, events from link.domain.state import Components, Processes, State, states -from link.service.services import delete, list_idle_entities, pull +from link.service.handlers import delete, list_idle_entities, pull from link.service.uow import UnitOfWork from tests.assignments import create_assignments, create_identifiers From 478f175ede7d2e7df4e2110bc4bdffdef46fd410 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:30:14 +0100 Subject: [PATCH 57/59] Rename some events --- link/adapters/gateway.py | 4 ++-- link/domain/events.py | 6 +++--- link/domain/state.py | 6 +++--- link/infrastructure/link.py | 2 +- link/service/gateway.py | 2 +- link/service/uow.py | 4 ++-- tests/integration/gateway.py | 2 +- tests/integration/test_datajoint_persistence.py | 2 +- tests/integration/test_uow.py | 12 ++++++------ tests/unit/entities/test_state.py | 12 +++++------- 10 files changed, 25 insertions(+), 27 deletions(-) diff --git a/link/adapters/gateway.py b/link/adapters/gateway.py index c05cf577..8e678105 100644 --- a/link/adapters/gateway.py +++ b/link/adapters/gateway.py @@ -52,10 +52,10 @@ def translate_tainted_primary_keys(primary_keys: Iterable[PrimaryKey]) -> set[Id tainted_identifiers=translate_tainted_primary_keys(self.facade.get_tainted_primary_keys()), ) - def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: + def apply(self, updates: Iterable[events.StateChanged]) -> None: """Apply updates to the persistent data representing the link.""" - def keyfunc(update: events.EntityStateChanged) -> int: + def keyfunc(update: events.StateChanged) -> int: assert update.command is not None return update.command.value diff --git a/link/domain/events.py b/link/domain/events.py index c9d61dd4..3c517315 100644 --- a/link/domain/events.py +++ b/link/domain/events.py @@ -16,7 +16,7 @@ class Event: @dataclass(frozen=True) -class EntityOperationApplied(Event): +class OperationApplied(Event): """An operation was applied to an entity.""" operation: Operations @@ -24,14 +24,14 @@ class EntityOperationApplied(Event): @dataclass(frozen=True) -class InvalidOperationRequested(EntityOperationApplied): +class InvalidOperationRequested(OperationApplied): """An operation that is invalid given the entities current state was requested.""" state: type[State] @dataclass(frozen=True) -class EntityStateChanged(EntityOperationApplied): +class StateChanged(OperationApplied): """The state of an entity changed during the application of an operation.""" transition: Transition diff --git a/link/domain/state.py b/link/domain/state.py index 272cd68f..52f3d680 100644 --- a/link/domain/state.py +++ b/link/domain/state.py @@ -7,7 +7,7 @@ from functools import partial from .custom_types import Identifier -from .events import EntityOperationApplied, EntityStateChanged, InvalidOperationRequested +from .events import InvalidOperationRequested, OperationApplied, StateChanged class State: @@ -42,7 +42,7 @@ def _transition_entity( entity.state = transition.new entity.current_process = new_process entity.events.append( - EntityStateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), + StateChanged(operation, entity.identifier, transition, TRANSITION_MAP[transition]), ) @@ -269,7 +269,7 @@ class Entity: state: type[State] current_process: Processes is_tainted: bool - events: deque[EntityOperationApplied] + events: deque[OperationApplied] def pull(self) -> None: """Pull the entity.""" diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index e9b9f5db..e407a3cc 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -53,7 +53,7 @@ def inner(obj: type) -> Any: list_idle_entities, uow=uow, output_port=idle_entities_updater ) event_handlers: EventHandlers = {} - event_handlers[events.EntityStateChanged] = [lambda event: None] + event_handlers[events.StateChanged] = [lambda event: None] event_handlers[events.InvalidOperationRequested] = [lambda event: None] bus = MessageBus(uow, command_handlers, event_handlers) controller = DJController(bus, translator) diff --git a/link/service/gateway.py b/link/service/gateway.py index fdc0552f..f7b3d375 100644 --- a/link/service/gateway.py +++ b/link/service/gateway.py @@ -16,5 +16,5 @@ def create_link(self) -> Link: """Create a link from the persistent data.""" @abstractmethod - def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: + def apply(self, updates: Iterable[events.StateChanged]) -> None: """Apply updates to the link's persistent data.""" diff --git a/link/service/uow.py b/link/service/uow.py index fd10d52d..3235d467 100644 --- a/link/service/uow.py +++ b/link/service/uow.py @@ -28,7 +28,7 @@ def __init__(self, gateway: LinkGateway) -> None: """Initialize the unit of work.""" self._gateway = gateway self._link: Link | None = None - self._updates: deque[events.EntityStateChanged] = deque() + self._updates: deque[events.StateChanged] = deque() self._events: deque[events.Event] = deque() self._seen: list[Entity] = [] @@ -57,7 +57,7 @@ def augmented(operation: Operations) -> None: return transition = Transition(current_state, new_state) command = TRANSITION_MAP[transition] - self._updates.append(events.EntityStateChanged(operation, entity.identifier, transition, command)) + self._updates.append(events.StateChanged(operation, entity.identifier, transition, command)) return augmented diff --git a/tests/integration/gateway.py b/tests/integration/gateway.py index 94ce53f1..075c6220 100644 --- a/tests/integration/gateway.py +++ b/tests/integration/gateway.py @@ -28,7 +28,7 @@ def __init__( def create_link(self) -> Link: return create_link(self.assignments, tainted_identifiers=self.tainted_identifiers, processes=self.processes) - def apply(self, updates: Iterable[events.EntityStateChanged]) -> None: + def apply(self, updates: Iterable[events.StateChanged]) -> None: for update in updates: if update.command is Commands.START_PULL_PROCESS: self.processes[Processes.PULL].add(update.identifier) diff --git a/tests/integration/test_datajoint_persistence.py b/tests/integration/test_datajoint_persistence.py index caa5bd0e..9b1aaff5 100644 --- a/tests/integration/test_datajoint_persistence.py +++ b/tests/integration/test_datajoint_persistence.py @@ -349,7 +349,7 @@ def apply_update(gateway: DJLinkGateway, operation: Operations, requested: Itera entity.apply(operation) while entity.events: event = entity.events.popleft() - if not isinstance(event, events.EntityStateChanged): + if not isinstance(event, events.StateChanged): continue gateway.apply([event]) diff --git a/tests/integration/test_uow.py b/tests/integration/test_uow.py index 12473f25..9aabc0fd 100644 --- a/tests/integration/test_uow.py +++ b/tests/integration/test_uow.py @@ -128,37 +128,37 @@ def test_correct_events_are_collected() -> None: uow.link.delete(create_identifiers("2")) uow.commit() expected = [ - events.EntityStateChanged( + events.StateChanged( Operations.START_PULL, create_identifier("1"), Transition(states.Idle, states.Activated), Commands.START_PULL_PROCESS, ), - events.EntityStateChanged( + events.StateChanged( Operations.PROCESS, create_identifier("1"), Transition(states.Activated, states.Received), Commands.ADD_TO_LOCAL, ), - events.EntityStateChanged( + events.StateChanged( Operations.PROCESS, create_identifier("1"), Transition(states.Received, states.Pulled), Commands.FINISH_PULL_PROCESS, ), - events.EntityStateChanged( + events.StateChanged( Operations.START_DELETE, create_identifier("2"), Transition(states.Pulled, states.Received), Commands.START_DELETE_PROCESS, ), - events.EntityStateChanged( + events.StateChanged( Operations.PROCESS, create_identifier("2"), Transition(states.Received, states.Activated), Commands.REMOVE_FROM_LOCAL, ), - events.EntityStateChanged( + events.StateChanged( Operations.PROCESS, create_identifier("2"), Transition(states.Activated, states.Idle), diff --git a/tests/unit/entities/test_state.py b/tests/unit/entities/test_state.py index 7e0f3da4..a097e303 100644 --- a/tests/unit/entities/test_state.py +++ b/tests/unit/entities/test_state.py @@ -62,7 +62,7 @@ def test_start_pulling_idle_entity_returns_correct_entity() -> None: assert entity.state is states.Activated assert entity.current_process is Processes.PULL assert list(entity.events) == [ - events.EntityStateChanged( + events.StateChanged( Operations.START_PULL, entity.identifier, Transition(states.Idle, states.Activated), @@ -94,7 +94,7 @@ def test_processing_activated_entity_returns_correct_entity( ) entity = next(iter(link)) entity.events.append( - events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + events.StateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ) entity.apply(Operations.PROCESS) assert entity.state == new_state @@ -124,7 +124,7 @@ def test_processing_received_entity_returns_correct_entity( ) entity = next(iter(link)) expected_events = [ - events.EntityStateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), + events.StateChanged(Operations.PROCESS, entity.identifier, Transition(entity.state, new_state), command), ] entity.apply(Operations.PROCESS) assert entity.state == new_state @@ -139,7 +139,7 @@ def test_starting_delete_on_pulled_entity_returns_correct_entity() -> None: entity = next(iter(link)) transition = Transition(states.Pulled, states.Received) expected_events = [ - events.EntityStateChanged( + events.StateChanged( Operations.START_DELETE, entity.identifier, transition, @@ -160,9 +160,7 @@ def test_starting_delete_on_tainted_entity_returns_correct_commands() -> None: entity = next(iter(link)) transition = Transition(states.Tainted, states.Received) expected_events = [ - events.EntityStateChanged( - Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS - ), + events.StateChanged(Operations.START_DELETE, entity.identifier, transition, Commands.START_DELETE_PROCESS), ] entity.apply(Operations.START_DELETE) assert entity.state == transition.new From bee5f0abfd24ce7201f6945e34de8241d548aab4 Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:11:23 +0100 Subject: [PATCH 58/59] Log entity state changes --- link/adapters/present.py | 20 ++++++++++++++++++++ link/infrastructure/link.py | 10 +++++++--- link/service/handlers.py | 5 +++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/link/adapters/present.py b/link/adapters/present.py index 09c13bfc..fb20f570 100644 --- a/link/adapters/present.py +++ b/link/adapters/present.py @@ -18,3 +18,23 @@ def update_idle_entities(response: events.IdleEntitiesListed) -> None: update(translator.to_primary_key(identifier) for identifier in response.identifiers) return update_idle_entities + + +def create_state_change_logger( + translator: IdentificationTranslator, log: Callable[[str], None] +) -> Callable[[events.StateChanged], None]: + """Create a logger that logs state changes of entities.""" + + def log_state_change(state_change: events.StateChanged) -> None: + context = { + "identifier": translator.to_primary_key(state_change.identifier), + "operation": state_change.operation.name, + "transition": { + "old": state_change.transition.current.__name__, + "new": state_change.transition.new.__name__, + }, + "command": state_change.command.name, + } + log(f"Entity state changed {context}") + + return log_state_change diff --git a/link/infrastructure/link.py b/link/infrastructure/link.py index e407a3cc..06f0f77d 100644 --- a/link/infrastructure/link.py +++ b/link/infrastructure/link.py @@ -1,6 +1,7 @@ """Contains the link decorator that is used by the user to establish a link.""" from __future__ import annotations +import logging from collections.abc import Callable from functools import partial from typing import Any, Mapping, Optional @@ -9,9 +10,9 @@ from link.adapters.custom_types import PrimaryKey from link.adapters.gateway import DJLinkGateway from link.adapters.identification import IdentificationTranslator -from link.adapters.present import create_idle_entities_updater +from link.adapters.present import create_idle_entities_updater, create_state_change_logger from link.domain import commands, events -from link.service.handlers import delete, list_idle_entities, pull +from link.service.handlers import delete, list_idle_entities, log_state_change, pull from link.service.messagebus import CommandHandlers, EventHandlers, MessageBus from link.service.uow import UnitOfWork @@ -46,6 +47,7 @@ def inner(obj: type) -> Any: uow = UnitOfWork(gateway) source_restriction: IterationCallbackList[PrimaryKey] = IterationCallbackList() idle_entities_updater = create_idle_entities_updater(translator, create_content_replacer(source_restriction)) + logger = logging.getLogger(obj.__name__) command_handlers: CommandHandlers = {} command_handlers[commands.PullEntities] = partial(pull, uow=uow) command_handlers[commands.DeleteEntities] = partial(delete, uow=uow) @@ -53,7 +55,9 @@ def inner(obj: type) -> Any: list_idle_entities, uow=uow, output_port=idle_entities_updater ) event_handlers: EventHandlers = {} - event_handlers[events.StateChanged] = [lambda event: None] + event_handlers[events.StateChanged] = [ + partial(log_state_change, log=create_state_change_logger(translator, logger.info)) + ] event_handlers[events.InvalidOperationRequested] = [lambda event: None] bus = MessageBus(uow, command_handlers, event_handlers) controller = DJController(bus, translator) diff --git a/link/service/handlers.py b/link/service/handlers.py index d3bdfe53..625c9c7e 100644 --- a/link/service/handlers.py +++ b/link/service/handlers.py @@ -32,3 +32,8 @@ def list_idle_entities( with uow: idle = uow.link.list_idle_entities() output_port(events.IdleEntitiesListed(idle)) + + +def log_state_change(event: events.StateChanged, log: Callable[[events.StateChanged], None]) -> None: + """Log the state change of an entity.""" + log(event) From eed964d3cb4666056db25014a037884ecb40909e Mon Sep 17 00:00:00 2001 From: Christoph Blessing <33834216+cblessing24@users.noreply.github.com> Date: Mon, 6 Nov 2023 18:08:02 +0100 Subject: [PATCH 59/59] Update minio image --- tests/functional/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index cc2dc7b6..b01b46f9 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -20,7 +20,7 @@ SCOPE = os.environ.get("SCOPE", "session") REMOVE = True DATABASE_IMAGE = "cblessing24/mariadb:11.1" -MINIO_IMAGE = "minio/minio:latest" +MINIO_IMAGE = "minio/minio:RELEASE.2023-11-01T18-37-25Z" DATABASE_ROOT_PASSWORD = "root" @@ -172,10 +172,10 @@ def get_runner_kwargs(docker_client, spec): ) elif isinstance(spec, MinIOSpec): processed_container_config = dict( - environment=dict(MINIO_ACCESS_KEY=spec.config.access_key, MINIO_SECRET_KEY=spec.config.secret_key), + environment=dict(MINIO_ROOT_USER=spec.config.access_key, MINIO_ROOT_PASSWORD=spec.config.secret_key), command=["server", "/data"], healthcheck=dict( - test=["CMD", "curl", "-f", "127.0.0.1:9000/minio/health/ready"], + test=["CMD", "mc", "ready", "local"], start_period=int(spec.container.health_check.start_period_seconds * 1e9), # nanoseconds interval=int(spec.container.health_check.interval_seconds * 1e9), # nanoseconds retries=spec.container.health_check.max_retries,