From 61d5a5792b6f1d7b8d7dc2dec0706d16390d20ca Mon Sep 17 00:00:00 2001 From: antazoey Date: Mon, 6 Jan 2025 15:00:29 -0600 Subject: [PATCH] feat: support individual isolation-scope configs (#2458) Co-authored-by: antazoey --- docs/userguides/testing.md | 14 ++++++++- src/ape/pytest/config.py | 11 ++++++- src/ape/pytest/fixtures.py | 22 ++++++++------ src/ape_test/config.py | 48 ++++++++++++++++++++++++++++++- tests/functional/test_fixtures.py | 11 +++---- tests/functional/test_test.py | 48 +++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 19 deletions(-) diff --git a/docs/userguides/testing.md b/docs/userguides/testing.md index 81e7e9c198..2be9404b5a 100644 --- a/docs/userguides/testing.md +++ b/docs/userguides/testing.md @@ -227,7 +227,7 @@ After each test completes, the chain reverts to that snapshot from the beginning By default, every `pytest` fixture is `function` scoped, meaning it will be replayed each time it is requested (no result-caching). For example, if you deploy a contract in a function-scoped fixture, it will be re-deployed each time the fixture gets used in your tests. -To only deploy once, you can use different scopes, such as `"session"`, `"package"`, `"module"`, or `"class"`, and you **must** use these fixtures right away, either via `autouse=True` or using them in the first collected tests. +To only deploy once, you can use different scopes, such as `"session"`, `"package"`, `"module"`, or `"class"`, and you **should** use these fixtures right away, either via `autouse=True` or using them in the first collected tests. Otherwise, higher-scoped fixtures that arrive late in a Pytest session will cause the snapshotting system to have to rebase itself, which can be costly. For example, if you define a session scoped fixture that deploys a contract and makes transactions, the state changes from those transactions remain in subsequent tests, whether those tests use that fixture or not. However, if a new fixture of a session scope comes into play after module, package, or class scoped snapshots have already been taken, those lower-scoped fixtures are now invalid and have to re-run after the session fixture to ensure the session fixture remains in the session-snapshot. @@ -285,6 +285,18 @@ def token_addresses(request): return tokens[request].address ``` +You can also disable isolation for individual scopes using Ape's config. +For example, the below config will disable isolation across all high-level scopes but maintain isolation for the function-scope. +This is useful if you want your individual tests to be isolated but not any session/module scoped fixtures. + +```toml +[tool.ape.test.isolation] +enable_session = false +enable_package = false +enable_module = false +enable_class = false +``` + ## Ape testing commands ```bash diff --git a/src/ape/pytest/config.py b/src/ape/pytest/config.py index 591c7c235e..f27bc37449 100644 --- a/src/ape/pytest/config.py +++ b/src/ape/pytest/config.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from _pytest.config import Config as PytestConfig + from ape.pytest.utils import Scope from ape.types.trace import ContractFunctionPath @@ -21,7 +22,7 @@ def _get_config_exclusions(config) -> list["ContractFunctionPath"]: class ConfigWrapper(ManagerAccessMixin): """ A class aggregating settings choices from both the pytest command line - as well as the ``ape-config.yaml`` file. Also serves as a wrapper around the + and the ``ape-config.yaml`` file. Also serves as a wrapper around the Pytest config object for ease-of-use and code-sharing. """ @@ -118,3 +119,11 @@ def get_pytest_plugin(self, name: str) -> Optional[Any]: return self.pytest_config.pluginmanager.get_plugin(name) return None + + def get_isolation(self, scope: "Scope"): + if self.pytest_config.getoption("disable_isolation"): + # Was disabled via command-line. + return False + + # Check Ape's config. + return self.ape_test_config.get_isolation(scope) diff --git a/src/ape/pytest/fixtures.py b/src/ape/pytest/fixtures.py index ad8f2813f5..93db3608e1 100644 --- a/src/ape/pytest/fixtures.py +++ b/src/ape/pytest/fixtures.py @@ -213,13 +213,13 @@ def _get_rebase(self, scope: Scope) -> Optional[FixtureRebase]: invalids = defaultdict(list) for next_snapshot in self.isolation_manager.next_snapshots(scope): if next_snapshot.identifier is None: - # Thankfully, we haven't reached this scope yet. - # In this case, things are running in a performant order. + # Thankfully, we haven't reached this scope yet (or it is disabled). + # In this case, things are running in a correct/performant order. continue if scope_to_revert is None: # Revert to the closest scope to use. For example, a new - # session comes in but we have already calculated a module + # session comes in, but we have already calculated a module # and a class, revert to pre-module and invalidate the module # and class fixtures. scope_to_revert = next_snapshot.scope @@ -604,16 +604,11 @@ def isolation(self, scope: Scope) -> Iterator[None]: else: yield - # NOTE: self._supported may have gotten set to False - # someplace else _after_ snapshotting succeeded. - if not self.supported: - return - self.restore(scope) def set_snapshot(self, scope: Scope): # Also can be used to re-set snapshot. - if not self.supported: + if not self.supported or not self.config_wrapper.get_isolation(scope): return try: @@ -640,6 +635,15 @@ def take_snapshot(self) -> Optional["SnapshotID"]: @allow_disconnected def restore(self, scope: Scope): + # NOTE: self._supported may have gotten set to False + # someplace else _after_ snapshotting succeeded. + if not self.supported or not self.config_wrapper.get_isolation(scope): + return + + self.restore_snapshot(scope) + + @allow_disconnected + def restore_snapshot(self, scope: Scope): snapshot_id = self.snapshots.get_snapshot_id(scope) if snapshot_id is None: return diff --git a/src/ape_test/config.py b/src/ape_test/config.py index 4963906570..3ce2609dfe 100644 --- a/src/ape_test/config.py +++ b/src/ape_test/config.py @@ -1,4 +1,4 @@ -from typing import NewType, Optional, Union +from typing import TYPE_CHECKING, NewType, Optional, Union from pydantic import NonNegativeInt, field_validator @@ -12,6 +12,9 @@ DEFAULT_TEST_MNEMONIC, ) +if TYPE_CHECKING: + from ape.pytest.utils import Scope + class EthTesterProviderConfig(PluginConfig): chain_id: int = DEFAULT_TEST_CHAIN_ID @@ -117,6 +120,36 @@ class CoverageConfig(PluginConfig): """ +class IsolationConfig(PluginConfig): + enable_session: bool = True + """ + Set to ``False`` to disable session isolation. + """ + + enable_package: bool = True + """ + Set to ``False`` to disable package isolation. + """ + + enable_module: bool = True + """ + Set to ``False`` to disable module isolation. + """ + + enable_class: bool = True + """ + Set to ``False`` to disable class isolation. + """ + + enable_function: bool = True + """ + Set to ``False`` to disable function isolation. + """ + + def get_isolation(self, scope: "Scope") -> bool: + return getattr(self, f"enable_{scope.name.lower()}") + + class ApeTestConfig(PluginConfig): balance: int = DEFAULT_TEST_ACCOUNT_BALANCE """ @@ -170,6 +203,12 @@ class ApeTestConfig(PluginConfig): useful for debugging the framework itself. """ + isolation: Union[bool, IsolationConfig] = True + """ + Configure which scope-specific isolation to enable. Set to + ``False`` to disable all and ``True`` (default) to disable all. + """ + @field_validator("balance", mode="before") @classmethod def validate_balance(cls, value): @@ -178,3 +217,10 @@ def validate_balance(cls, value): if isinstance(value, int) else ManagerAccessMixin.conversion_manager.convert(value, int) ) + + def get_isolation(self, scope: "Scope") -> bool: + return ( + self.isolation + if isinstance(self.isolation, bool) + else self.isolation.get_isolation(scope) + ) diff --git a/tests/functional/test_fixtures.py b/tests/functional/test_fixtures.py index e2669b29b9..3b223f3b15 100644 --- a/tests/functional/test_fixtures.py +++ b/tests/functional/test_fixtures.py @@ -110,18 +110,15 @@ def restore(self, scope: Scope): assert isolation_manager.restore_called_with == [Scope.FUNCTION] -def test_isolation_when_snapshot_fails_avoids_restore(fixtures): +def test_isolation_when_snapshot_fails_avoids_restore(mocker, fixtures): class IsolationManagerFailingAtSnapshotting(IsolationManager): take_called = False - restore_called = False def take_snapshot(self) -> Optional["SnapshotID"]: self.take_called = True raise NotImplementedError() - def restore(self, scope: Scope): - self.restore_called = True - + restore_spy = mocker.spy(IsolationManagerFailingAtSnapshotting, "restore_snapshot") isolation_manager = IsolationManagerFailingAtSnapshotting( fixtures.isolation_manager.config_wrapper, fixtures.isolation_manager.receipt_capture, @@ -129,10 +126,10 @@ def restore(self, scope: Scope): isolation_context = isolation_manager.isolation(Scope.FUNCTION) next(isolation_context) # Enter. assert isolation_manager.take_called - assert not isolation_manager.restore_called + assert not restore_spy.call_count next(isolation_context, None) # Exit. # It doesn't even try! - assert not isolation_manager.restore_called + assert not restore_spy.call_count def test_isolation_restore_fails_avoids_snapshot_next_time(fixtures): diff --git a/tests/functional/test_test.py b/tests/functional/test_test.py index bf9824be12..797380800a 100644 --- a/tests/functional/test_test.py +++ b/tests/functional/test_test.py @@ -10,6 +10,7 @@ from ape.pytest.warnings import InvalidIsolationWarning from ape_test import ApeTestConfig from ape_test._watch import run_with_observer +from ape_test.config import IsolationConfig @pytest.fixture @@ -98,6 +99,53 @@ def get_opt(name: str): wrapper = ConfigWrapper(pytest_cfg) assert wrapper.verbosity is True + @pytest.mark.parametrize("flag", (True, None)) + def test_isolation_command_line(self, mocker, flag): + pytest_cfg = mocker.MagicMock() + + def get_opt(name: str): + if name == "disable_isolation": + return flag + + pytest_cfg.getoption.side_effect = get_opt + wrapper = ConfigWrapper(pytest_cfg) + + if flag: + assert not wrapper.isolation + for scope in Scope: + assert not wrapper.get_isolation(scope) + + else: + assert wrapper.isolation + + def test_isolation_config(self, mocker): + pytest_cfg = mocker.MagicMock() + pytest_cfg.getoption.return_value = None + ape_test_cfg = ApeTestConfig() + wrapper = ConfigWrapper(pytest_cfg) + wrapper.__dict__["ape_test_config"] = ape_test_cfg + + # Show can configure isolation as True. + ape_test_cfg.isolation = True + for scope in Scope: + assert wrapper.get_isolation(scope) + + # Show can configure isolation as False. + ape_test_cfg.isolation = False + for scope in Scope: + assert not wrapper.get_isolation(scope) + + # Show can configure individual scopes. + ape_test_cfg.isolation = IsolationConfig( + enable_session=True, + enable_package=False, + enable_function=True, + ) + assert wrapper.get_isolation(Scope.SESSION) + assert not wrapper.get_isolation(Scope.PACKAGE) # default + assert wrapper.get_isolation(Scope.MODULE) # default + assert wrapper.get_isolation(Scope.FUNCTION) + def test_connect_to_mainnet_by_default(mocker): """