From 0642d549d8dcf3be774fa82cfdb4c51b8edaec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Thu, 2 Jan 2025 16:10:34 +0100 Subject: [PATCH] Support running playbooks from Ansible collections Ansible collections can also ship playbooks, which can be given to `ansible-playbook` command as any actual playbook path. They have a special format, `..`. Playbook "paths" of this format are now accepted and executed just as any regular playbooks. --- tmt/steps/prepare/ansible.py | 62 +++++++++++++++++++++++++-------- tmt/steps/provision/__init__.py | 50 +++++++++++++++++--------- tmt/steps/provision/local.py | 2 +- tmt/steps/provision/podman.py | 2 +- 4 files changed, 83 insertions(+), 33 deletions(-) diff --git a/tmt/steps/prepare/ansible.py b/tmt/steps/prepare/ansible.py index 9ce088a264..61962597c2 100644 --- a/tmt/steps/prepare/ansible.py +++ b/tmt/steps/prepare/ansible.py @@ -13,7 +13,12 @@ import tmt.steps.provision import tmt.utils from tmt.result import PhaseResult -from tmt.steps.provision import Guest +from tmt.steps.provision import ( + ANSIBLE_COLLECTION_PLAYBOOK_PATTERN, + AnsibleApplicable, + AnsibleCollectionPlaybook, + Guest, + ) from tmt.utils import ( DEFAULT_RETRIABLE_HTTP_CODES, ENVFILE_RETRY_SESSION_RETRIES, @@ -127,8 +132,10 @@ class PrepareAnsible(tmt.steps.prepare.PreparePlugin[PrepareAnsibleData]): prepare --how ansible --playbook one.yml --playbook two.yml --extra-args '-vvv' - Remote playbooks can be referenced as well as local ones, and both - kinds can be intermixed: + Remote playbooks - provided as URLs starting with ``http://`` or + ``https://`` -, local playbooks - optionally starting with a + ``file://`` schema - , and playbooks bundled with collections can be + referenced as well as local ones, and all kinds can be intermixed: .. code-block:: yaml @@ -137,10 +144,15 @@ class PrepareAnsible(tmt.steps.prepare.PreparePlugin[PrepareAnsibleData]): playbook: - https://foo.bar/one.yml - two.yml + - file://three.yml + - ansible_galaxy_namespace.cool_collection.four .. code-block:: shell - prepare --how ansible --playbook https://foo.bar/two.yml --playbook two.yml + prepare --how ansible --playbook https://foo.bar/two.yml \\ + --playbook two.yml \\ + --playbook file://three.yml \\ + --playbook ansible_galaxy_namespace.cool_collection.four """ _data_class = PrepareAnsibleData @@ -155,13 +167,12 @@ def go( results = super().go(guest=guest, environment=environment, logger=logger) # Apply each playbook on the guest - for playbook in self.data.playbook: - logger.info('playbook', playbook, 'green') + for _playbook in self.data.playbook: + logger.info('playbook', _playbook, 'green') - lowercased_playbook = playbook.lower() - playbook_path = Path(playbook) + lowercased_playbook = _playbook.lower() - if lowercased_playbook.startswith(('http://', 'https://')): + def normalize_remote_playbook(raw_playbook: str) -> Path: assert self.step.plan.my_run is not None # narrow type assert self.step.plan.my_run.tree is not None # narrow type assert self.step.plan.my_run.tree.root is not None # narrow type @@ -171,15 +182,15 @@ def go( with retry_session( retries=ENVFILE_RETRY_SESSION_RETRIES, status_forcelist=DEFAULT_RETRIABLE_HTTP_CODES) as session: - response = session.get(playbook) + response = session.get(raw_playbook) if not response.ok: raise PrepareError( - f"Failed to fetch remote playbook '{playbook}'.") + f"Failed to fetch remote playbook '{raw_playbook}'.") except requests.RequestException as exc: raise PrepareError( - f"Failed to fetch remote playbook '{playbook}'.") from exc + f"Failed to fetch remote playbook '{raw_playbook}'.") from exc with tempfile.NamedTemporaryFile( mode='w+b', @@ -190,12 +201,33 @@ def go( file.write(response.content) file.flush() - playbook_path = Path(file.name).relative_to(root_path) + return Path(file.name).relative_to(root_path) + + def normalize_local_playbook(raw_playbook: str) -> Path: + if raw_playbook.startswith('file://'): + return Path(raw_playbook[7:]) + + return Path(raw_playbook) + + def normalize_collection_playbook(raw_playbook: str) -> AnsibleCollectionPlaybook: + return AnsibleCollectionPlaybook(raw_playbook) + + playbook: AnsibleApplicable + + if lowercased_playbook.startswith(('http://', 'https://')): + playbook = normalize_remote_playbook(lowercased_playbook) + + elif lowercased_playbook.startswith('file://'): + playbook = normalize_local_playbook(lowercased_playbook) + + elif ANSIBLE_COLLECTION_PLAYBOOK_PATTERN.match(lowercased_playbook): + playbook = normalize_collection_playbook(lowercased_playbook) - logger.info('playbook-path', playbook_path, 'green') + else: + playbook = normalize_local_playbook(lowercased_playbook) guest.ansible( - playbook_path, + playbook, playbook_root=self.step.plan.anchor_path, extra_args=self.data.extra_args) diff --git a/tmt/steps/provision/__init__.py b/tmt/steps/provision/__init__.py index ac0767d800..84a4bd02f5 100644 --- a/tmt/steps/provision/__init__.py +++ b/tmt/steps/provision/__init__.py @@ -20,6 +20,7 @@ Any, Callable, Literal, + NewType, Optional, TypeVar, Union, @@ -48,6 +49,7 @@ from tmt.steps import Action, ActionTask, PhaseQueue from tmt.utils import ( Command, + GeneralError, OnProcessStartCallback, Path, ProvisionError, @@ -62,6 +64,7 @@ if TYPE_CHECKING: import tmt.base import tmt.cli + from tmt._compat.typing import TypeAlias #: How many seconds to wait for a connection to succeed after guest reboot. @@ -78,6 +81,13 @@ RECONNECT_WAIT_TICK = 5 RECONNECT_WAIT_TICK_INCREASE = 1.0 +# Types for things Ansible can execute +ANSIBLE_COLLECTION_PLAYBOOK_PATTERN = re.compile(r'[a-zA-z0-9_]+\.[a-zA-z0-9_]+\.[a-zA-z0-9_]+') + +AnsiblePlaybook: 'TypeAlias' = Path +AnsibleCollectionPlaybook = NewType('AnsibleCollectionPlaybook', str) +AnsibleApplicable = Union[AnsibleCollectionPlaybook, AnsiblePlaybook] + def configure_ssh_options() -> tmt.utils.RawCommand: """ Extract custom SSH options from environment variables """ @@ -1122,8 +1132,8 @@ def _ansible_summary(self, output: Optional[str]) -> None: def _sanitize_ansible_playbook_path( self, - playbook: Path, - playbook_root: Optional[Path]) -> Path: + playbook: AnsibleApplicable, + playbook_root: Optional[Path]) -> AnsibleApplicable: """ Prepare full ansible playbook path. @@ -1136,21 +1146,29 @@ def _sanitize_ansible_playbook_path( the eventual playbook path is not absolute. """ - # Some playbooks must be under playbook root, which is often - # a metadata tree root. - if playbook_root is not None: - playbook = playbook_root / playbook.unrooted() + if isinstance(playbook, Path): + # Some playbooks must be under playbook root, which is often + # a metadata tree root. + if playbook_root is not None: + playbook = playbook_root / playbook.unrooted() - if not playbook.is_relative_to(playbook_root): - raise tmt.utils.GeneralError( - f"'{playbook}' is not relative to the expected root '{playbook_root}'.") + if not playbook.is_relative_to(playbook_root): + raise tmt.utils.GeneralError( + f"'{playbook}' is not relative to the expected root '{playbook_root}'.") + + if not playbook.exists(): + raise tmt.utils.FileError(f"Playbook '{playbook}' does not exist.") + + self.debug(f"Playbook full path: '{playbook}'", level=2) + + return playbook - if not playbook.exists(): - raise tmt.utils.FileError(f"Playbook '{playbook}' does not exist.") + if isinstance(playbook, str): + self.debug(f"Collection playbook: '{playbook}'", level=2) - self.debug(f"Playbook full path: '{playbook}'", level=2) + return playbook - return playbook + raise GeneralError(f"Unknown Ansible object type, '{type(playbook)}'.") def _prepare_environment( self, @@ -1229,7 +1247,7 @@ def _run_guest_command( def _run_ansible( self, - playbook: Path, + playbook: AnsibleApplicable, playbook_root: Optional[Path] = None, extra_args: Optional[str] = None, friendly_command: Optional[str] = None, @@ -1258,7 +1276,7 @@ def _run_ansible( def ansible( self, - playbook: Path, + playbook: AnsibleApplicable, playbook_root: Optional[Path] = None, extra_args: Optional[str] = None, friendly_command: Optional[str] = None, @@ -1807,7 +1825,7 @@ def _unlink_ssh_master_socket_path(self) -> None: def _run_ansible( self, - playbook: Path, + playbook: AnsibleApplicable, playbook_root: Optional[Path] = None, extra_args: Optional[str] = None, friendly_command: Optional[str] = None, diff --git a/tmt/steps/provision/local.py b/tmt/steps/provision/local.py index 78d2de1d3c..7d9920532a 100644 --- a/tmt/steps/provision/local.py +++ b/tmt/steps/provision/local.py @@ -28,7 +28,7 @@ def is_ready(self) -> bool: def _run_ansible( self, - playbook: Path, + playbook: tmt.steps.provision.AnsibleApplicable, playbook_root: Optional[Path] = None, extra_args: Optional[str] = None, friendly_command: Optional[str] = None, diff --git a/tmt/steps/provision/podman.py b/tmt/steps/provision/podman.py index 36f6533be2..58ad8c2c12 100644 --- a/tmt/steps/provision/podman.py +++ b/tmt/steps/provision/podman.py @@ -235,7 +235,7 @@ def reboot(self, hard: bool = False, def _run_ansible( self, - playbook: Path, + playbook: tmt.steps.provision.AnsibleApplicable, playbook_root: Optional[Path] = None, extra_args: Optional[str] = None, friendly_command: Optional[str] = None,