diff --git a/commodore/catalog.py b/commodore/catalog.py index bfc32545..7d984ed0 100644 --- a/commodore/catalog.py +++ b/commodore/catalog.py @@ -12,7 +12,6 @@ import yaml import json -from .component import Component from .gitrepo import GitRepo, GitCommandError from .helpers import ( ApiError, @@ -21,7 +20,7 @@ sliding_window, IndentedListDumper, ) -from .cluster import Cluster +from .cluster import Cluster, CompileMeta from .config import Config, Migration from .k8sobject import K8sObject @@ -34,45 +33,6 @@ def fetch_catalog(config: Config, cluster: Cluster) -> GitRepo: return GitRepo.clone(repo_url, config.catalog_dir, config) -def _pretty_print_component_commit(name, component: Component) -> str: - short_sha = component.repo.head_short_sha - return f" * {name}: {component.version} ({short_sha})" - - -def _pretty_print_config_commit(name, repo: GitRepo) -> str: - short_sha = repo.head_short_sha - return f" * {name}: {short_sha}" - - -def _render_catalog_commit_msg(cfg) -> str: - # pylint: disable=import-outside-toplevel - import datetime - - now = datetime.datetime.now().isoformat(timespec="milliseconds") - - component_commits = [ - _pretty_print_component_commit(cn, c) - for cn, c in sorted(cfg.get_components().items()) - ] - component_commits_str = "\n".join(component_commits) - - config_commits = [ - _pretty_print_config_commit(c, r) for c, r in cfg.get_configs().items() - ] - config_commits_str = "\n".join(config_commits) - - return f"""Automated catalog update from Commodore - -Component commits: -{component_commits_str} - -Configuration commits: -{config_commits_str} - -Compilation timestamp: {now} -""" - - def clean_catalog(repo: GitRepo): if repo.working_tree_dir is None: raise click.ClickException("Catalog repo has no working tree") @@ -96,6 +56,8 @@ def _push_catalog(cfg: Config, repo: GitRepo, commit_message: str): * User has requested pushing with `--push` Ask user to confirm push if `--interactive` is specified + + Returns True if the push was actually done and successful. False otherwise. """ if cfg.local: repo.reset(working_tree=False) @@ -126,10 +88,13 @@ def _push_catalog(cfg: Config, repo: GitRepo, commit_message: str): raise click.ClickException( f"Failed to push to the catalog repository: {summary}" ) - else: - click.echo(" > Skipping commit+push to catalog...") - click.echo(" > Use flag --push to commit and push the catalog repo") - click.echo(" > Add flag --interactive to show the diff and decide on the push") + + return True + + click.echo(" > Skipping commit+push to catalog...") + click.echo(" > Use flag --push to commit and push the catalog repo") + click.echo(" > Add flag --interactive to show the diff and decide on the push") + return False def _is_semantic_diff_kapitan_029_030(win: tuple[str, str]) -> bool: @@ -219,7 +184,16 @@ def _ignore_yaml_formatting_difffunc( return diff_lines, len(diff_lines) == 0 -def update_catalog(cfg: Config, targets: Iterable[str], repo: GitRepo): +def update_catalog( + cfg: Config, targets: Iterable[str], repo: GitRepo, compile_meta: CompileMeta +): + """Updates cluster catalog repo if there are any changes + + Prints diff of changes (with smart diffing if requested), and calls _push_catalog() + which will determine if the changes should actually be committed and pushed. + + Returns True if a commit was successfully pushed. False otherwise. + """ if repo.working_tree_dir is None: raise click.ClickException("Catalog repo has no working tree") @@ -251,14 +225,16 @@ def update_catalog(cfg: Config, targets: Iterable[str], repo: GitRepo): message = " > No changes." click.echo(message) - commit_message = _render_catalog_commit_msg(cfg) + commit_message = compile_meta.render_catalog_commit_message() if cfg.debug: click.echo(" > Commit message will be") click.echo(textwrap.indent(commit_message, " ")) + if changed: - _push_catalog(cfg, repo, commit_message) - else: - click.echo(" > Skipping commit+push to catalog...") + return _push_catalog(cfg, repo, commit_message) + + click.echo(" > Skipping commit+push to catalog...") + return False def catalog_list(cfg, out: str, sort_by: str = "id", tenant: str = ""): diff --git a/commodore/cluster.py b/commodore/cluster.py index edf7d813..c9ba9108 100644 --- a/commodore/cluster.py +++ b/commodore/cluster.py @@ -1,13 +1,17 @@ from __future__ import annotations +import json import os +import textwrap +from datetime import datetime from typing import Any, Optional, Union import click -from . import __kustomize_wrapper__ +from . import __kustomize_wrapper__, __git_version__, __version__ from .helpers import ( + lieutenant_post, lieutenant_query, yaml_dump, yaml_load, @@ -258,3 +262,81 @@ def update_params(inv: Inventory, cluster: Cluster): file = inv.params_file os.makedirs(file.parent, exist_ok=True) yaml_dump(render_params(inv, cluster), file) + + +class CompileMeta: + def __init__(self, cfg: Config): + self.build_info = {"version": __version__, "gitVersion": __git_version__} + self.instances = cfg.get_component_alias_versioninfos() + self.packages = cfg.get_package_versioninfos() + self.global_repo = cfg.global_version_info + self.tenant_repo = cfg.tenant_version_info + self.timestamp = datetime.now().astimezone(None) + + def as_dict(self): + return { + "commodoreBuildInfo": self.build_info, + "global": self.global_repo.as_dict(), + "instances": { + a: info.as_dict() for a, info in sorted(self.instances.items()) + }, + "lastCompile": self.timestamp.isoformat(timespec="milliseconds"), + "packages": { + p: info.as_dict() for p, info in sorted(self.packages.items()) + }, + "tenant": self.tenant_repo.as_dict(), + } + + def render_catalog_commit_message(self) -> str: + component_commits = [ + info.pretty_print(i) for i, info in sorted(self.instances.items()) + ] + component_commits_str = "\n".join(component_commits) + + package_commits = [ + info.pretty_print(p) for p, info in sorted(self.packages.items()) + ] + package_commits_str = "\n".join(package_commits) + + config_commits = [ + self.global_repo.pretty_print("global"), + self.tenant_repo.pretty_print("tenant"), + ] + config_commits_str = "\n".join(config_commits) + + return f"""Automated catalog update from Commodore + +Component instance commits: +{component_commits_str} + +Package commits: +{package_commits_str} + +Configuration commits: +{config_commits_str} + +Compilation timestamp: {self.timestamp.isoformat(timespec="milliseconds")} +""" + + +def report_compile_metadata( + cfg: Config, compile_meta: CompileMeta, cluster_id: str, report=False +): + if cfg.verbose: + if report: + action = "will be reported to Lieutenant" + else: + action = "would be reported to Lieutenant on a successful catalog push" + click.echo( + f" > The following compile metadata {action}:\n" + + textwrap.indent(json.dumps(compile_meta.as_dict(), indent=2), " "), + ) + + if report: + lieutenant_post( + cfg.api_url, + cfg.api_token, + f"clusters/{cluster_id}", + "compileMeta", + post_data=compile_meta.as_dict(), + ) diff --git a/commodore/compile.py b/commodore/compile.py index fdf8d03b..ddc865b0 100644 --- a/commodore/compile.py +++ b/commodore/compile.py @@ -9,8 +9,10 @@ from .catalog import fetch_catalog, clean_catalog, update_catalog from .cluster import ( Cluster, + CompileMeta, load_cluster_from_api, read_cluster_and_tenant, + report_compile_metadata, update_params, update_target, ) @@ -279,7 +281,10 @@ def compile(config, cluster_id): postprocess_components(config, inventory, config.get_components()) - update_catalog(config, targets, catalog_repo) + compile_meta = CompileMeta(config) + + push_done = update_catalog(config, targets, catalog_repo, compile_meta) + report_compile_metadata(config, compile_meta, cluster_id, report=push_done) click.secho("Catalog compiled! 🎉", bold=True) diff --git a/commodore/component/__init__.py b/commodore/component/__init__.py index 704b0553..7d58e9af 100644 --- a/commodore/component/__init__.py +++ b/commodore/component/__init__.py @@ -118,6 +118,10 @@ def version(self) -> Optional[str]: def version(self, version: str): self._version = version + @property + def sub_path(self) -> str: + return self._sub_path + @property def repo_directory(self) -> P: return self._dir @@ -173,6 +177,9 @@ def checkout(self): ) self._dependency.checkout_component(self.name, self.version) + def is_checked_out(self) -> bool: + return self.target_dir is not None and self.target_dir.is_dir() + def checkout_is_dirty(self) -> bool: if self._dependency: dep_repo = self._dependency.bare_repo diff --git a/commodore/config.py b/commodore/config.py index 9826674d..8da8adef 100644 --- a/commodore/config.py +++ b/commodore/config.py @@ -27,6 +27,65 @@ class Migration(Enum): IGNORE_YAML_FORMATTING = "ignore-yaml-formatting" +class VersionInfo: + def __init__( + self, + url: str, + version: str, + git_sha: str, + short_sha: str, + path: Optional[str] = None, + ): + self.url = url + self.version = version + self.path = path + self.git_sha = git_sha + self.short_sha = short_sha + + def as_dict(self): + info = { + "gitSha": self.git_sha, + "url": self.url, + "version": self.version, + } + if self.path: + info["path"] = self.path + + return info + + def pretty_print(self, name: str) -> str: + path = "" + if self.path: + path = f"\n path: {self.path}" + return ( + f" * {name}: {self.version} ({self.short_sha})\n" + + f" url: {self.url}{path}" + ) + + +class InstanceVersionInfo(VersionInfo): + def __init__(self, component: Component): + super().__init__( + component.repo_url, + component.version or component.repo.default_version, + component.repo.head_sha, + component.repo.head_short_sha, + path=component.sub_path, + ) + self.component = component.name + + def as_dict(self): + info = super().as_dict() + info["component"] = self.component + return info + + def pretty_print(self, name: str) -> str: + pretty_name = name + if self.component != name: + pretty_name = f"{name} ({self.component})" + return super().pretty_print(pretty_name) + + # pylint: disable=too-many-instance-attributes,too-many-public-methods class Config: _inventory: Inventory @@ -183,6 +242,16 @@ def global_repo_revision_override(self): def global_repo_revision_override(self, rev): self._global_repo_revision_override = rev + @property + def global_version_info(self) -> VersionInfo: + repo = self._config_repos["global"] + return VersionInfo( + repo.remote, + self.global_repo_revision_override or repo.default_version, + repo.head_sha, + repo.head_short_sha, + ) + @property def tenant_repo_revision_override(self): return self._tenant_repo_revision_override @@ -191,6 +260,16 @@ def tenant_repo_revision_override(self): def tenant_repo_revision_override(self, rev): self._tenant_repo_revision_override = rev + @property + def tenant_version_info(self) -> VersionInfo: + repo = self._config_repos["customer"] + return VersionInfo( + repo.remote, + self.tenant_repo_revision_override or repo.default_version, + repo.head_sha, + repo.head_short_sha, + ) + @property def migration(self): return self._migration @@ -253,6 +332,18 @@ def get_packages(self) -> dict[str, Package]: def register_package(self, pkg_name: str, pkg: Package): self._packages[pkg_name] = pkg + def get_package_versioninfos(self) -> dict[str, VersionInfo]: + return { + p: VersionInfo( + pkg.url, + pkg.version or pkg.repo.default_version, + pkg.repo.head_sha, + pkg.repo.head_short_sha, + path=pkg.sub_path, + ) + for p, pkg in self._packages.items() + } + def register_dependency_repo(self, repo_url: str) -> MultiDependency: """Register dependency repository, if it isn't registered yet. @@ -285,6 +376,12 @@ def verify_component_aliases(self, cluster_parameters: dict): f"Component {cn} with alias {alias} does not support instantiation." ) + def get_component_alias_versioninfos(self) -> dict[str, InstanceVersionInfo]: + return { + a: InstanceVersionInfo(self._components[cn]) + for a, cn in self._component_aliases.items() + } + def register_deprecation_notice(self, notice: str): self._deprecation_notices.append(notice) diff --git a/commodore/dependency_syncer.py b/commodore/dependency_syncer.py index bd8100cf..0a200a00 100644 --- a/commodore/dependency_syncer.py +++ b/commodore/dependency_syncer.py @@ -180,7 +180,7 @@ def ensure_branch(d: Union[Component, Package], branch_name: str): commit.""" deptype = type_name(d) - if not d.repo: + if not d.is_checked_out(): raise ValueError(f"{deptype} repo not initialized") r = d.repo.repo has_sync_branch = any(h.name == branch_name for h in r.heads) @@ -204,7 +204,7 @@ def ensure_pr( """Create or update template sync PR.""" deptype = type_name(d) - if not d.repo: + if not d.is_checked_out(): raise ValueError(f"{deptype} repo not initialized") prs = gr.get_pulls(state="open") diff --git a/commodore/gitrepo/__init__.py b/commodore/gitrepo/__init__.py index 04290949..82e79a9d 100644 --- a/commodore/gitrepo/__init__.py +++ b/commodore/gitrepo/__init__.py @@ -164,9 +164,13 @@ def working_tree_dir(self) -> Optional[Path]: @property def head_short_sha(self) -> str: - sha = self._repo.head.commit.hexsha + sha = self.head_sha return self._repo.git.rev_parse(sha, short=6) + @property + def head_sha(self) -> str: + return self._repo.head.commit.hexsha + @property def _author_env(self) -> dict[str, str]: return { @@ -204,6 +208,10 @@ def author(self) -> Actor: self._author = Actor("Commodore", "commodore@syn.tools") return self._author + @property + def default_version(self) -> str: + return self._default_version() + def _remote_prefix(self) -> str: """ Find prefix of Git remote, will usually be 'origin/'. diff --git a/commodore/helpers.py b/commodore/helpers.py index f1593ead..479cf50b 100644 --- a/commodore/helpers.py +++ b/commodore/helpers.py @@ -13,6 +13,8 @@ import requests import yaml +from enum import Enum + # pylint: disable=redefined-builtin from requests.exceptions import ConnectionError, HTTPError from url_normalize import url_normalize @@ -111,18 +113,52 @@ def yaml_dump_all(obj, file): yaml.dump_all(obj, outf, Dumper=IndentedListDumper) -def lieutenant_query(api_url, api_token, api_endpoint, api_id, params={}, timeout=5): +class RequestMethod(Enum): + GET = "GET" + POST = "POST" + + +def _lieutenant_request( + method: RequestMethod, + api_url: str, + api_token: str, + api_endpoint: str, + api_id: str, + params={}, + timeout=5, + **kwargs, +): + url = url_normalize(f"{api_url}/{api_endpoint}/{api_id}") + headers = {"Authorization": f"Bearer {api_token}"} try: - r = requests.get( - url_normalize(f"{api_url}/{api_endpoint}/{api_id}"), - headers={"Authorization": f"Bearer {api_token}"}, - params=params, - timeout=timeout, - ) + if method == RequestMethod.GET: + r = requests.get(url, headers=headers, params=params, timeout=timeout) + elif method == RequestMethod.POST: + headers["Content-Type"] = "application/json" + data = kwargs.get("post_data", {}) + r = requests.post( + url, + json.dumps(data), + headers=headers, + params=params, + timeout=timeout, + ) + else: + raise NotImplementedError(f"QueryType {method} not implemented") except ConnectionError as e: raise ApiError(f"Unable to connect to Lieutenant at {api_url}") from e + except NotImplementedError as e: + raise e + + return _handle_lieutenant_response(r) + + +def _handle_lieutenant_response(r: requests.Response): try: - resp = json.loads(r.text) + if r.text: + resp = json.loads(r.text) + else: + resp = {} except json.JSONDecodeError as e: raise ApiError("Client error: Unable to parse JSON") from e try: @@ -139,6 +175,27 @@ def lieutenant_query(api_url, api_token, api_endpoint, api_id, params={}, timeou return resp +def lieutenant_query(api_url, api_token, api_endpoint, api_id, params={}, timeout=5): + return _lieutenant_request( + RequestMethod.GET, api_url, api_token, api_endpoint, api_id, params, timeout + ) + + +def lieutenant_post( + api_url, api_token, api_endpoint, api_id, post_data, params={}, timeout=5 +): + return _lieutenant_request( + RequestMethod.POST, + api_url, + api_token, + api_endpoint, + api_id, + params, + timeout, + post_data=post_data, + ) + + def _verbose_rmtree(tree, *args, **kwargs): click.echo(f" > deleting {tree}/") shutil.rmtree(tree, *args, **kwargs) diff --git a/commodore/package/__init__.py b/commodore/package/__init__.py index c27dcfc2..46699207 100644 --- a/commodore/package/__init__.py +++ b/commodore/package/__init__.py @@ -40,10 +40,11 @@ def __init__( self._sub_path = sub_path self._dependency = dependency self._dependency.register_package(name, target_dir) + self._dir = target_dir self._gitrepo = None @property - def url(self) -> Optional[str]: + def url(self) -> str: return self._dependency.url @property @@ -59,8 +60,8 @@ def repository_dir(self) -> Optional[Path]: return self._dependency.get_package(self._name) @property - def repo(self) -> Optional[GitRepo]: - if not self._gitrepo and self.target_dir and self.target_dir.is_dir(): + def repo(self) -> GitRepo: + if not self._gitrepo: if self._dependency: dep_repo = self._dependency.bare_repo author_name = dep_repo.author.name @@ -71,7 +72,7 @@ def repo(self) -> Optional[GitRepo]: author_email = None self._gitrepo = GitRepo( None, - self.target_dir, + self._dir, author_name=author_name, author_email=author_email, ) @@ -88,6 +89,9 @@ def target_dir(self) -> Optional[Path]: def checkout(self): self._dependency.checkout_package(self._name, self._version) + def is_checked_out(self) -> bool: + return self.target_dir is not None and self.target_dir.is_dir() + def checkout_is_dirty(self) -> bool: dep_repo = self._dependency.bare_repo author_name = dep_repo.author.name diff --git a/docs/modules/ROOT/pages/explanation/compilation-metadata.adoc b/docs/modules/ROOT/pages/explanation/compilation-metadata.adoc new file mode 100644 index 00000000..55bc9cc9 --- /dev/null +++ b/docs/modules/ROOT/pages/explanation/compilation-metadata.adoc @@ -0,0 +1,17 @@ += Compilation metadata reporting + +The reporting is implemented according to https://syn.tools/syn/SDDs/0031-component-version-tracking.html[SDD 0031 - Central Component Version tracking]. + +Commodore will only report metadata for catalog compilations that result in a new catalog commit which was successfully pushed to the catalog repository. + +Currently, Commodore reports the following metadata: + +* Component instance URLs, versions, subpaths, and Git commit hashes +* Package URLs, versions, subpaths, and Git commit hashes +* Global repo URL, version and Git commit hash +* Tenant repo URL, version and Git commit hash +* Commodore Python package version and Git version +* The timestamp of the successful compilation + + +Commodore uses the same data that's reported to Lieutenant to generate the catalog commit message. diff --git a/docs/modules/ROOT/partials/nav-explanation.adoc b/docs/modules/ROOT/partials/nav-explanation.adoc index ad06a5f5..a0bbdfb4 100644 --- a/docs/modules/ROOT/partials/nav-explanation.adoc +++ b/docs/modules/ROOT/partials/nav-explanation.adoc @@ -2,3 +2,4 @@ * xref:commodore:ROOT:explanation/dependencies.adoc[Manage Dependencies] * xref:commodore:ROOT:explanation/running-commodore.adoc[Running Commodore] * xref:commodore:ROOT:explanation/migrate-kapitan-0.29-0.30.adoc[Migrating from Kapitan 0.29 to 0.30] +* xref:commodore:ROOT:explanation/compilation-metadata.adoc[] diff --git a/tests/conftest.py b/tests/conftest.py index 9e5dd351..33b9442a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,8 +30,8 @@ def gitconfig(tmp_path: Path) -> Path: """Ensure that tests have a predictable empty gitconfig. We set autouse=True, so that the fixture is automatically used for all - tests. Tests that want to access the mock gitconfig can explicitly specify - the fixutre, so they get the path to the mock gitconfig. + tests. Tests that want to access the mock gitconfig can explicitly specify + the fixture, so they get the path to the mock gitconfig. """ os.environ["GIT_CONFIG_NOSYSTEM"] = "true" os.environ["HOME"] = str(tmp_path) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index d5290ce5..99b270a8 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -18,7 +18,9 @@ from commodore import catalog from commodore.config import Config -from commodore.cluster import Cluster +from commodore.cluster import Cluster, CompileMeta + +from test_compile_meta import _setup_config_repos cluster_resp = { "id": "c-test", @@ -86,18 +88,6 @@ def fresh_cluster(tmp_path: Path) -> Cluster: ) -def test_catalog_commit_message(tmp_path: Path): - config = Config( - tmp_path, - api_url="https://syn.example.com", - api_token="token", - ) - - commit_message = catalog._render_catalog_commit_msg(config) - assert not commit_message.startswith("\n") - assert commit_message.startswith("Automated catalog update from Commodore\n\n") - - def test_fetch_catalog_inexistent( tmp_path: Path, config: Config, inexistent_cluster: Cluster ): @@ -271,12 +261,18 @@ def write_target_file_2(target: Path, name="test.txt", change=True): "migration", ["", "kapitan-0.29-to-0.30", "ignore-yaml-formatting"] ) def test_update_catalog( - capsys, tmp_path: Path, config: Config, fresh_cluster: Cluster, migration: str + capsys, + tmp_path: Path, + config: Config, + fresh_cluster: Cluster, + migration: str, ): repo = catalog.fetch_catalog(config, fresh_cluster) upstream = git.Repo(tmp_path / "repo.git") config.push = True + _setup_config_repos(config) + compile_meta = CompileMeta(config) target = tmp_path / "compiled" / "test" target.mkdir(parents=True, exist_ok=True) @@ -285,7 +281,7 @@ def test_update_catalog( write_target_file_1(target, name="a.yaml") write_target_file_1(target, name="b.yaml") - catalog.update_catalog(config, ["test"], repo) + catalog.update_catalog(config, ["test"], repo, compile_meta) captured = capsys.readouterr() assert upstream.active_branch.commit.message.startswith( @@ -304,7 +300,7 @@ def test_update_catalog( write_target_file_2(target, name="a.yaml") write_target_file_2(target, name="b.yaml", change=False) config.migration = migration - catalog.update_catalog(config, ["test"], repo) + catalog.update_catalog(config, ["test"], repo, compile_meta) addl_indent = "" # Diff with real changes is shown with correct additional diff --git a/tests/test_catalog_compile.py b/tests/test_catalog_compile.py index 4dd3c6c8..cfb39ca0 100644 --- a/tests/test_catalog_compile.py +++ b/tests/test_catalog_compile.py @@ -326,6 +326,5 @@ def test_catalog_compile_local(capsys, tmp_path: Path, config: Config): print(captured.out) assert captured.out.startswith("Running in local mode\n") assert "Updating catalog repository...\n > No changes." in captured.out - assert captured.out.endswith( - " > Skipping commit+push to catalog...\nCatalog compiled! 🎉\n" - ) + assert " > Skipping commit+push to catalog...\n" in captured.out + assert captured.out.endswith("Catalog compiled! 🎉\n") diff --git a/tests/test_compile_meta.py b/tests/test_compile_meta.py new file mode 100644 index 00000000..85781f33 --- /dev/null +++ b/tests/test_compile_meta.py @@ -0,0 +1,223 @@ +import os + +from datetime import datetime, timedelta +from pathlib import Path +from unittest.mock import patch + +import git +import pytest +import responses + +from responses import matchers + +from commodore.config import Config +from commodore.dependency_mgmt import fetch_components, fetch_packages +from commodore.gitrepo import GitRepo + +from test_dependency_mgmt import setup_components_upstream, _setup_packages + +from commodore.catalog import CompileMeta +from commodore.cluster import report_compile_metadata + + +def _setup_config_repos(cfg: Config, tenant="t-test-tenant"): + os.makedirs(cfg.inventory.inventory_dir) + global_repo = GitRepo( + "ssh://git@git.example.com/global-defaults", + cfg.inventory.global_config_dir, + force_init=True, + ) + with open( + cfg.inventory.global_config_dir / "commodore.yml", "w", encoding="utf-8" + ) as f: + f.write("---\n") + global_repo.stage_all() + global_repo.commit("Initial commit") + cfg.register_config("global", global_repo) + + tenant_repo = GitRepo( + "ssh://git@git.example.com/test-tenant", + cfg.inventory.tenant_config_dir(tenant), + force_init=True, + ) + with open( + cfg.inventory.tenant_config_dir(tenant) / "common.yml", + "w", + encoding="utf-8", + ) as f: + f.write("---\n") + tenant_repo.stage_all() + tenant_repo.commit("Initial commit") + cfg.register_config("customer", tenant_repo) + + return (global_repo.repo.head.commit.hexsha, tenant_repo.repo.head.commit.hexsha) + + +def test_compile_meta_render_catalog_commit_message_no_leading_newline( + config: Config, tmp_path: Path +): + _setup_config_repos(config) + + compile_meta = CompileMeta(config) + + commit_message = compile_meta.render_catalog_commit_message() + + assert not commit_message.startswith("\n") + assert commit_message.startswith("Automated catalog update from Commodore\n\n") + + +@patch("commodore.dependency_mgmt._read_components") +@patch("commodore.dependency_mgmt._discover_components") +@patch("commodore.dependency_mgmt._read_packages") +@patch("commodore.dependency_mgmt._discover_packages") +def test_compile_meta_creation( + patch_discover_packages, + patch_read_packages, + patch_discover_components, + patch_read_components, + config: Config, + tmp_path: Path, +): + components = ["foo", "bar", "baz", "qux"] + packages = ["foo", "bar"] + + # setup mock components + patch_discover_components.return_value = (components, {}) + # setup_components_upstream sets version=None for all components + cdeps = setup_components_upstream(tmp_path, components) + + # NOTE: We assume that setup_components_upstream creates upstream repos in + # `tmp_path/upstream/` + crepos = {cn: git.Repo(tmp_path / "upstream" / cn) for cn in components} + + crepos["foo"].version = "master" + crepos["bar"].create_tag("v1.2.3") + cdeps["bar"].version = "v1.2.3" + cdeps["baz"].version = crepos["baz"].head.commit.hexsha + + patch_read_components.return_value = cdeps + + # setup mock packages + patch_discover_packages.return_value = packages + pdeps = _setup_packages(tmp_path / "packages_upstream", packages) + prepos = { + pkg: git.Repo(tmp_path / "packages_upstream" / f"{pkg}.git") for pkg in packages + } + + patch_read_packages.return_value = pdeps + + # setup mock tenant&global repo + global_sha, tenant_sha = _setup_config_repos(config) + + # use regular logic to fetch mocked packages & components + fetch_packages(config) + fetch_components(config) + + config.get_components()["qux"]._sub_path = "component" + + aliases = {cn: cn for cn in components} + # create alias to verify instance reporting + aliases["quxxer"] = "qux" + config.register_component_aliases(aliases) + + meta = CompileMeta(config) + meta_dict = meta.as_dict() + assert set(meta_dict.keys()) == { + "commodoreBuildInfo", + "global", + "instances", + "lastCompile", + "packages", + "tenant", + } + + assert set(meta_dict["commodoreBuildInfo"].keys()) == {"gitVersion", "version"} + # dummy value that's overwritten by the release CI + assert meta_dict["commodoreBuildInfo"]["gitVersion"] == "0" + # dummy value that's overwritten by the release CI + assert meta_dict["commodoreBuildInfo"]["version"] == "0.0.0" + + # sanity check last compile timestamp. We can't check an exact value but it should + # be <1s in the past regardless of how slow the test is. We expect the last compile + # timestamp to be in ISO format with a timezone. If that isn't the case the + # conversion or comparison should raise an exception. + assert datetime.now().astimezone() - datetime.fromisoformat( + meta_dict["lastCompile"] + ) < timedelta(seconds=1) + + # check instances + assert set(meta_dict["instances"].keys()) == {"foo", "bar", "baz", "qux", "quxxer"} + for alias, info in meta_dict["instances"].items(): + cn = alias[0:3] + assert len({"url", "version", "gitSha", "component"} - set(info.keys())) == 0 + assert info["component"] == cn + assert info["url"] == cdeps[cn].url + # NOTE(sg): We currently make sure that the tests use an empty gitconfig which + # means that we should always get "master" as the default branch. + assert info["version"] == cdeps[cn].version or "master" + assert info["gitSha"] == crepos[cn].head.commit.hexsha + assert "path" in meta_dict["instances"]["qux"] + assert "path" in meta_dict["instances"]["quxxer"] + assert meta_dict["instances"]["qux"]["path"] == "component" + assert meta_dict["instances"]["quxxer"]["path"] == "component" + + # check packages + assert set(meta_dict["packages"].keys()) == {"foo", "bar"} + for pkg, info in meta_dict["packages"].items(): + assert set(info.keys()) == {"url", "version", "gitSha"} + assert info["url"] == pdeps[pkg].url + assert info["version"] == pdeps[pkg].version or "master" + assert info["gitSha"] == prepos[pkg].head.commit.hexsha + + # check global & tenant version info + assert set(meta_dict["global"].keys()) == {"url", "version", "gitSha"} + assert meta_dict["global"]["url"] == "ssh://git@git.example.com/global-defaults" + assert meta_dict["global"]["version"] == "master" + assert meta_dict["global"]["gitSha"] == global_sha + assert set(meta_dict["tenant"].keys()) == {"url", "version", "gitSha"} + assert meta_dict["tenant"]["url"] == "ssh://git@git.example.com/test-tenant" + assert meta_dict["tenant"]["version"] == "master" + assert meta_dict["tenant"]["gitSha"] == tenant_sha + + +def test_compile_meta_config_overrides(tmp_path: Path, config: Config): + # NOTE: The CompileMeta initialization doesn't validate that the checked out + # revision of the repo matches the version override. + global_sha, tenant_sha = _setup_config_repos(config) + config.tenant_repo_revision_override = "feat/test" + config.global_repo_revision_override = global_sha[0:10] + + compile_meta = CompileMeta(config) + + assert compile_meta.global_repo.version == global_sha[0:10] + assert compile_meta.global_repo.git_sha == global_sha + assert compile_meta.tenant_repo.version == "feat/test" + assert compile_meta.tenant_repo.git_sha == tenant_sha + + +@pytest.mark.parametrize("report", [False, True]) +@responses.activate +def test_report_compile_meta(tmp_path: Path, config: Config, capsys, report): + _setup_config_repos(config, "t-tenant-1234") + config.update_verbosity(1) + + compile_meta = CompileMeta(config) + responses.add( + responses.POST, + f"{config.api_url}/clusters/c-cluster-1234/compileMeta", + content_type="application/json", + status=204, + body=None, + match=[matchers.json_params_matcher(compile_meta.as_dict())], + ) + report_compile_metadata(config, compile_meta, "c-cluster-1234", report) + + captured = capsys.readouterr() + if report: + assert captured.out.startswith( + " > The following compile metadata will be reported to Lieutenant:\n" + ) + else: + assert captured.out.startswith( + " > The following compile metadata would be reported to Lieutenant on a successful catalog push:\n" + ) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index c65abbc4..edcaf9fd 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -13,6 +13,7 @@ import click import pytest import responses +from responses import matchers from url_normalize import url_normalize import commodore.helpers as helpers @@ -176,11 +177,11 @@ def test_sliding_window(sequence, winsize, expected): assert windows == expected -def _verify_call_status(query_url): +def _verify_call_status(query_url, token="token"): assert len(responses.calls) == 1 call = responses.calls[0] assert "Authorization" in call.request.headers - assert call.request.headers["Authorization"] == "Bearer token" + assert call.request.headers["Authorization"] == f"Bearer {token}" assert call.request.url == query_url @@ -262,6 +263,91 @@ def test_lieutenant_query_response_errors(response, expected): _verify_call_status(query_url) +@pytest.mark.parametrize( + "request_data,response,expected", + [ + ( + { + "token": "token", + "payload": {"some": "data", "other": "data"}, + }, + { + "status": 204, + }, + "", + ), + ( + { + "token": "", + "payload": {"some": "data", "other": "data"}, + }, + { + "status": 400, + "json": {"reason": "missing or malformed jwt"}, + }, + "API returned 400: missing or malformed jwt", + ), + ], +) +@responses.activate +def test_lieutenant_post(request_data, response, expected): + base_url = "https://syn.example.com/" + + post_url = url_normalize(f"{base_url}/clusters/c-cluster-1234/compileMeta") + + if response["status"] == 204: + # successful post response from Lieutenant API has no body + responses.add( + responses.POST, + post_url, + content_type="application/json", + status=204, + body=None, + match=[matchers.json_params_matcher(request_data["payload"])], + ) + else: + responses.add( + responses.POST, + post_url, + content_type="application/json", + status=response["status"], + json=response["json"], + match=[matchers.json_params_matcher(request_data["payload"])], + ) + + if response["status"] == 204: + resp = helpers.lieutenant_post( + base_url, + request_data["token"], + "clusters/c-cluster-1234", + "compileMeta", + post_data=request_data["payload"], + ) + assert resp == {} + else: + with pytest.raises(helpers.ApiError, match=expected): + helpers.lieutenant_post( + base_url, + request_data["token"], + "clusters/c-cluster-1234", + "compileMeta", + post_data=request_data["payload"], + ) + + _verify_call_status(post_url, token=request_data["token"]) + + +def test_unimplemented_query_method(): + with pytest.raises(NotImplementedError, match="QueryType PATCH not implemented"): + helpers._lieutenant_request( + "PATCH", + "https://api.example.com", + "token", + "clusters", + "", + ) + + def test_relsymlink(tmp_path: Path): test_file = tmp_path / "src" / "test" test_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_login.py b/tests/test_login.py index bdbbe776..f2c6d1ca 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -377,16 +377,16 @@ def test_run_callback_server(config, tmp_path): config.oidc_client = "test-client" token_url = "https://idp.example.com/token" c = WebApplicationClient(config.oidc_client) - s = login.OIDCCallbackServer(c, token_url, config.api_url, 5, port=19000) + s = login.OIDCCallbackServer(c, token_url, config.api_url, 5, port=18999) s.start() - resp = requests.get("http://localhost:19000/healthz", timeout=5) + resp = requests.get("http://localhost:18999/healthz", timeout=5) assert resp.status_code == 200 assert resp.text == "ok" # calls to /healthz don't close the server, so we make a second request - resp = requests.get("http://localhost:19000/?foo=bar", timeout=5) + resp = requests.get("http://localhost:18999/?foo=bar", timeout=5) assert resp.status_code == 422 assert resp.text == "invalid callback: no code provided"