Merge pull request #971 from projectsyn/feat/report-compile-meta

Report compilation metadata to Lieutenant according to SDD-0031

commodore/catalog.py

@@ -12,7 +12,6 @@
 import yaml
 import json
 
-from .component import Component
 from .gitrepo import GitRepo, GitCommandError
 from .helpers import (
     ApiError,
@@ -21,7 +20,7 @@
     sliding_window,
     IndentedListDumper,
 )
-from .cluster import Cluster
+from .cluster import Cluster, CompileMeta
 from .config import Config, Migration
 from .k8sobject import K8sObject
 
@@ -34,45 +33,6 @@ def fetch_catalog(config: Config, cluster: Cluster) -> GitRepo:
     return GitRepo.clone(repo_url, config.catalog_dir, config)
 
 
-def _pretty_print_component_commit(name, component: Component) -> str:
-    short_sha = component.repo.head_short_sha
-    return f" * {name}: {component.version} ({short_sha})"
-
-
-def _pretty_print_config_commit(name, repo: GitRepo) -> str:
-    short_sha = repo.head_short_sha
-    return f" * {name}: {short_sha}"
-
-
-def _render_catalog_commit_msg(cfg) -> str:
-    # pylint: disable=import-outside-toplevel
-    import datetime
-
-    now = datetime.datetime.now().isoformat(timespec="milliseconds")
-
-    component_commits = [
-        _pretty_print_component_commit(cn, c)
-        for cn, c in sorted(cfg.get_components().items())
-    ]
-    component_commits_str = "\n".join(component_commits)
-
-    config_commits = [
-        _pretty_print_config_commit(c, r) for c, r in cfg.get_configs().items()
-    ]
-    config_commits_str = "\n".join(config_commits)
-
-    return f"""Automated catalog update from Commodore
-
-Component commits:
-{component_commits_str}
-
-Configuration commits:
-{config_commits_str}
-
-Compilation timestamp: {now}
-"""
-
-
 def clean_catalog(repo: GitRepo):
     if repo.working_tree_dir is None:
         raise click.ClickException("Catalog repo has no working tree")
@@ -96,6 +56,8 @@ def _push_catalog(cfg: Config, repo: GitRepo, commit_message: str):
     * User has requested pushing with `--push`
 
     Ask user to confirm push if `--interactive` is specified
+
+    Returns True if the push was actually done and successful. False otherwise.
     """
     if cfg.local:
         repo.reset(working_tree=False)
@@ -126,10 +88,13 @@ def _push_catalog(cfg: Config, repo: GitRepo, commit_message: str):
                 raise click.ClickException(
                     f"Failed to push to the catalog repository: {summary}"
                 )
-    else:
-        click.echo(" > Skipping commit+push to catalog...")
-        click.echo(" > Use flag --push to commit and push the catalog repo")
-        click.echo(" > Add flag --interactive to show the diff and decide on the push")
+
+        return True
+
+    click.echo(" > Skipping commit+push to catalog...")
+    click.echo(" > Use flag --push to commit and push the catalog repo")
+    click.echo(" > Add flag --interactive to show the diff and decide on the push")
+    return False
 
 
 def _is_semantic_diff_kapitan_029_030(win: tuple[str, str]) -> bool:
@@ -219,7 +184,16 @@ def _ignore_yaml_formatting_difffunc(
     return diff_lines, len(diff_lines) == 0
 
 
-def update_catalog(cfg: Config, targets: Iterable[str], repo: GitRepo):
+def update_catalog(
+    cfg: Config, targets: Iterable[str], repo: GitRepo, compile_meta: CompileMeta
+):
+    """Updates cluster catalog repo if there are any changes
+
+    Prints diff of changes (with smart diffing if requested), and calls _push_catalog()
+    which will determine if the changes should actually be committed and pushed.
+
+    Returns True if a commit was successfully pushed. False otherwise.
+    """
     if repo.working_tree_dir is None:
         raise click.ClickException("Catalog repo has no working tree")
 
@@ -251,14 +225,16 @@ def update_catalog(cfg: Config, targets: Iterable[str], repo: GitRepo):
         message = " > No changes."
     click.echo(message)
 
-    commit_message = _render_catalog_commit_msg(cfg)
+    commit_message = compile_meta.render_catalog_commit_message()
     if cfg.debug:
         click.echo(" > Commit message will be")
         click.echo(textwrap.indent(commit_message, "   "))
+
     if changed:
-        _push_catalog(cfg, repo, commit_message)
-    else:
-        click.echo(" > Skipping commit+push to catalog...")
+        return _push_catalog(cfg, repo, commit_message)
+
+    click.echo(" > Skipping commit+push to catalog...")
+    return False
 
 
 def catalog_list(cfg, out: str, sort_by: str = "id", tenant: str = ""):

commodore/cluster.py

@@ -1,13 +1,17 @@
 from __future__ import annotations
 
+import json
 import os
+import textwrap
 
+from datetime import datetime
 from typing import Any, Optional, Union
 
 import click
 
-from . import __kustomize_wrapper__
+from . import __kustomize_wrapper__, __git_version__, __version__
 from .helpers import (
+    lieutenant_post,
     lieutenant_query,
     yaml_dump,
     yaml_load,
@@ -258,3 +262,81 @@ def update_params(inv: Inventory, cluster: Cluster):
     file = inv.params_file
     os.makedirs(file.parent, exist_ok=True)
     yaml_dump(render_params(inv, cluster), file)
+
+
+class CompileMeta:
+    def __init__(self, cfg: Config):
+        self.build_info = {"version": __version__, "gitVersion": __git_version__}
+        self.instances = cfg.get_component_alias_versioninfos()
+        self.packages = cfg.get_package_versioninfos()
+        self.global_repo = cfg.global_version_info
+        self.tenant_repo = cfg.tenant_version_info
+        self.timestamp = datetime.now().astimezone(None)
+
+    def as_dict(self):
+        return {
+            "commodoreBuildInfo": self.build_info,
+            "global": self.global_repo.as_dict(),
+            "instances": {
+                a: info.as_dict() for a, info in sorted(self.instances.items())
+            },
+            "lastCompile": self.timestamp.isoformat(timespec="milliseconds"),
+            "packages": {
+                p: info.as_dict() for p, info in sorted(self.packages.items())
+            },
+            "tenant": self.tenant_repo.as_dict(),
+        }
+
+    def render_catalog_commit_message(self) -> str:
+        component_commits = [
+            info.pretty_print(i) for i, info in sorted(self.instances.items())
+        ]
+        component_commits_str = "\n".join(component_commits)
+
+        package_commits = [
+            info.pretty_print(p) for p, info in sorted(self.packages.items())
+        ]
+        package_commits_str = "\n".join(package_commits)
+
+        config_commits = [
+            self.global_repo.pretty_print("global"),
+            self.tenant_repo.pretty_print("tenant"),
+        ]
+        config_commits_str = "\n".join(config_commits)
+
+        return f"""Automated catalog update from Commodore
+
+Component instance commits:
+{component_commits_str}
+
+Package commits:
+{package_commits_str}
+
+Configuration commits:
+{config_commits_str}
+
+Compilation timestamp: {self.timestamp.isoformat(timespec="milliseconds")}
+"""
+
+
+def report_compile_metadata(
+    cfg: Config, compile_meta: CompileMeta, cluster_id: str, report=False
+):
+    if cfg.verbose:
+        if report:
+            action = "will be reported to Lieutenant"
+        else:
+            action = "would be reported to Lieutenant on a successful catalog push"
+        click.echo(
+            f" > The following compile metadata {action}:\n"
+            + textwrap.indent(json.dumps(compile_meta.as_dict(), indent=2), "    "),
+        )
+
+    if report:
+        lieutenant_post(
+            cfg.api_url,
+            cfg.api_token,
+            f"clusters/{cluster_id}",
+            "compileMeta",
+            post_data=compile_meta.as_dict(),
+        )

commodore/compile.py

@@ -9,8 +9,10 @@
 from .catalog import fetch_catalog, clean_catalog, update_catalog
 from .cluster import (
     Cluster,
+    CompileMeta,
     load_cluster_from_api,
     read_cluster_and_tenant,
+    report_compile_metadata,
     update_params,
     update_target,
 )
@@ -279,7 +281,10 @@ def compile(config, cluster_id):
 
     postprocess_components(config, inventory, config.get_components())
 
-    update_catalog(config, targets, catalog_repo)
+    compile_meta = CompileMeta(config)
+
+    push_done = update_catalog(config, targets, catalog_repo, compile_meta)
+    report_compile_metadata(config, compile_meta, cluster_id, report=push_done)
 
     click.secho("Catalog compiled! 🎉", bold=True)
 

commodore/component/__init__.py

@@ -118,6 +118,10 @@ def version(self) -> Optional[str]:
     def version(self, version: str):
         self._version = version
 
+    @property
+    def sub_path(self) -> str:
+        return self._sub_path
+
     @property
     def repo_directory(self) -> P:
         return self._dir
@@ -173,6 +177,9 @@ def checkout(self):
             )
         self._dependency.checkout_component(self.name, self.version)
 
+    def is_checked_out(self) -> bool:
+        return self.target_dir is not None and self.target_dir.is_dir()
+
     def checkout_is_dirty(self) -> bool:
         if self._dependency:
             dep_repo = self._dependency.bare_repo