Merge pull request #3011 from wolfv/test-recipe-yaml

feat: test v1 recipe.yaml migrations

conda_forge_tick/migrators/migration_yaml.py

@@ -280,7 +280,10 @@ def migrate(
                         yaml_safe_dump(cfg, fp)
 
             with pushd(recipe_dir):
-                self.set_build_number("meta.yaml")
+                if os.path.exists("recipe.yaml"):
+                    self.set_build_number("recipe.yaml")
+                else:
+                    self.set_build_number("meta.yaml")
 
             return super().migrate(recipe_dir, attrs)
 

tests/test_migrators_v1.py

@@ -0,0 +1,346 @@
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+import pytest
+from test_recipe_yaml_parsing import TEST_RECIPE_YAML_PATH
+
+from conda_forge_tick.contexts import ClonedFeedstockContext
+from conda_forge_tick.feedstock_parser import (
+    parse_recipe_yaml,
+    populate_feedstock_attributes,
+)
+from conda_forge_tick.migrators import (
+    MigrationYaml,
+    Migrator,
+    MiniMigrator,
+    Replacement,
+    Version,
+)
+from conda_forge_tick.os_utils import pushd
+from conda_forge_tick.utils import frozen_to_json_friendly
+
+
+class NoFilter:
+    def filter(self, attrs, not_bad_str_start=""):
+        return False
+
+
+class _MigrationYaml(NoFilter, MigrationYaml):
+    pass
+
+
+yaml_rebuild = _MigrationYaml(yaml_contents="hello world", name="hi")
+yaml_rebuild.cycles = []
+yaml_rebuild_no_build_number = _MigrationYaml(
+    yaml_contents="hello world",
+    name="hi",
+    bump_number=0,
+)
+yaml_rebuild_no_build_number.cycles = []
+
+
+def requirements_from_yaml(reqs: list) -> set[str]:
+    res = set()
+    for req in reqs:
+        if isinstance(req, dict):
+            if "pin_compatible" in req:
+                res.add(req["pin_compatible"]["name"])
+            elif "pin_subpackage" in req:
+                res.add(req["pin_subpackage"]["name"])
+            else:
+                # add if and else branch
+                res |= set(req["then"])
+                res |= set(req.get("else", []))
+        else:
+            res.add(req)
+
+    return res
+
+
+def run_test_yaml_migration(
+    m, *, inp, output, kwargs, prb, mr_out, tmpdir, should_filter=False, is_v1=False
+):
+    os.makedirs(os.path.join(tmpdir, "recipe"), exist_ok=True)
+
+    with open(os.path.join(tmpdir, "recipe", "recipe.yaml"), "w") as f:
+        f.write(inp)
+
+    with pushd(tmpdir):
+        subprocess.run(["git", "init"])
+    # Load the recipe.yaml (this is done in the graph)
+    try:
+        pmy = parse_recipe_yaml(inp)
+    except Exception:
+        pmy = {}
+    if pmy:
+        pmy["version"] = pmy["package"]["version"]
+        pmy["req"] = set()
+        for k in ["build", "host", "run"]:
+            reqs = requirements_from_yaml(pmy.get("requirements", {}).get(k, set()))
+            pmy["req"] |= reqs
+        try:
+            pmy["recipe_yaml"] = parse_recipe_yaml(inp)
+        except Exception:
+            pmy["recipe_yaml"] = {}
+    pmy["raw_meta_yaml"] = inp
+    pmy.update(kwargs)
+
+    assert m.filter(pmy) is should_filter
+    if should_filter:
+        return
+
+    mr = m.migrate(os.path.join(tmpdir, "recipe"), pmy)
+    assert mr_out == mr
+    pmy["pr_info"] = {}
+    pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)])
+    with open(os.path.join(tmpdir, "recipe/recipe.yaml")) as f:
+        actual_output = f.read()
+    assert actual_output == output
+    assert os.path.exists(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml"))
+    with open(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml")) as f:
+        saved_migration = f.read()
+    assert saved_migration == m.yaml_contents
+
+
+def sample_yaml_rebuild() -> str:
+    yaml = TEST_RECIPE_YAML_PATH / "scipy_migrate.yaml"
+    sample_yaml_rebuild = yaml.read_text()
+    return sample_yaml_rebuild
+
+
+def test_yaml_migration_rebuild(tmpdir):
+    """Test that the build number is bumped"""
+    sample = sample_yaml_rebuild()
+    updated_yaml_rebuild = sample.replace("number: 0", "number: 1")
+
+    run_test_yaml_migration(
+        m=yaml_rebuild,
+        inp=sample,
+        output=updated_yaml_rebuild,
+        kwargs={"feedstock_name": "scipy"},
+        prb="This PR has been triggered in an effort to update **hi**.",
+        mr_out={
+            "migrator_name": yaml_rebuild.__class__.__name__,
+            "migrator_version": yaml_rebuild.migrator_version,
+            "name": "hi",
+            "bot_rerun": False,
+        },
+        tmpdir=tmpdir,
+        is_v1=True,
+    )
+
+
+def test_yaml_migration_rebuild_no_buildno(tmpdir):
+    sample = sample_yaml_rebuild()
+
+    run_test_yaml_migration(
+        m=yaml_rebuild_no_build_number,
+        inp=sample,
+        output=sample,
+        kwargs={"feedstock_name": "scipy"},
+        prb="This PR has been triggered in an effort to update **hi**.",
+        mr_out={
+            "migrator_name": yaml_rebuild.__class__.__name__,
+            "migrator_version": yaml_rebuild.migrator_version,
+            "name": "hi",
+            "bot_rerun": False,
+        },
+        tmpdir=tmpdir,
+    )
+
+
+##################################################################
+# Run Matplotlib mini-migrator                               ###
+##################################################################
+
+version = Version(set())
+
+matplotlib = Replacement(
+    old_pkg="matplotlib",
+    new_pkg="matplotlib-base",
+    rationale=(
+        "Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`."
+    ),
+    pr_limit=5,
+)
+
+
+class MockLazyJson:
+    def __init__(self, data):
+        self.data = data
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        pass
+
+
+os.environ["RUN_URL"] = "hi world"
+
+
+def run_test_migration(
+    m: Migrator,
+    inp: str,
+    output: str,
+    kwargs: dict,
+    prb: str,
+    mr_out: dict,
+    tmpdir: str,
+    should_filter: bool = False,
+    make_body: bool = False,
+):
+    if mr_out:
+        mr_out.update(bot_rerun=False)
+
+    Path(tmpdir).joinpath("recipe.yaml").write_text(inp)
+
+    # read the conda-forge.yml
+    cf_yml_path = Path(tmpdir).parent / "conda-forge.yml"
+    cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}"
+
+    # Load the recipe.yaml (this is done in the graph)
+    try:
+        name = parse_recipe_yaml(inp)["package"]["name"]
+    except Exception:
+        name = "blah"
+
+    pmy = populate_feedstock_attributes(
+        name, sub_graph={}, recipe_yaml=inp, conda_forge_yaml=cf_yml
+    )
+
+    # these are here for legacy migrators
+    pmy["version"] = pmy["recipe_yaml"]["package"]["version"]
+    pmy["req"] = set()
+    for k in ["build", "host", "run"]:
+        reqs = requirements_from_yaml(pmy.get("requirements", {}).get(k, set()))
+        pmy["req"] |= reqs
+    pmy["raw_meta_yaml"] = inp
+    pmy.update(kwargs)
+
+    try:
+        if "new_version" in kwargs:
+            pmy["version_pr_info"] = {"new_version": kwargs["new_version"]}
+        assert m.filter(pmy) == should_filter
+    finally:
+        pmy.pop("version_pr_info", None)
+    if should_filter:
+        return pmy
+
+    m.run_pre_piggyback_migrations(
+        tmpdir,
+        pmy,
+        hash_type=pmy.get("hash_type", "sha256"),
+    )
+    mr = m.migrate(tmpdir, pmy, hash_type=pmy.get("hash_type", "sha256"))
+    m.run_post_piggyback_migrations(
+        tmpdir,
+        pmy,
+        hash_type=pmy.get("hash_type", "sha256"),
+    )
+
+    if make_body:
+        fctx = ClonedFeedstockContext(
+            feedstock_name=name,
+            attrs=pmy,
+            local_clone_dir=Path(tmpdir),
+        )
+        m.effective_graph.add_node(name)
+        m.effective_graph.nodes[name]["payload"] = MockLazyJson({})
+        m.pr_body(fctx)
+
+    assert mr_out == mr
+    if not mr:
+        return pmy
+
+    pmy["pr_info"] = {}
+    pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)])
+    with open(os.path.join(tmpdir, "recipe.yaml")) as f:
+        actual_output = f.read()
+
+    # strip jinja comments
+    pat = re.compile(r"{#.*#}")
+    actual_output = pat.sub("", actual_output)
+    output = pat.sub("", output)
+    assert actual_output == output
+    # TODO: fix subgraph here (need this to be xsh file)
+    if isinstance(m, Version):
+        pass
+    else:
+        assert prb in m.pr_body(None)
+    try:
+        if "new_version" in kwargs:
+            pmy["version_pr_info"] = {"new_version": kwargs["new_version"]}
+        assert m.filter(pmy) is True
+    finally:
+        pmy.pop("version_pr_info", None)
+
+    return pmy
+
+
+def run_minimigrator(
+    migrator: MiniMigrator,
+    inp: str,
+    output: str,
+    mr_out: dict,
+    tmpdir: str,
+    should_filter: bool = False,
+):
+    if mr_out:
+        mr_out.update(bot_rerun=False)
+    with open(os.path.join(tmpdir, "recipe.yaml"), "w") as f:
+        f.write(inp)
+
+    # read the conda-forge.yml
+    if os.path.exists(os.path.join(tmpdir, "..", "conda-forge.yml")):
+        with open(os.path.join(tmpdir, "..", "conda-forge.yml")) as fp:
+            cf_yml = fp.read()
+    else:
+        cf_yml = "{}"
+
+    # Load the recipe.yaml (this is done in the graph)
+    try:
+        name = parse_recipe_yaml(inp)["package"]["name"]
+    except Exception:
+        name = "blah"
+
+    pmy = populate_feedstock_attributes(name, {}, inp, None, cf_yml)
+    filtered = migrator.filter(pmy)
+    if should_filter and filtered:
+        return migrator
+    assert filtered == should_filter
+
+    with open(os.path.join(tmpdir, "recipe.yaml")) as f:
+        actual_output = f.read()
+    # strip jinja comments
+    pat = re.compile(r"{#.*#}")
+    actual_output = pat.sub("", actual_output)
+    output = pat.sub("", output)
+    assert actual_output == output
+
+
+def test_generic_replacement(tmpdir):
+    sample_matplotlib = TEST_RECIPE_YAML_PATH / "sample_matplotlib.yaml"
+    sample_matplotlib = sample_matplotlib.read_text()
+    sample_matplotlib_correct = sample_matplotlib.replace(
+        "    - matplotlib", "    - matplotlib-base"
+    )
+    # "recipe_yaml generic parsing not implemented yet" is raised here!
+    with pytest.raises(NotImplementedError):
+        run_test_migration(
+            m=matplotlib,
+            inp=sample_matplotlib,
+            output=sample_matplotlib_correct,
+            kwargs={},
+            prb="I noticed that this recipe depends on `matplotlib` instead of ",
+            mr_out={
+                "migrator_name": "Replacement",
+                "migrator_version": matplotlib.migrator_version,
+                "name": "matplotlib-to-matplotlib-base",
+            },
+            tmpdir=tmpdir,
+        )