From 8a1f0dbb572a4351c3cb62ee4788948a727ac928 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 08:41:14 +0200 Subject: [PATCH 01/12] add build number editing for v1 recipes --- conda_forge_tick/migrators/core.py | 25 ++++---- conda_forge_tick/update_recipe/v1/__init__.py | 3 + .../update_recipe/v1/build_number.py | 61 +++++++++++++++++++ conda_forge_tick/update_recipe/v1/yaml.py | 22 +++++++ .../build_number/test_1/expected.yaml | 10 +++ .../recipe_v1/build_number/test_1/recipe.yaml | 10 +++ .../build_number/test_2/expected.yaml | 11 ++++ .../recipe_v1/build_number/test_2/recipe.yaml | 11 ++++ tests/test_recipe_editing_v1.py | 21 +++++++ 9 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 conda_forge_tick/update_recipe/v1/__init__.py create mode 100644 conda_forge_tick/update_recipe/v1/build_number.py create mode 100644 conda_forge_tick/update_recipe/v1/yaml.py create mode 100644 tests/recipe_v1/build_number/test_1/expected.yaml create mode 100644 tests/recipe_v1/build_number/test_1/recipe.yaml create mode 100644 tests/recipe_v1/build_number/test_2/expected.yaml create mode 100644 tests/recipe_v1/build_number/test_2/recipe.yaml create mode 100644 tests/test_recipe_editing_v1.py diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index c0bfa47c9..8fc3d2550 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -5,6 +5,7 @@ import logging import re import typing +from pathlib import Path from typing import Any, List, Sequence, Set import dateutil.parser @@ -14,7 +15,7 @@ from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort -from conda_forge_tick.update_recipe import update_build_number +from conda_forge_tick.update_recipe import update_build_number, v2 from conda_forge_tick.utils import ( frozen_to_json_friendly, get_bot_run_url, @@ -592,7 +593,7 @@ def order( } return cyclic_topological_sort(graph, top_level) - def set_build_number(self, filename: str) -> None: + def set_build_number(self, filename: str | Path) -> None: """Bump the build number of the specified recipe. Parameters @@ -600,17 +601,19 @@ def set_build_number(self, filename: str) -> None: filename : str Path the the meta.yaml """ - with open(filename) as f: - raw = f.read() + filename = Path(filename) + if filename.name == "recipe.yaml": + filename.write_text(v2.update_build_number(filename, self.new_build_number)) + else: + raw = filename.read_text() - new_myaml = update_build_number( - raw, - self.new_build_number, - build_patterns=self.build_patterns, - ) + new_myaml = update_build_number( + raw, + self.new_build_number, + build_patterns=self.build_patterns, + ) - with open(filename, "w") as f: - f.write(new_myaml) + filename.write_text(new_myaml) def new_build_number(self, old_number: int) -> int: """Determine the new build number to use. diff --git a/conda_forge_tick/update_recipe/v1/__init__.py b/conda_forge_tick/update_recipe/v1/__init__.py new file mode 100644 index 000000000..b8551a6ab --- /dev/null +++ b/conda_forge_tick/update_recipe/v1/__init__.py @@ -0,0 +1,3 @@ +from .build_number import update_build_number + +__all__ = ["update_build_number"] diff --git a/conda_forge_tick/update_recipe/v1/build_number.py b/conda_forge_tick/update_recipe/v1/build_number.py new file mode 100644 index 000000000..ae328613b --- /dev/null +++ b/conda_forge_tick/update_recipe/v1/build_number.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Literal + +from conda_forge_tick.update_recipe.v1.yaml import _dump_yaml_to_str, _load_yaml + +if TYPE_CHECKING: + from pathlib import Path + +logger = logging.getLogger(__name__) + +HashType = Literal["md5", "sha256"] + + +def _update_build_number_in_context( + recipe: dict[str, Any], new_build_number: int +) -> bool: + for key in recipe.get("context", {}): + if key.startswith("build_") or key == "build": + recipe["context"][key] = new_build_number + return True + return False + + +def _update_build_number_in_recipe( + recipe: dict[str, Any], new_build_number: int +) -> bool: + is_modified = False + if "build" in recipe and "number" in recipe["build"]: + recipe["build"]["number"] = new_build_number + is_modified = True + + if "outputs" in recipe: + for output in recipe["outputs"]: + if "build" in output and "number" in output["build"]: + output["build"]["number"] = new_build_number + is_modified = True + + return is_modified + + +def update_build_number(file: Path, new_build_number: int = 0) -> str: + """ + Update the build number in the recipe file. + + Arguments: + ---------- + * `file` - The path to the recipe file. + * `new_build_number` - The new build number to use. (default: 0) + + Returns: + -------- + * The updated recipe as a string. + """ + data = _load_yaml(file) + build_number_modified = _update_build_number_in_context(data, new_build_number) + if not build_number_modified: + _update_build_number_in_recipe(data, new_build_number) + + return _dump_yaml_to_str(data) diff --git a/conda_forge_tick/update_recipe/v1/yaml.py b/conda_forge_tick/update_recipe/v1/yaml.py new file mode 100644 index 000000000..fd5990636 --- /dev/null +++ b/conda_forge_tick/update_recipe/v1/yaml.py @@ -0,0 +1,22 @@ +import io +from pathlib import Path + +from ruamel.yaml import YAML + +yaml = YAML() +yaml.preserve_quotes = True +yaml.width = 320 +yaml.indent(mapping=2, sequence=4, offset=2) + + +def _load_yaml(file: Path) -> dict: + """Load a YAML file.""" + with file.open("r") as f: + return yaml.load(f) + + +def _dump_yaml_to_str(data: dict) -> str: + """Dump a dictionary to a YAML string.""" + with io.StringIO() as f: + yaml.dump(data, f) + return f.getvalue() diff --git a/tests/recipe_v1/build_number/test_1/expected.yaml b/tests/recipe_v1/build_number/test_1/expected.yaml new file mode 100644 index 000000000..f22fd1978 --- /dev/null +++ b/tests/recipe_v1/build_number/test_1/expected.yaml @@ -0,0 +1,10 @@ +# set the build number to something +context: + build: 0 + +package: + name: recipe_1 + version: "0.1.0" + +build: + number: ${{ build }} diff --git a/tests/recipe_v1/build_number/test_1/recipe.yaml b/tests/recipe_v1/build_number/test_1/recipe.yaml new file mode 100644 index 000000000..5ea4da2af --- /dev/null +++ b/tests/recipe_v1/build_number/test_1/recipe.yaml @@ -0,0 +1,10 @@ +# set the build number to something +context: + build: 123 + +package: + name: recipe_1 + version: "0.1.0" + +build: + number: ${{ build }} diff --git a/tests/recipe_v1/build_number/test_2/expected.yaml b/tests/recipe_v1/build_number/test_2/expected.yaml new file mode 100644 index 000000000..3095c27dd --- /dev/null +++ b/tests/recipe_v1/build_number/test_2/expected.yaml @@ -0,0 +1,11 @@ +# set the build number to something +package: + name: recipe_1 + version: "0.1.0" + +# set the build number to something directly in the recipe text +build: + number: 0 + +source: + - url: foo diff --git a/tests/recipe_v1/build_number/test_2/recipe.yaml b/tests/recipe_v1/build_number/test_2/recipe.yaml new file mode 100644 index 000000000..d0906a9a5 --- /dev/null +++ b/tests/recipe_v1/build_number/test_2/recipe.yaml @@ -0,0 +1,11 @@ +# set the build number to something +package: + name: recipe_1 + version: "0.1.0" + +# set the build number to something directly in the recipe text +build: + number: 321 + +source: +- url: foo diff --git a/tests/test_recipe_editing_v1.py b/tests/test_recipe_editing_v1.py new file mode 100644 index 000000000..102a3e567 --- /dev/null +++ b/tests/test_recipe_editing_v1.py @@ -0,0 +1,21 @@ +from pathlib import Path + +import pytest + +from conda_forge_tick.update_recipe.v1 import update_build_number + + +@pytest.fixture +def data_dir() -> Path: + return Path(__file__).parent / "recipe_v1" + + +def test_build_number_mod(data_dir: Path) -> None: + tests = data_dir / "build_number" + result = update_build_number(tests / "test_1/recipe.yaml", 0) + expected = tests / "test_1/expected.yaml" + assert result == expected.read_text() + + result = update_build_number(tests / "test_2/recipe.yaml", 0) + expected = tests / "test_2/expected.yaml" + assert result == expected.read_text() From 0bb212761caabd93921f232a907dd29de2a26193 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 10:27:07 +0200 Subject: [PATCH 02/12] adjust import --- conda_forge_tick/migrators/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 8fc3d2550..d431e1704 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -15,7 +15,7 @@ from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort -from conda_forge_tick.update_recipe import update_build_number, v2 +from conda_forge_tick.update_recipe import update_build_number, v1 from conda_forge_tick.utils import ( frozen_to_json_friendly, get_bot_run_url, @@ -603,7 +603,7 @@ def set_build_number(self, filename: str | Path) -> None: """ filename = Path(filename) if filename.name == "recipe.yaml": - filename.write_text(v2.update_build_number(filename, self.new_build_number)) + filename.write_text(v1.update_build_number(filename, self.new_build_number)) else: raw = filename.read_text() From 8ca6b11499cc5b4333d704e8c9604662f9694f3c Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:05:16 +0200 Subject: [PATCH 03/12] allow callable --- .../update_recipe/v1/build_number.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/update_recipe/v1/build_number.py b/conda_forge_tick/update_recipe/v1/build_number.py index ae328613b..1a5f315d0 100644 --- a/conda_forge_tick/update_recipe/v1/build_number.py +++ b/conda_forge_tick/update_recipe/v1/build_number.py @@ -1,7 +1,8 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Literal +import re +from typing import TYPE_CHECKING, Any, Callable, Literal from conda_forge_tick.update_recipe.v1.yaml import _dump_yaml_to_str, _load_yaml @@ -12,6 +13,26 @@ HashType = Literal["md5", "sha256"] +RE_PATTERN = re.compile(r"(?:build|build_number|number):\s*(\d+)") + + +def old_build_number(recipe_text: str) -> int: + """ + Extract the build number from the recipe text. + + Arguments: + ---------- + * `recipe_text` - The recipe text. + + Returns: + -------- + * The build number. + """ + match = re.search(RE_PATTERN, recipe_text) + if match is not None: + return int(match.group(1)) + return 0 + def _update_build_number_in_context( recipe: dict[str, Any], new_build_number: int @@ -40,7 +61,7 @@ def _update_build_number_in_recipe( return is_modified -def update_build_number(file: Path, new_build_number: int = 0) -> str: +def update_build_number(file: Path, new_build_number: int | Callable = 0) -> str: """ Update the build number in the recipe file. @@ -54,7 +75,13 @@ def update_build_number(file: Path, new_build_number: int = 0) -> str: * The updated recipe as a string. """ data = _load_yaml(file) + + if callable(new_build_number): + detected_build_number = old_build_number(file.read_text()) + new_build_number = new_build_number(detected_build_number) + build_number_modified = _update_build_number_in_context(data, new_build_number) + if not build_number_modified: _update_build_number_in_recipe(data, new_build_number) From 5753390b9325460f9b946535b19237de0ae358e6 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:09:49 +0200 Subject: [PATCH 04/12] fix precommit --- conda_forge_tick/recipe_parser/_parser.py | 2 +- conda_forge_tick/update_recipe/v1/__init__.py | 3 - .../update_recipe/v1/build_number.py | 88 ------------------- conda_forge_tick/update_recipe/v1/yaml.py | 22 ----- 4 files changed, 1 insertion(+), 114 deletions(-) delete mode 100644 conda_forge_tick/update_recipe/v1/__init__.py delete mode 100644 conda_forge_tick/update_recipe/v1/build_number.py delete mode 100644 conda_forge_tick/update_recipe/v1/yaml.py diff --git a/conda_forge_tick/recipe_parser/_parser.py b/conda_forge_tick/recipe_parser/_parser.py index 6f6aa5f68..89531a64d 100644 --- a/conda_forge_tick/recipe_parser/_parser.py +++ b/conda_forge_tick/recipe_parser/_parser.py @@ -41,7 +41,7 @@ BAD_MULTILINE_STRING_WITH_SELECTOR = re.compile(r"[^|#]*\|\s+#") -def _get_yaml_parser(): +def _get_yaml_parser(typ="jinja2"): """yaml parser that is jinja2 aware""" # using a function here so settings are always the same parser = YAML(typ="jinja2") diff --git a/conda_forge_tick/update_recipe/v1/__init__.py b/conda_forge_tick/update_recipe/v1/__init__.py deleted file mode 100644 index b8551a6ab..000000000 --- a/conda_forge_tick/update_recipe/v1/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .build_number import update_build_number - -__all__ = ["update_build_number"] diff --git a/conda_forge_tick/update_recipe/v1/build_number.py b/conda_forge_tick/update_recipe/v1/build_number.py deleted file mode 100644 index 1a5f315d0..000000000 --- a/conda_forge_tick/update_recipe/v1/build_number.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import TYPE_CHECKING, Any, Callable, Literal - -from conda_forge_tick.update_recipe.v1.yaml import _dump_yaml_to_str, _load_yaml - -if TYPE_CHECKING: - from pathlib import Path - -logger = logging.getLogger(__name__) - -HashType = Literal["md5", "sha256"] - -RE_PATTERN = re.compile(r"(?:build|build_number|number):\s*(\d+)") - - -def old_build_number(recipe_text: str) -> int: - """ - Extract the build number from the recipe text. - - Arguments: - ---------- - * `recipe_text` - The recipe text. - - Returns: - -------- - * The build number. - """ - match = re.search(RE_PATTERN, recipe_text) - if match is not None: - return int(match.group(1)) - return 0 - - -def _update_build_number_in_context( - recipe: dict[str, Any], new_build_number: int -) -> bool: - for key in recipe.get("context", {}): - if key.startswith("build_") or key == "build": - recipe["context"][key] = new_build_number - return True - return False - - -def _update_build_number_in_recipe( - recipe: dict[str, Any], new_build_number: int -) -> bool: - is_modified = False - if "build" in recipe and "number" in recipe["build"]: - recipe["build"]["number"] = new_build_number - is_modified = True - - if "outputs" in recipe: - for output in recipe["outputs"]: - if "build" in output and "number" in output["build"]: - output["build"]["number"] = new_build_number - is_modified = True - - return is_modified - - -def update_build_number(file: Path, new_build_number: int | Callable = 0) -> str: - """ - Update the build number in the recipe file. - - Arguments: - ---------- - * `file` - The path to the recipe file. - * `new_build_number` - The new build number to use. (default: 0) - - Returns: - -------- - * The updated recipe as a string. - """ - data = _load_yaml(file) - - if callable(new_build_number): - detected_build_number = old_build_number(file.read_text()) - new_build_number = new_build_number(detected_build_number) - - build_number_modified = _update_build_number_in_context(data, new_build_number) - - if not build_number_modified: - _update_build_number_in_recipe(data, new_build_number) - - return _dump_yaml_to_str(data) diff --git a/conda_forge_tick/update_recipe/v1/yaml.py b/conda_forge_tick/update_recipe/v1/yaml.py deleted file mode 100644 index fd5990636..000000000 --- a/conda_forge_tick/update_recipe/v1/yaml.py +++ /dev/null @@ -1,22 +0,0 @@ -import io -from pathlib import Path - -from ruamel.yaml import YAML - -yaml = YAML() -yaml.preserve_quotes = True -yaml.width = 320 -yaml.indent(mapping=2, sequence=4, offset=2) - - -def _load_yaml(file: Path) -> dict: - """Load a YAML file.""" - with file.open("r") as f: - return yaml.load(f) - - -def _dump_yaml_to_str(data: dict) -> str: - """Dump a dictionary to a YAML string.""" - with io.StringIO() as f: - yaml.dump(data, f) - return f.getvalue() From bad38e3aa3d569e2ffb9324c514887b07fe651cc Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:14:55 +0200 Subject: [PATCH 05/12] rename to v1_recipe and remove yaml module --- conda_forge_tick/recipe_parser/_parser.py | 2 +- .../update_recipe/v1_recipe/__init__.py | 3 + .../update_recipe/v1_recipe/build_number.py | 102 ++++++++++++++++++ tests/test_recipe_editing_v1.py | 2 +- 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 conda_forge_tick/update_recipe/v1_recipe/__init__.py create mode 100644 conda_forge_tick/update_recipe/v1_recipe/build_number.py diff --git a/conda_forge_tick/recipe_parser/_parser.py b/conda_forge_tick/recipe_parser/_parser.py index 89531a64d..f30a8225d 100644 --- a/conda_forge_tick/recipe_parser/_parser.py +++ b/conda_forge_tick/recipe_parser/_parser.py @@ -44,7 +44,7 @@ def _get_yaml_parser(typ="jinja2"): """yaml parser that is jinja2 aware""" # using a function here so settings are always the same - parser = YAML(typ="jinja2") + parser = YAML(typ=typ) parser.indent(mapping=2, sequence=4, offset=2) parser.width = 320 parser.preserve_quotes = True diff --git a/conda_forge_tick/update_recipe/v1_recipe/__init__.py b/conda_forge_tick/update_recipe/v1_recipe/__init__.py new file mode 100644 index 000000000..b8551a6ab --- /dev/null +++ b/conda_forge_tick/update_recipe/v1_recipe/__init__.py @@ -0,0 +1,3 @@ +from .build_number import update_build_number + +__all__ = ["update_build_number"] diff --git a/conda_forge_tick/update_recipe/v1_recipe/build_number.py b/conda_forge_tick/update_recipe/v1_recipe/build_number.py new file mode 100644 index 000000000..0553d7ae9 --- /dev/null +++ b/conda_forge_tick/update_recipe/v1_recipe/build_number.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import logging +import re +import io +from typing import TYPE_CHECKING, Any, Callable, Literal +from conda_forge_tick.recipe_parser._parser import _get_yaml_parser + +if TYPE_CHECKING: + from pathlib import Path + +logger = logging.getLogger(__name__) + +HashType = Literal["md5", "sha256"] + +RE_PATTERN = re.compile(r"(?:build|build_number|number):\s*(\d+)") + + +def old_build_number(recipe_text: str) -> int: + """ + Extract the build number from the recipe text. + + Arguments: + ---------- + * `recipe_text` - The recipe text. + + Returns: + -------- + * The build number. + """ + match = re.search(RE_PATTERN, recipe_text) + if match is not None: + return int(match.group(1)) + return 0 + + +def _update_build_number_in_context( + recipe: dict[str, Any], new_build_number: int +) -> bool: + for key in recipe.get("context", {}): + if key in {"build_number", "build", "number"}: + recipe["context"][key] = new_build_number + return True + return False + + +def _update_build_number_in_recipe( + recipe: dict[str, Any], new_build_number: int +) -> bool: + is_modified = False + if "build" in recipe and "number" in recipe["build"]: + recipe["build"]["number"] = new_build_number + is_modified = True + + if "outputs" in recipe: + for output in recipe["outputs"]: + if "build" in output and "number" in output["build"]: + output["build"]["number"] = new_build_number + is_modified = True + + return is_modified + + +def _load_yaml(file: Path): + yaml = _get_yaml_parser(typ=None) + with file.open("r") as f: + return yaml.load(f) + + +def _dump_yaml_to_str(data: dict) -> str: + """Dump a dictionary to a YAML string.""" + yaml = _get_yaml_parser(typ=None) + with io.StringIO() as f: + yaml.dump(data, f) + return f.getvalue() + + +def update_build_number(file: Path, new_build_number: int | Callable = 0) -> str: + """ + Update the build number in the recipe file. + + Arguments: + ---------- + * `file` - The path to the recipe file. + * `new_build_number` - The new build number to use. (default: 0) + + Returns: + -------- + * The updated recipe as a string. + """ + data = _load_yaml(file) + + if callable(new_build_number): + detected_build_number = old_build_number(file.read_text()) + new_build_number = new_build_number(detected_build_number) + + build_number_modified = _update_build_number_in_context(data, new_build_number) + + if not build_number_modified: + _update_build_number_in_recipe(data, new_build_number) + + return _dump_yaml_to_str(data) diff --git a/tests/test_recipe_editing_v1.py b/tests/test_recipe_editing_v1.py index 102a3e567..9ea532be4 100644 --- a/tests/test_recipe_editing_v1.py +++ b/tests/test_recipe_editing_v1.py @@ -2,7 +2,7 @@ import pytest -from conda_forge_tick.update_recipe.v1 import update_build_number +from conda_forge_tick.update_recipe.v1_recipe import update_build_number @pytest.fixture From dc769fc2882d6cca2badd4f788af0f04c1d5eab2 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:15:25 +0200 Subject: [PATCH 06/12] fix lint --- conda_forge_tick/update_recipe/v1_recipe/build_number.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/v1_recipe/build_number.py b/conda_forge_tick/update_recipe/v1_recipe/build_number.py index 0553d7ae9..a57c02197 100644 --- a/conda_forge_tick/update_recipe/v1_recipe/build_number.py +++ b/conda_forge_tick/update_recipe/v1_recipe/build_number.py @@ -1,9 +1,10 @@ from __future__ import annotations +import io import logging import re -import io from typing import TYPE_CHECKING, Any, Callable, Literal + from conda_forge_tick.recipe_parser._parser import _get_yaml_parser if TYPE_CHECKING: From 4fd1233d6308fb24a2123f2662d8680984b033ee Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:18:22 +0200 Subject: [PATCH 07/12] fix import --- conda_forge_tick/migrators/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index d431e1704..055b0daed 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -15,7 +15,7 @@ from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort -from conda_forge_tick.update_recipe import update_build_number, v1 +from conda_forge_tick.update_recipe import update_build_number, v1_recipe from conda_forge_tick.utils import ( frozen_to_json_friendly, get_bot_run_url, @@ -603,7 +603,7 @@ def set_build_number(self, filename: str | Path) -> None: """ filename = Path(filename) if filename.name == "recipe.yaml": - filename.write_text(v1.update_build_number(filename, self.new_build_number)) + filename.write_text(v1_recipe.update_build_number(filename, self.new_build_number)) else: raw = filename.read_text() From b971072c3ad14f40b2b525b8e01c05c51a145f13 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 17:18:31 +0200 Subject: [PATCH 08/12] fix lint --- conda_forge_tick/migrators/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 055b0daed..9dbf91e1e 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -603,7 +603,9 @@ def set_build_number(self, filename: str | Path) -> None: """ filename = Path(filename) if filename.name == "recipe.yaml": - filename.write_text(v1_recipe.update_build_number(filename, self.new_build_number)) + filename.write_text( + v1_recipe.update_build_number(filename, self.new_build_number) + ) else: raw = filename.read_text() From c0dccc381b28677d9834d6e51baedb30665229b7 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Tue, 24 Sep 2024 11:07:10 -0500 Subject: [PATCH 09/12] Update conda_forge_tick/update_recipe/v1_recipe/build_number.py --- conda_forge_tick/update_recipe/v1_recipe/build_number.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/v1_recipe/build_number.py b/conda_forge_tick/update_recipe/v1_recipe/build_number.py index a57c02197..d52ef9ee5 100644 --- a/conda_forge_tick/update_recipe/v1_recipe/build_number.py +++ b/conda_forge_tick/update_recipe/v1_recipe/build_number.py @@ -63,7 +63,7 @@ def _update_build_number_in_recipe( def _load_yaml(file: Path): - yaml = _get_yaml_parser(typ=None) + yaml = _get_yaml_parser(typ="safe") with file.open("r") as f: return yaml.load(f) From ebec1468433526cd425dad7d155f95593861b5a2 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 24 Sep 2024 19:24:57 +0200 Subject: [PATCH 10/12] use rt loader --- conda_forge_tick/update_recipe/v1_recipe/build_number.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/update_recipe/v1_recipe/build_number.py b/conda_forge_tick/update_recipe/v1_recipe/build_number.py index d52ef9ee5..131e88f10 100644 --- a/conda_forge_tick/update_recipe/v1_recipe/build_number.py +++ b/conda_forge_tick/update_recipe/v1_recipe/build_number.py @@ -63,14 +63,14 @@ def _update_build_number_in_recipe( def _load_yaml(file: Path): - yaml = _get_yaml_parser(typ="safe") + yaml = _get_yaml_parser(typ="rt") with file.open("r") as f: return yaml.load(f) def _dump_yaml_to_str(data: dict) -> str: """Dump a dictionary to a YAML string.""" - yaml = _get_yaml_parser(typ=None) + yaml = _get_yaml_parser(typ="rt") with io.StringIO() as f: yaml.dump(data, f) return f.getvalue() From 6b670c752601a5c34267a30a4f44c3ee3041607b Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 25 Sep 2024 10:06:58 +0200 Subject: [PATCH 11/12] simplify --- conda_forge_tick/migrators/migration_yaml.py | 5 +- tests/test_migrators_v1.py | 346 ++++++++++++++++++ tests/test_recipe_yaml/sample_matplotlib.yaml | 43 +++ tests/test_recipe_yaml/scipy_migrate.yaml | 71 ++++ 4 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 tests/test_migrators_v1.py create mode 100644 tests/test_recipe_yaml/sample_matplotlib.yaml create mode 100644 tests/test_recipe_yaml/scipy_migrate.yaml diff --git a/conda_forge_tick/migrators/migration_yaml.py b/conda_forge_tick/migrators/migration_yaml.py index 7a4ce273d..7840f0a82 100644 --- a/conda_forge_tick/migrators/migration_yaml.py +++ b/conda_forge_tick/migrators/migration_yaml.py @@ -280,7 +280,10 @@ def migrate( yaml_safe_dump(cfg, fp) with pushd(recipe_dir): - self.set_build_number("meta.yaml") + if os.path.exists("recipe.yaml"): + self.set_build_number("recipe.yaml") + else: + self.set_build_number("meta.yaml") return super().migrate(recipe_dir, attrs) diff --git a/tests/test_migrators_v1.py b/tests/test_migrators_v1.py new file mode 100644 index 000000000..0cf98b78c --- /dev/null +++ b/tests/test_migrators_v1.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path + +import pytest +from test_recipe_yaml_parsing import TEST_RECIPE_YAML_PATH + +from conda_forge_tick.contexts import ClonedFeedstockContext +from conda_forge_tick.feedstock_parser import ( + parse_recipe_yaml, + populate_feedstock_attributes, +) +from conda_forge_tick.migrators import ( + MigrationYaml, + Migrator, + MiniMigrator, + Replacement, + Version, +) +from conda_forge_tick.os_utils import pushd +from conda_forge_tick.utils import frozen_to_json_friendly + + +class NoFilter: + def filter(self, attrs, not_bad_str_start=""): + return False + + +class _MigrationYaml(NoFilter, MigrationYaml): + pass + + +yaml_rebuild = _MigrationYaml(yaml_contents="hello world", name="hi") +yaml_rebuild.cycles = [] +yaml_rebuild_no_build_number = _MigrationYaml( + yaml_contents="hello world", + name="hi", + bump_number=0, +) +yaml_rebuild_no_build_number.cycles = [] + + +def requirements_from_yaml(reqs: list) -> set[str]: + res = set() + for req in reqs: + if isinstance(req, dict): + if "pin_compatible" in req: + res.add(req["pin_compatible"]["name"]) + elif "pin_subpackage" in req: + res.add(req["pin_subpackage"]["name"]) + else: + # add if and else branch + res |= set(req["then"]) + res |= set(req.get("else", [])) + else: + res.add(req) + + return res + + +def run_test_yaml_migration( + m, *, inp, output, kwargs, prb, mr_out, tmpdir, should_filter=False, is_v1=False +): + os.makedirs(os.path.join(tmpdir, "recipe"), exist_ok=True) + + with open(os.path.join(tmpdir, "recipe", "recipe.yaml"), "w") as f: + f.write(inp) + + with pushd(tmpdir): + subprocess.run(["git", "init"]) + # Load the recipe.yaml (this is done in the graph) + try: + pmy = parse_recipe_yaml(inp) + except Exception: + pmy = {} + if pmy: + pmy["version"] = pmy["package"]["version"] + pmy["req"] = set() + for k in ["build", "host", "run"]: + reqs = requirements_from_yaml(pmy.get("requirements", {}).get(k, set())) + pmy["req"] |= reqs + try: + pmy["recipe_yaml"] = parse_recipe_yaml(inp) + except Exception: + pmy["recipe_yaml"] = {} + pmy["raw_meta_yaml"] = inp + pmy.update(kwargs) + + assert m.filter(pmy) is should_filter + if should_filter: + return + + mr = m.migrate(os.path.join(tmpdir, "recipe"), pmy) + assert mr_out == mr + pmy["pr_info"] = {} + pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)]) + with open(os.path.join(tmpdir, "recipe/recipe.yaml")) as f: + actual_output = f.read() + assert actual_output == output + assert os.path.exists(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml")) + with open(os.path.join(tmpdir, ".ci_support/migrations/hi.yaml")) as f: + saved_migration = f.read() + assert saved_migration == m.yaml_contents + + +def sample_yaml_rebuild() -> str: + yaml = TEST_RECIPE_YAML_PATH / "scipy_migrate.yaml" + sample_yaml_rebuild = yaml.read_text() + return sample_yaml_rebuild + + +def test_yaml_migration_rebuild(tmpdir): + """Test that the build number is bumped""" + sample = sample_yaml_rebuild() + updated_yaml_rebuild = sample.replace("number: 0", "number: 1") + + run_test_yaml_migration( + m=yaml_rebuild, + inp=sample, + output=updated_yaml_rebuild, + kwargs={"feedstock_name": "scipy"}, + prb="This PR has been triggered in an effort to update **hi**.", + mr_out={ + "migrator_name": yaml_rebuild.__class__.__name__, + "migrator_version": yaml_rebuild.migrator_version, + "name": "hi", + "bot_rerun": False, + }, + tmpdir=tmpdir, + is_v1=True, + ) + + +def test_yaml_migration_rebuild_no_buildno(tmpdir): + sample = sample_yaml_rebuild() + + run_test_yaml_migration( + m=yaml_rebuild_no_build_number, + inp=sample, + output=sample, + kwargs={"feedstock_name": "scipy"}, + prb="This PR has been triggered in an effort to update **hi**.", + mr_out={ + "migrator_name": yaml_rebuild.__class__.__name__, + "migrator_version": yaml_rebuild.migrator_version, + "name": "hi", + "bot_rerun": False, + }, + tmpdir=tmpdir, + ) + + +################################################################## +# Run Matplotlib mini-migrator ### +################################################################## + +version = Version(set()) + +matplotlib = Replacement( + old_pkg="matplotlib", + new_pkg="matplotlib-base", + rationale=( + "Unless you need `pyqt`, recipes should depend only on " "`matplotlib-base`." + ), + pr_limit=5, +) + + +class MockLazyJson: + def __init__(self, data): + self.data = data + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +os.environ["RUN_URL"] = "hi world" + + +def run_test_migration( + m: Migrator, + inp: str, + output: str, + kwargs: dict, + prb: str, + mr_out: dict, + tmpdir: str, + should_filter: bool = False, + make_body: bool = False, +): + if mr_out: + mr_out.update(bot_rerun=False) + + Path(tmpdir).joinpath("recipe.yaml").write_text(inp) + + # read the conda-forge.yml + cf_yml_path = Path(tmpdir).parent / "conda-forge.yml" + cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" + + # Load the recipe.yaml (this is done in the graph) + try: + name = parse_recipe_yaml(inp)["package"]["name"] + except Exception: + name = "blah" + + pmy = populate_feedstock_attributes( + name, sub_graph={}, recipe_yaml=inp, conda_forge_yaml=cf_yml + ) + + # these are here for legacy migrators + pmy["version"] = pmy["recipe_yaml"]["package"]["version"] + pmy["req"] = set() + for k in ["build", "host", "run"]: + reqs = requirements_from_yaml(pmy.get("requirements", {}).get(k, set())) + pmy["req"] |= reqs + pmy["raw_meta_yaml"] = inp + pmy.update(kwargs) + + try: + if "new_version" in kwargs: + pmy["version_pr_info"] = {"new_version": kwargs["new_version"]} + assert m.filter(pmy) == should_filter + finally: + pmy.pop("version_pr_info", None) + if should_filter: + return pmy + + m.run_pre_piggyback_migrations( + tmpdir, + pmy, + hash_type=pmy.get("hash_type", "sha256"), + ) + mr = m.migrate(tmpdir, pmy, hash_type=pmy.get("hash_type", "sha256")) + m.run_post_piggyback_migrations( + tmpdir, + pmy, + hash_type=pmy.get("hash_type", "sha256"), + ) + + if make_body: + fctx = ClonedFeedstockContext( + feedstock_name=name, + attrs=pmy, + local_clone_dir=Path(tmpdir), + ) + m.effective_graph.add_node(name) + m.effective_graph.nodes[name]["payload"] = MockLazyJson({}) + m.pr_body(fctx) + + assert mr_out == mr + if not mr: + return pmy + + pmy["pr_info"] = {} + pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)]) + with open(os.path.join(tmpdir, "recipe.yaml")) as f: + actual_output = f.read() + + # strip jinja comments + pat = re.compile(r"{#.*#}") + actual_output = pat.sub("", actual_output) + output = pat.sub("", output) + assert actual_output == output + # TODO: fix subgraph here (need this to be xsh file) + if isinstance(m, Version): + pass + else: + assert prb in m.pr_body(None) + try: + if "new_version" in kwargs: + pmy["version_pr_info"] = {"new_version": kwargs["new_version"]} + assert m.filter(pmy) is True + finally: + pmy.pop("version_pr_info", None) + + return pmy + + +def run_minimigrator( + migrator: MiniMigrator, + inp: str, + output: str, + mr_out: dict, + tmpdir: str, + should_filter: bool = False, +): + if mr_out: + mr_out.update(bot_rerun=False) + with open(os.path.join(tmpdir, "recipe.yaml"), "w") as f: + f.write(inp) + + # read the conda-forge.yml + if os.path.exists(os.path.join(tmpdir, "..", "conda-forge.yml")): + with open(os.path.join(tmpdir, "..", "conda-forge.yml")) as fp: + cf_yml = fp.read() + else: + cf_yml = "{}" + + # Load the recipe.yaml (this is done in the graph) + try: + name = parse_recipe_yaml(inp)["package"]["name"] + except Exception: + name = "blah" + + pmy = populate_feedstock_attributes(name, {}, inp, None, cf_yml) + filtered = migrator.filter(pmy) + if should_filter and filtered: + return migrator + assert filtered == should_filter + + with open(os.path.join(tmpdir, "recipe.yaml")) as f: + actual_output = f.read() + # strip jinja comments + pat = re.compile(r"{#.*#}") + actual_output = pat.sub("", actual_output) + output = pat.sub("", output) + assert actual_output == output + + +def test_generic_replacement(tmpdir): + sample_matplotlib = TEST_RECIPE_YAML_PATH / "sample_matplotlib.yaml" + sample_matplotlib = sample_matplotlib.read_text() + sample_matplotlib_correct = sample_matplotlib.replace( + " - matplotlib", " - matplotlib-base" + ) + # "recipe_yaml generic parsing not implemented yet" is raised here! + with pytest.raises(NotImplementedError): + run_test_migration( + m=matplotlib, + inp=sample_matplotlib, + output=sample_matplotlib_correct, + kwargs={}, + prb="I noticed that this recipe depends on `matplotlib` instead of ", + mr_out={ + "migrator_name": "Replacement", + "migrator_version": matplotlib.migrator_version, + "name": "matplotlib-to-matplotlib-base", + }, + tmpdir=tmpdir, + ) diff --git a/tests/test_recipe_yaml/sample_matplotlib.yaml b/tests/test_recipe_yaml/sample_matplotlib.yaml new file mode 100644 index 000000000..85212efa2 --- /dev/null +++ b/tests/test_recipe_yaml/sample_matplotlib.yaml @@ -0,0 +1,43 @@ +context: + version: "0.9" + +package: + name: viscm + version: ${{ version }} + +source: + url: https://pypi.io/packages/source/v/viscm/viscm-${{ version }}.tar.gz + sha256: c770e4b76f726e653d2b7c2c73f71941a88de6eb47ccf8fb8e984b55562d05a2 + +build: + number: 0 + noarch: python + script: python -m pip install --no-deps --ignore-installed . + +requirements: + host: + - python + - pip + - numpy + run: + - python + - numpy + - matplotlib + - colorspacious + +tests: + - python: + imports: + - viscm + +about: + homepage: https://github.com/bids/viscm + license: MIT + license_file: LICENSE + license_family: MIT + # license_file: '' we need to an issue upstream to get a license in the source dist. + summary: A colormap tool + +extra: + recipe-maintainers: + - kthyng diff --git a/tests/test_recipe_yaml/scipy_migrate.yaml b/tests/test_recipe_yaml/scipy_migrate.yaml new file mode 100644 index 000000000..f20d54706 --- /dev/null +++ b/tests/test_recipe_yaml/scipy_migrate.yaml @@ -0,0 +1,71 @@ +context: + version: "1.3.2" + +package: + name: scipy + version: ${{ version }} + +source: + url: https://github.com/scipy/scipy/archive/v${{ version }}.tar.gz + sha256: ac0937d29a3f93cc26737fdf318c09408e9a48adee1648a25d0cdce5647b8eb4 + patches: + - gh10591.patch + - ${{ "relax_gmres_error_check.patch" if aarch64 }} + - ${{ "skip_problematic_boost_test.patch" if aarch64 or ppc64le }} + - ${{ "skip_problematic_root_finding.patch" if aarch64 or ppc64le }} + # remove this patch when updating to 1.3.3 + - if: version == "1.3.2" + then: + - ${{ "scipy-1.3.2-bad-tests.patch" if osx and match(python, "3.8") }} + - ${{ "gh11046.patch" if ppc64le }} + +build: + number: 0 + skip: + - win or py2k + +requirements: + build: + - ${{ compiler('fortran') }} + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + host: + - libblas + - libcblas + - liblapack + - python + - setuptools + - cython + - numpy + - pip + run: + - python + - ${{ pin_compatible('numpy') }} + +tests: + - requirements: + run: + - pytest + - pytest-xdist + - mpmath + - ${{ "blas * netlib" if ppc64le and version == "1.3.2" }} + script: pytest ./foo + +about: + homepage: http://www.scipy.org/ + license: BSD-3-Clause + license_file: LICENSE.txt + summary: Scientific Library for Python + description: | + SciPy is a Python-based ecosystem of open-source software for mathematics, + science, and engineering. + documentation: http://www.scipy.org/docs.html + repository: https://github.com/scipy/scipy + +extra: + recipe-maintainers: + - jakirkham + - msarahan + - rgommers + - ocefpaf + - beckermr From 3312df034458643de3d4e5d9aa179e83c14f53ea Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Wed, 25 Sep 2024 05:51:32 -0500 Subject: [PATCH 12/12] fix: ensure RUN_URL is passed to container --- conda_forge_tick/migration_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_forge_tick/migration_runner.py b/conda_forge_tick/migration_runner.py index 18a9df01c..e07127a81 100644 --- a/conda_forge_tick/migration_runner.py +++ b/conda_forge_tick/migration_runner.py @@ -171,6 +171,7 @@ def run_migration_containerized( if isinstance(node_attrs, LazyJson) else dumps(node_attrs) ), + extra_container_args=["-e", "RUN_URL"], ) sync_dirs(