From f6a830e0a6bf4159610131e3d1e101b7ae728966 Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jan 2025 15:20:21 +0100 Subject: [PATCH 01/40] Add version selshaurl for recipe v1 --- tests/test_recipe_yaml/version_selshaurl.yaml | 51 +++++++++++++++++++ .../version_selshaurl_correct.yaml | 49 ++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 tests/test_recipe_yaml/version_selshaurl.yaml create mode 100644 tests/test_recipe_yaml/version_selshaurl_correct.yaml diff --git a/tests/test_recipe_yaml/version_selshaurl.yaml b/tests/test_recipe_yaml/version_selshaurl.yaml new file mode 100644 index 000000000..b724539e8 --- /dev/null +++ b/tests/test_recipe_yaml/version_selshaurl.yaml @@ -0,0 +1,51 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +schema_version: 1 + +context: + name: embree + version: 2.14.0 + +package: + name: ${{ name }} + version: ${{ version }} + +source: + - if: linux + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz + sha256: 8cca2d7ef6e3f18668246c9eed609e03e720e4033d069164c991c5feb078443c + file_name: ${{ name }}-${{ version }}.tar.gz + - if: osx + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.tar.gz + sha256: f6113506bab9430f98773b0ab7776efe387f4d40c8785d8f8c427a91c36f4cfe + file_name: ${{ name }}-${{ version }}.tar.gz + - if: win + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip + sha256: 6432449254f56b037f17d876403f919f6c136cc16ec91f7778001b10eea115ac + file_name: ${{ name }}-${{ version }}.zip + +build: + number: 100 + prefix_detection: + ignore_binary_files: true + script: + - mkdir doc + - touch doc/LICENSE.txt + +requirements: + build: + - if: win + then: python + +about: + license: Apache-2.0 + license_file: doc/LICENSE.txt + summary: High Performance Ray Tracing Kernels + homepage: https://embree.github.io/ + +extra: + recipe-maintainers: + - scopatz + - Xarthisius diff --git a/tests/test_recipe_yaml/version_selshaurl_correct.yaml b/tests/test_recipe_yaml/version_selshaurl_correct.yaml new file mode 100644 index 000000000..61b238e36 --- /dev/null +++ b/tests/test_recipe_yaml/version_selshaurl_correct.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +schema_version: 1 + +context: + name: embree + version: 3.7.0 + +package: + name: ${{ name }} + version: ${{ version }} + +source: + - if: linux + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz + sha256: 671a3aa7cc1c8501f1290dd051b42a337a692ea6552a07436779439d649e3e29 + file_name: ${{ name }}-${{ version }}.tar.gz + - if: osx + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.tar.gz + sha256: 17c31f67efb9afc3ed658fcaa5886bc10c6f67f1e364d6494e494d189d8b8c70 + file_name: ${{ name }}-${{ version }}.tar.gz + - if: win + then: + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip + sha256: 442c8933fa3a21d66c0459ded83e1a4c896b1a26c4e46ea62e65ffbfec273be2 + file_name: ${{ name }}-${{ version }}.zip + +build: + number: 0 + prefix_detection: + ignore_binary_files: true + script: + - mkdir doc + - touch doc/LICENSE.txt + +requirements: + build: + - if: win + then: python + +about: + summary: High Performance Ray Tracing Kernels + homepage: https://embree.github.io/ + +extra: + recipe-maintainers: + - scopatz + - Xarthisius From 448aa395134c53567547d6aad743fdb3530b2015 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Mon, 6 Jan 2025 18:46:24 +0100 Subject: [PATCH 02/40] add version bumping for v1 recipes --- conda_forge_tick/update_recipe/version.py | 114 ++++++++++++++++++++-- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 1a58fad73..de5f524b5 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -10,6 +10,7 @@ import tempfile import traceback from typing import Any, MutableMapping +from pathlib import Path import jinja2 import jinja2.sandbox @@ -116,7 +117,7 @@ def _compile_all_selectors(cmeta: Any, src: str): return set(selectors) -def _try_url_and_hash_it(url: str, hash_type: str): +def _try_url_and_hash_it(url: str, hash_type: str) -> str | None: logger.debug("downloading url: %s", url) try: @@ -543,6 +544,7 @@ def update_version_feedstock_dir( A set of strings giving any errors found when updating the version. The set will be empty if there were no errors. """ + feedstock_dir = Path(feedstock_dir) if should_use_container(use_container=use_container): return _update_version_feedstock_dir_containerized( feedstock_dir, @@ -557,20 +559,24 @@ def update_version_feedstock_dir( ) -def _update_version_feedstock_dir_local(feedstock_dir, version, hash_type): - with open(os.path.join(feedstock_dir, "recipe", "meta.yaml")) as f: +def _update_version_feedstock_dir_local(feedstock_dir, version, hash_type) -> (bool, set): + if os.path.join(feedstock_dir, "recipe", "recipe.yaml"): + return update_version_v1(feedstock_dir, version, hash_type) + + recipe_path = os.path.join(feedstock_dir, "recipe", "meta.yaml") + with open(recipe_path) as f: raw_meta_yaml = f.read() updated_meta_yaml, errors = update_version( raw_meta_yaml, version, hash_type=hash_type ) if updated_meta_yaml is not None: - with open(os.path.join(feedstock_dir, "recipe", "meta.yaml"), "w") as f: + with open(recipe_path, "w") as f: f.write(updated_meta_yaml) return updated_meta_yaml is not None, errors -def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_type): +def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_type, recipe_version: int = 0): with tempfile.TemporaryDirectory() as tmpdir: tmp_feedstock_dir = os.path.join(tmpdir, os.path.basename(feedstock_dir)) sync_dirs( @@ -625,7 +631,73 @@ def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_typ return data["updated"], data["errors"] -def update_version(raw_meta_yaml, version, hash_type="sha256"): +def update_version_v1(feedstock_dir: str, version: str, hash_type: str) -> (bool , set[str]): + """Update the version in a recipe. + + Parameters + ---------- + raw_meta_yaml : str + The recipe meta.yaml as a string. + version : str + The version of the recipe. + hash_type : str + The kind of hash used on the source. + """ + # extract all the URL sources from a given recipe / feedstock directory + from rattler_build_conda_compat.loader import load_yaml + from rattler_build_conda_compat.recipe_sources import render_all_sources + + feedstock_dir = Path(feedstock_dir) + recipe_path = feedstock_dir / "recipe" / "recipe.yaml" + recipe = load_yaml(recipe_path.read_text()) + variants = feedstock_dir.glob(".ci_support/*.yaml") + # load all variants + variants = [load_yaml(variant.read_text()) for variant in variants] + + rendered_sources = render_all_sources(recipe, variants, override_version=version) + + print("rendered_sources", rendered_sources) + + # load recipe text + recipe_path = feedstock_dir / "recipe" / "recipe.yaml" + recipe = recipe_path.read_text() + + # update the version with a regex replace + for line in recipe.splitlines(): + if match := re.match(r"^(\s+)version:\s.*$", line): + indentation = match.group(1) + recipe = recipe.replace(line, f"{indentation}version: \"{version}\"") + break + + for source in rendered_sources: + # update the hash value + urls = source.url + if not isinstance(urls, list): + urls = [urls] + found_hash = False + for url in source.url: + if source.sha256 is not None: + hash_type = "sha256" + elif source.md5 is not None: + hash_type = "md5" + new_hash = _try_url_and_hash_it(source.url, hash_type) + if new_hash is not None: + if hash_type == "sha256": + recipe = recipe.replace(source.sha256, new_hash) + else: + recipe = recipe.replace(source.md5, new_hash) + found_hash = True + break + + if not found_hash: + return False, set(["could not find a hash for the source"]) + + # write the updated recipe back to the file + recipe_path.write_text(recipe) + + return True, set() + +def update_version(raw_meta_yaml, version, hash_type="sha256", recipe_version: int = 0) -> (str, set[str]): """Update the version in a recipe. Parameters @@ -657,6 +729,9 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"): ) return None, errors + if recipe_version == 1: + update_version_v1(raw_meta_yaml, version, hash_type) + try: cmeta = CondaMetaYAML(raw_meta_yaml) except Exception as e: @@ -773,3 +848,30 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"): else: logger.critical("Recipe did not change in version migration!") return None, errors + + +if __name__ == "__main__": + # parse args and invoke update_version_feedstock_dir + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("feedstock_dir", help="The feedstock directory to update.") + parser.add_argument("version", help="The new version to update to.") + parser.add_argument( + "--hash-type", + default="sha256", + help="The hash type to use for the source.", + ) + parser.add_argument( + "--use-container", + action="store_true", + help="Use a container to run the version parsing.", + ) + + args = parser.parse_args() + updated, errors = update_version_feedstock_dir( + args.feedstock_dir, + args.version, + hash_type=args.hash_type, + use_container=args.use_container, + ) From 95e7d33ec039371a8b8e4bb304d4f5ffdb87b426 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 7 Jan 2025 10:09:11 +0100 Subject: [PATCH 03/40] add first test --- conda_forge_tick/migrators/version.py | 69 ++++++++--- conda_forge_tick/update_recipe/__init__.py | 2 +- conda_forge_tick/update_recipe/version.py | 115 +++++++++++++----- conda_forge_tick/url_transforms.py | 5 + tests/test_migrators.py | 73 +++++++---- tests/test_v1_yaml/version_pypi_url.yaml | 40 ++++++ .../version_pypi_url_correct.yaml | 40 ++++++ tests/test_version_migrator.py | 48 +++++++- 8 files changed, 313 insertions(+), 79 deletions(-) create mode 100644 tests/test_v1_yaml/version_pypi_url.yaml create mode 100644 tests/test_v1_yaml/version_pypi_url_correct.yaml diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index e92c46d30..b45f4fa11 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -1,11 +1,11 @@ import copy import functools import logging -import os import random import secrets import typing import warnings +from pathlib import Path from typing import Any, List, Sequence import conda.exceptions @@ -15,9 +15,8 @@ from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator from conda_forge_tick.models.pr_info import MigratorName -from conda_forge_tick.os_utils import pushd from conda_forge_tick.update_deps import get_dep_updates_and_hints -from conda_forge_tick.update_recipe import update_version +from conda_forge_tick.update_recipe import update_version, update_version_v1 from conda_forge_tick.utils import get_keys_default, sanitize_string if typing.TYPE_CHECKING: @@ -62,6 +61,7 @@ class Version(Migrator): migrator_version = 0 rerender = True name = MigratorName.VERSION + allowed_schema_versions = {0, 1} def __init__(self, python_nodes, *args, **kwargs): if not hasattr(self, "_init_args"): @@ -93,11 +93,36 @@ def filter( self._new_version = new_version # if no jinja2 version, then move on - if "raw_meta_yaml" in attrs and "{% set version" not in attrs["raw_meta_yaml"]: - return True + schema_version = get_keys_default( + attrs, + ["meta_yaml", "schema_version"], + {}, + 0, + ) + print("Checking schema version: %d" % schema_version) + if schema_version == 0: + if ( + "raw_meta_yaml" in attrs + and "{% set version" not in attrs["raw_meta_yaml"] + ): + print("ugh") + return True + elif schema_version == 1: + print("Checking ... ") + # load yaml and check if context is there + if "raw_meta_yaml" in attrs: + from rattler_build_conda_compat.loader import load_yaml + + yaml = load_yaml(attrs["raw_meta_yaml"]) + if "context" in yaml: + if "version" not in yaml["context"]: + return True + else: + raise NotImplementedError("Schema version not implemented!") + print("Still OK") conditional = super().filter(attrs) - + print("Conditional: %r" % conditional) result = bool( conditional # if archived/finished/schema version skip or len( @@ -197,21 +222,27 @@ def migrate( **kwargs: Any, ) -> "MigrationUidTypedDict": version = attrs["new_version"] - - with open(os.path.join(recipe_dir, "meta.yaml")) as fp: - raw_meta_yaml = fp.read() - - updated_meta_yaml, errors = update_version( - raw_meta_yaml, - version, - hash_type=hash_type, - ) + recipe_dir = Path(recipe_dir) + recipe_file = recipe_dir / "recipe" / "meta.yaml" + recipe_yaml = recipe_dir / "recipe" / "recipe.yaml" + if recipe_file.exists(): + raw_meta_yaml = recipe_file.read_text() + updated_meta_yaml, errors = update_version( + raw_meta_yaml, + version, + hash_type=hash_type, + ) + elif recipe_yaml.exists(): + recipe_file = recipe_yaml + updated_meta_yaml, errors = update_version_v1( + recipe_dir, + version, + hash_type=hash_type, + ) if len(errors) == 0 and updated_meta_yaml is not None: - with pushd(recipe_dir): - with open("meta.yaml", "w") as fp: - fp.write(updated_meta_yaml) - self.set_build_number("meta.yaml") + recipe_file.write_text(updated_meta_yaml) + self.set_build_number(recipe_file) return super().migrate(recipe_dir, attrs) else: diff --git a/conda_forge_tick/update_recipe/__init__.py b/conda_forge_tick/update_recipe/__init__.py index e2deb50c1..91ddbf0e4 100644 --- a/conda_forge_tick/update_recipe/__init__.py +++ b/conda_forge_tick/update_recipe/__init__.py @@ -1,2 +1,2 @@ from .build_number import DEFAULT_BUILD_PATTERNS, update_build_number # noqa -from .version import update_version # noqa +from .version import update_version, update_version_v1 # noqa diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index de5f524b5..31a495f17 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -9,8 +9,8 @@ import shutil import tempfile import traceback -from typing import Any, MutableMapping from pathlib import Path +from typing import Any, MutableMapping import jinja2 import jinja2.sandbox @@ -153,20 +153,35 @@ def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: return None, None orig_pypi_name = None - orig_pypi_name_candidates = [ - url_tmpl.split("/")[-2], - context.get("name", None), - (cmeta.meta.get("package", {}) or {}).get("name", None), - ] - if "outputs" in cmeta.meta: - for output in cmeta.meta["outputs"]: - output = output or {} - orig_pypi_name_candidates.append(output.get("name", None)) + + # this is a v0 recipe + if hasattr(cmeta, "meta"): + orig_pypi_name_candidates = [ + url_tmpl.split("/")[-2], + context.get("name", None), + (cmeta.meta.get("package", {}) or {}).get("name", None), + ] + if "outputs" in cmeta.meta: + for output in cmeta.meta["outputs"]: + output = output or {} + orig_pypi_name_candidates.append(output.get("name", None)) + else: + # this is a v1 recipe + orig_pypi_name_candidates = [ + url_tmpl.split("/")[-2], + context.get("name", None), + cmeta.get("package", {}).get("name", None), + ] + # for v1 recipe compatibility + if "outputs" in cmeta: + if package_name := output.get("package", {}).get("name", None): + orig_pypi_name_candidates.append(package_name) + orig_pypi_name_candidates = sorted( {nc for nc in orig_pypi_name_candidates if nc is not None and len(nc) > 0}, key=lambda x: len(x), ) - logger.info("PyPI name candidates: %s", orig_pypi_name_candidates) + for _orig_pypi_name in orig_pypi_name_candidates: if _orig_pypi_name is None: continue @@ -220,11 +235,11 @@ def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: if "name" in context: for tmpl in [ "{{ name }}", - "{{ name.lower() }}", - "{{ name.replace('-', '_') }}", - "{{ name.replace('_', '-') }}", - "{{ name.replace('-', '_').lower() }}", - "{{ name.replace('_', '-').lower() }}", + "{{ name | lower }}", + "{{ name | replace('-', '_') }}", + "{{ name | replace('_', '-') }}", + "{{ name | replace('-', '_') | lower }}", + "{{ name | replace('_', '-') | lower }}", ]: if pypi_name == _render_jinja2(tmpl, context) + "-": name_tmpl = tmpl @@ -559,7 +574,9 @@ def update_version_feedstock_dir( ) -def _update_version_feedstock_dir_local(feedstock_dir, version, hash_type) -> (bool, set): +def _update_version_feedstock_dir_local( + feedstock_dir, version, hash_type +) -> (bool, set): if os.path.join(feedstock_dir, "recipe", "recipe.yaml"): return update_version_v1(feedstock_dir, version, hash_type) @@ -576,7 +593,9 @@ def _update_version_feedstock_dir_local(feedstock_dir, version, hash_type) -> (b return updated_meta_yaml is not None, errors -def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_type, recipe_version: int = 0): +def _update_version_feedstock_dir_containerized( + feedstock_dir, version, hash_type, recipe_version: int = 0 +): with tempfile.TemporaryDirectory() as tmpdir: tmp_feedstock_dir = os.path.join(tmpdir, os.path.basename(feedstock_dir)) sync_dirs( @@ -631,15 +650,17 @@ def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_typ return data["updated"], data["errors"] -def update_version_v1(feedstock_dir: str, version: str, hash_type: str) -> (bool , set[str]): +def update_version_v1( + feedstock_dir: str, version: str, hash_type: str +) -> (str | None, set[str]): """Update the version in a recipe. Parameters ---------- - raw_meta_yaml : str - The recipe meta.yaml as a string. + feedstock_dir : str + The feedstock directory to update. version : str - The version of the recipe. + The new version of the recipe. hash_type : str The kind of hash used on the source. """ @@ -649,12 +670,17 @@ def update_version_v1(feedstock_dir: str, version: str, hash_type: str) -> (bool feedstock_dir = Path(feedstock_dir) recipe_path = feedstock_dir / "recipe" / "recipe.yaml" - recipe = load_yaml(recipe_path.read_text()) + recipe_yaml = load_yaml(recipe_path.read_text()) variants = feedstock_dir.glob(".ci_support/*.yaml") # load all variants variants = [load_yaml(variant.read_text()) for variant in variants] + if not len(variants): + # if there are no variants, then we need to add an empty one + variants = [{}] - rendered_sources = render_all_sources(recipe, variants, override_version=version) + rendered_sources = render_all_sources( + recipe_yaml, variants, override_version=version + ) print("rendered_sources", rendered_sources) @@ -666,38 +692,61 @@ def update_version_v1(feedstock_dir: str, version: str, hash_type: str) -> (bool for line in recipe.splitlines(): if match := re.match(r"^(\s+)version:\s.*$", line): indentation = match.group(1) - recipe = recipe.replace(line, f"{indentation}version: \"{version}\"") + recipe = recipe.replace(line, f'{indentation}version: "{version}"') break for source in rendered_sources: # update the hash value urls = source.url + # zip url and template if not isinstance(urls, list): - urls = [urls] + urls = zip([urls], [source.template]) + else: + urls = zip(urls, source.template) + found_hash = False - for url in source.url: + for url, template in urls: if source.sha256 is not None: hash_type = "sha256" elif source.md5 is not None: hash_type = "md5" - new_hash = _try_url_and_hash_it(source.url, hash_type) + + # convert to regular jinja2 template + template = template.replace("${{", "{{") + new_tmpl, new_hash = _get_new_url_tmpl_and_hash( + template, + source.context, + hash_type, + recipe_yaml, + ) + print("new_hash", new_hash) + print("new_tmpl", new_tmpl) + if new_hash is not None: if hash_type == "sha256": recipe = recipe.replace(source.sha256, new_hash) else: recipe = recipe.replace(source.md5, new_hash) found_hash = True + + # convert back to v1 minijinja template + print("New template", new_tmpl) + new_tmpl = new_tmpl.replace("{{", "${{") + print("replace", source.template, new_tmpl) + if new_tmpl != source.template: + recipe = recipe.replace(source.template, new_tmpl) + break if not found_hash: - return False, set(["could not find a hash for the source"]) + return None, {"could not find a hash for the source"} - # write the updated recipe back to the file - recipe_path.write_text(recipe) + return recipe, set() - return True, set() -def update_version(raw_meta_yaml, version, hash_type="sha256", recipe_version: int = 0) -> (str, set[str]): +def update_version( + raw_meta_yaml, version, hash_type="sha256", recipe_version: int = 0 +) -> (str, set[str]): """Update the version in a recipe. Parameters diff --git a/conda_forge_tick/url_transforms.py b/conda_forge_tick/url_transforms.py index 6b5b72046..7cd43fead 100644 --- a/conda_forge_tick/url_transforms.py +++ b/conda_forge_tick/url_transforms.py @@ -132,3 +132,8 @@ def gen_transformed_urls(url): if new_url not in yielded: yield new_url yielded.add(new_url) + + +if __name__ == "__main__": + url = "https://github.com/xtensor-stack/xtensor/archive/{{ version }}.tar.gz" + print(set(gen_transformed_urls(url))) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 0540a47ba..60889a47d 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -17,7 +17,11 @@ ) from conda_forge_tick.migrators.migration_yaml import all_noarch from conda_forge_tick.os_utils import pushd -from conda_forge_tick.utils import frozen_to_json_friendly, parse_meta_yaml +from conda_forge_tick.utils import ( + frozen_to_json_friendly, + parse_meta_yaml, + parse_recipe_yaml, +) sample_yaml_rebuild = """ {% set version = "1.3.2" %} @@ -466,35 +470,60 @@ def run_test_migration( tmpdir: str, should_filter: bool = False, make_body: bool = False, + recipe_version: int = 0, ): + tmpdir_p = Path(tmpdir) if mr_out: mr_out.update(bot_rerun=False) - Path(tmpdir).joinpath("meta.yaml").write_text(inp) - + if recipe_version == 0: + tmpdir_p.joinpath("meta.yaml").write_text(inp) + else: + tmpdir_p.joinpath(".ci_support").mkdir() + tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) + (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( + "target_platform: linux-64" + ) # read the conda-forge.yml cf_yml_path = Path(tmpdir).parent / "conda-forge.yml" cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" # Load the meta.yaml (this is done in the graph) - try: - name = parse_meta_yaml(inp)["package"]["name"] - except Exception: - name = "blah" + if recipe_version == 0: + try: + name = parse_meta_yaml(inp)["package"]["name"] + except Exception: + name = "blah" - pmy = populate_feedstock_attributes( - name, sub_graph={}, meta_yaml=inp, conda_forge_yaml=cf_yml - ) + pmy = populate_feedstock_attributes( + name, sub_graph={}, meta_yaml=inp, conda_forge_yaml=cf_yml + ) - # these are here for legacy migrators - pmy["version"] = pmy["meta_yaml"]["package"]["version"] - pmy["req"] = set() - for k in ["build", "host", "run"]: - req = pmy["meta_yaml"].get("requirements", {}) or {} - _set = req.get(k) or set() - pmy["req"] |= set(_set) - pmy["raw_meta_yaml"] = inp - pmy.update(kwargs) + # these are here for legacy migrators + pmy["version"] = pmy["meta_yaml"]["package"]["version"] + pmy["req"] = set() + for k in ["build", "host", "run"]: + req = pmy["meta_yaml"].get("requirements", {}) or {} + _set = req.get(k) or set() + pmy["req"] |= set(_set) + pmy["raw_meta_yaml"] = inp + pmy.update(kwargs) + else: + try: + name = parse_recipe_yaml(inp)["package"]["name"] + except Exception: + name = "blah" + + pmy = populate_feedstock_attributes( + name, + sub_graph={}, + recipe_yaml=inp, + conda_forge_yaml=cf_yml, + feedstock_dir=tmpdir_p, + ) + pmy["version"] = pmy["meta_yaml"]["package"]["version"] + pmy["raw_meta_yaml"] = inp + pmy.update(kwargs) try: if "new_version" in kwargs: @@ -533,8 +562,10 @@ def run_test_migration( pmy["pr_info"] = {} pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)]) - with open(os.path.join(tmpdir, "meta.yaml")) as f: - actual_output = f.read() + if recipe_version == 0: + actual_output = tmpdir_p.joinpath("meta.yaml").read_text() + else: + actual_output = tmpdir_p.joinpath("recipe", "recipe.yaml").read_text() # strip jinja comments pat = re.compile(r"{#.*#}") actual_output = pat.sub("", actual_output) diff --git a/tests/test_v1_yaml/version_pypi_url.yaml b/tests/test_v1_yaml/version_pypi_url.yaml new file mode 100644 index 000000000..889c5f721 --- /dev/null +++ b/tests/test_v1_yaml/version_pypi_url.yaml @@ -0,0 +1,40 @@ +context: + name: dagster_spark + version: 0.6.5 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: "https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name }}-${{ version }}.tar.gz" + sha256: 4b7bf11c1436a277b757afaeb79e07562720d4339e8293d6d5d4849d5f4c1549 + +build: + number: 100 + noarch: python + script: ${{ PYTHON }} -m pip install . -vv + +requirements: + host: + - pip + - python + - setuptools + run: + - python + - dagster ${{ version }}.* + +tests: + - python: + imports: + - dagster_spark + +about: + homepage: "https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-spark" + license: Apache-2.0 + license_file: LICENSE + summary: "Package for Spark Dagster framework components." + +extra: + recipe-maintainers: + - conda-forge/dagster diff --git a/tests/test_v1_yaml/version_pypi_url_correct.yaml b/tests/test_v1_yaml/version_pypi_url_correct.yaml new file mode 100644 index 000000000..67ff524fe --- /dev/null +++ b/tests/test_v1_yaml/version_pypi_url_correct.yaml @@ -0,0 +1,40 @@ +context: + name: dagster_spark + version: "0.7.1" + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: "https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name | replace('_', '-') }}-${{ version }}.tar.gz" + sha256: f3fe3f89011899b82451669cf1dbe4978523b8ac0f62c9c116429876fe8b6be8 + +build: + number: 0 + noarch: python + script: ${{ PYTHON }} -m pip install . -vv + +requirements: + host: + - pip + - python + - setuptools + run: + - python + - dagster ${{ version }}.* + +tests: + - python: + imports: + - dagster_spark + +about: + homepage: "https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-spark" + license: Apache-2.0 + license_file: LICENSE + summary: "Package for Spark Dagster framework components." + +extra: + recipe-maintainers: + - conda-forge/dagster diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 8cdc97ab5..62333fb32 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -12,7 +12,8 @@ VERSION = Version(set()) -YAML_PATH = os.path.join(os.path.dirname(__file__), "test_yaml") +YAML_PATH = Path(__file__).parent / "test_yaml" +YAML_V1_PATH = Path(__file__).parent / "test_v1_yaml" VARIANT_SOURCES_NOT_IMPLEMENTED = ( "Sources that depend on conda build config variants are not supported yet." @@ -92,11 +93,47 @@ def test_version_up(case, new_ver, tmpdir, caplog): logger="conda_forge_tick.migrators.version", ) - with open(os.path.join(YAML_PATH, "version_%s.yaml" % case)) as fp: - in_yaml = fp.read() + in_yaml = (YAML_PATH / f"version_{case}.yaml").read_text() + out_yaml = (YAML_PATH / f"version_{case}_correct.yaml").read_text() - with open(os.path.join(YAML_PATH, "version_%s_correct.yaml" % case)) as fp: - out_yaml = fp.read() + kwargs = {"new_version": new_ver} + if case == "sha1": + kwargs["hash_type"] = "sha1" + + run_test_migration( + m=VERSION, + inp=in_yaml, + output=out_yaml, + kwargs=kwargs, + prb="Dependencies have been updated if changed", + mr_out={ + "migrator_name": Version.name, + "migrator_version": Version.migrator_version, + "version": new_ver, + }, + tmpdir=tmpdir, + ) + + +@pytest.mark.parametrize( + "case,new_ver", + [ + ("pypi_url", "0.7.1"), + ], +) +@flaky +def test_version_up_v1(case, new_ver, tmpdir, caplog): + caplog.set_level( + logging.DEBUG, + logger="conda_forge_tick.migrators.version", + ) + + in_yaml = (YAML_V1_PATH / f"version_{case}.yaml").read_text() + out_yaml = (YAML_V1_PATH / f"version_{case}_correct.yaml").read_text() + + # create a tempdir that looks like a feedstock with a recipe/recipe.yaml file + recipe_dir = tmpdir.mkdir("recipe") + recipe_dir.join("recipe.yaml").write(in_yaml) kwargs = {"new_version": new_ver} if case == "sha1": @@ -114,6 +151,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): "version": new_ver, }, tmpdir=tmpdir, + recipe_version=1, ) From 06bb1857cad00ce60f21c675ab34152f381f360a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 7 Jan 2025 17:21:15 +0100 Subject: [PATCH 04/40] update expected test results for jinja filter replacement --- tests/test_yaml/stdlib_rucio-clients_after_meta.yaml | 2 +- tests/test_yaml/stdlib_rucio-clients_before_meta.yaml | 2 +- tests/test_yaml/version_21cmfast_correct.yaml | 2 +- tests/test_yaml/version_dash_extensions_correct.yaml | 2 +- tests/test_yaml/version_pypiurl_correct.yaml | 2 +- tests/test_yaml/version_pyrsmq_correct.yaml | 2 +- tests/test_yaml/version_quart_trio_correct.yaml | 2 +- tests/test_yaml/version_riskfolio_lib_correct.yaml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_yaml/stdlib_rucio-clients_after_meta.yaml b/tests/test_yaml/stdlib_rucio-clients_after_meta.yaml index 729581d3c..c3a891e18 100644 --- a/tests/test_yaml/stdlib_rucio-clients_after_meta.yaml +++ b/tests/test_yaml/stdlib_rucio-clients_after_meta.yaml @@ -7,7 +7,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_') }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('-', '_') }}-{{ version }}.tar.gz sha256: 92229d097012be2347266a633c655b46ca66ead0628becfa506ee6c4a9b38057 build: diff --git a/tests/test_yaml/stdlib_rucio-clients_before_meta.yaml b/tests/test_yaml/stdlib_rucio-clients_before_meta.yaml index f5dd35ec0..78923a16f 100644 --- a/tests/test_yaml/stdlib_rucio-clients_before_meta.yaml +++ b/tests/test_yaml/stdlib_rucio-clients_before_meta.yaml @@ -7,7 +7,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_') }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('-', '_') }}-{{ version }}.tar.gz sha256: c3fdaae0beb3ac10d82337d29cd5260c49bbb0f942d443bec6e997c51af74c0d build: diff --git a/tests/test_yaml/version_21cmfast_correct.yaml b/tests/test_yaml/version_21cmfast_correct.yaml index 53d114817..377337f48 100644 --- a/tests/test_yaml/version_21cmfast_correct.yaml +++ b/tests/test_yaml/version_21cmfast_correct.yaml @@ -6,7 +6,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | lower }}-{{ version }}.tar.gz sha256: aff3b5a2adb30ad9a6c2274461901686606e9fdb5e3ff7040cbdf22755d7469f diff --git a/tests/test_yaml/version_dash_extensions_correct.yaml b/tests/test_yaml/version_dash_extensions_correct.yaml index d35994137..8b4be34a1 100644 --- a/tests/test_yaml/version_dash_extensions_correct.yaml +++ b/tests/test_yaml/version_dash_extensions_correct.yaml @@ -7,7 +7,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_') }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('-', '_') }}-{{ version }}.tar.gz sha256: b36fcf6fd74d87cafdbabc9568c3ae0097712ccee8f7d59be8e916b51d40b106 build: diff --git a/tests/test_yaml/version_pypiurl_correct.yaml b/tests/test_yaml/version_pypiurl_correct.yaml index bc78fb394..97641ffc1 100644 --- a/tests/test_yaml/version_pypiurl_correct.yaml +++ b/tests/test_yaml/version_pypiurl_correct.yaml @@ -6,7 +6,7 @@ package: version: "{{ version }}" source: - url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('_', '-') }}-{{ version }}.tar.gz" + url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('_', '-') }}-{{ version }}.tar.gz" sha256: f3fe3f89011899b82451669cf1dbe4978523b8ac0f62c9c116429876fe8b6be8 build: diff --git a/tests/test_yaml/version_pyrsmq_correct.yaml b/tests/test_yaml/version_pyrsmq_correct.yaml index 8dfccc9fb..a48dc77cf 100644 --- a/tests/test_yaml/version_pyrsmq_correct.yaml +++ b/tests/test_yaml/version_pyrsmq_correct.yaml @@ -6,7 +6,7 @@ package: version: "{{ version }}" source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | lower }}-{{ version }}.tar.gz sha256: dd1f8467e541935489be018dbb0ba1df8b903eb855bf1725947ceee41df92fa4 build: diff --git a/tests/test_yaml/version_quart_trio_correct.yaml b/tests/test_yaml/version_quart_trio_correct.yaml index ee3795746..eadfc14d2 100644 --- a/tests/test_yaml/version_quart_trio_correct.yaml +++ b/tests/test_yaml/version_quart_trio_correct.yaml @@ -6,7 +6,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_').lower() }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('-', '_') | lower }}-{{ version }}.tar.gz sha256: 149c9c65c2faafdf455a4461b600e1983b71e593b6f8c8b91b592bbda36cea98 build: diff --git a/tests/test_yaml/version_riskfolio_lib_correct.yaml b/tests/test_yaml/version_riskfolio_lib_correct.yaml index 548723d3c..893920bc4 100644 --- a/tests/test_yaml/version_riskfolio_lib_correct.yaml +++ b/tests/test_yaml/version_riskfolio_lib_correct.yaml @@ -6,7 +6,7 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_').lower() }}-{{ version }}.tar.gz + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name | replace('-', '_') | lower }}-{{ version }}.tar.gz sha256: 1048655b53a714ac045e756215275a302ae5c5816f3c73459b26056b054dbb46 patches: - devendor-eigen-spectra.patch From 9e24812c711055df7c660b496725ac6bb1451fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 7 Jan 2025 17:47:40 +0100 Subject: [PATCH 05/40] fix paths in Version migrator All migrations expect `recipe_dir` to be the actual `recipe` directory rather than feedstock directory, so change the path construction back. We need to figure out a better way of doing this. --- conda_forge_tick/migrators/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index b45f4fa11..70a9cbcd8 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -223,8 +223,8 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] recipe_dir = Path(recipe_dir) - recipe_file = recipe_dir / "recipe" / "meta.yaml" - recipe_yaml = recipe_dir / "recipe" / "recipe.yaml" + recipe_file = recipe_dir / "meta.yaml" + recipe_yaml = recipe_dir / "recipe.yaml" if recipe_file.exists(): raw_meta_yaml = recipe_file.read_text() updated_meta_yaml, errors = update_version( From 5fe883b29889d3224cc9103fe5a0b95382f959c2 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 09:49:01 +0100 Subject: [PATCH 06/40] remove debug code and broken dead code --- conda_forge_tick/migrators/version.py | 4 ---- conda_forge_tick/update_recipe/version.py | 11 +---------- conda_forge_tick/url_transforms.py | 5 ----- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 70a9cbcd8..09867e529 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -106,10 +106,8 @@ def filter( "raw_meta_yaml" in attrs and "{% set version" not in attrs["raw_meta_yaml"] ): - print("ugh") return True elif schema_version == 1: - print("Checking ... ") # load yaml and check if context is there if "raw_meta_yaml" in attrs: from rattler_build_conda_compat.loader import load_yaml @@ -120,9 +118,7 @@ def filter( return True else: raise NotImplementedError("Schema version not implemented!") - print("Still OK") conditional = super().filter(attrs) - print("Conditional: %r" % conditional) result = bool( conditional # if archived/finished/schema version skip or len( diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 31a495f17..1b07dfa8b 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -682,8 +682,6 @@ def update_version_v1( recipe_yaml, variants, override_version=version ) - print("rendered_sources", rendered_sources) - # load recipe text recipe_path = feedstock_dir / "recipe" / "recipe.yaml" recipe = recipe_path.read_text() @@ -719,8 +717,6 @@ def update_version_v1( hash_type, recipe_yaml, ) - print("new_hash", new_hash) - print("new_tmpl", new_tmpl) if new_hash is not None: if hash_type == "sha256": @@ -730,9 +726,7 @@ def update_version_v1( found_hash = True # convert back to v1 minijinja template - print("New template", new_tmpl) new_tmpl = new_tmpl.replace("{{", "${{") - print("replace", source.template, new_tmpl) if new_tmpl != source.template: recipe = recipe.replace(source.template, new_tmpl) @@ -745,7 +739,7 @@ def update_version_v1( def update_version( - raw_meta_yaml, version, hash_type="sha256", recipe_version: int = 0 + raw_meta_yaml, version, hash_type="sha256" ) -> (str, set[str]): """Update the version in a recipe. @@ -778,9 +772,6 @@ def update_version( ) return None, errors - if recipe_version == 1: - update_version_v1(raw_meta_yaml, version, hash_type) - try: cmeta = CondaMetaYAML(raw_meta_yaml) except Exception as e: diff --git a/conda_forge_tick/url_transforms.py b/conda_forge_tick/url_transforms.py index 7cd43fead..6b5b72046 100644 --- a/conda_forge_tick/url_transforms.py +++ b/conda_forge_tick/url_transforms.py @@ -132,8 +132,3 @@ def gen_transformed_urls(url): if new_url not in yielded: yield new_url yielded.add(new_url) - - -if __name__ == "__main__": - url = "https://github.com/xtensor-stack/xtensor/archive/{{ version }}.tar.gz" - print(set(gen_transformed_urls(url))) From 95aacdde1ba59c799a1c9e72bd5b9fd22754608e Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 10:03:08 +0100 Subject: [PATCH 07/40] fix all migration tests --- conda_forge_tick/migrators/version.py | 19 ++++++++++++------- conda_forge_tick/update_recipe/version.py | 4 +--- tests/test_migrators.py | 11 +++++++---- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 09867e529..a760e7899 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -219,26 +219,31 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] recipe_dir = Path(recipe_dir) - recipe_file = recipe_dir / "meta.yaml" + file = recipe_dir / "meta.yaml" recipe_yaml = recipe_dir / "recipe.yaml" - if recipe_file.exists(): - raw_meta_yaml = recipe_file.read_text() + if file.exists(): + raw_meta_yaml = file.read_text() updated_meta_yaml, errors = update_version( raw_meta_yaml, version, hash_type=hash_type, ) elif recipe_yaml.exists(): - recipe_file = recipe_yaml + file = recipe_yaml updated_meta_yaml, errors = update_version_v1( - recipe_dir, + # we need to give the "feedstock_dir" (not recipe dir) + recipe_dir.parent, version, hash_type=hash_type, ) + else: + raise FileNotFoundError( + f"Neither {file} nor {recipe_yaml} exists in {recipe_dir}", + ) if len(errors) == 0 and updated_meta_yaml is not None: - recipe_file.write_text(updated_meta_yaml) - self.set_build_number(recipe_file) + file.write_text(updated_meta_yaml) + self.set_build_number(file) return super().migrate(recipe_dir, attrs) else: diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 1b07dfa8b..ad8c6a416 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -738,9 +738,7 @@ def update_version_v1( return recipe, set() -def update_version( - raw_meta_yaml, version, hash_type="sha256" -) -> (str, set[str]): +def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str]): """Update the version in a recipe. Parameters diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 60889a47d..8988a91c0 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -478,12 +478,15 @@ def run_test_migration( if recipe_version == 0: tmpdir_p.joinpath("meta.yaml").write_text(inp) + recipe_dir = str(tmpdir_p) else: tmpdir_p.joinpath(".ci_support").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( "target_platform: linux-64" ) + + recipe_dir = str(tmpdir_p / "recipe") # read the conda-forge.yml cf_yml_path = Path(tmpdir).parent / "conda-forge.yml" cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" @@ -535,13 +538,13 @@ def run_test_migration( return pmy m.run_pre_piggyback_migrations( - tmpdir, + recipe_dir, pmy, hash_type=pmy.get("hash_type", "sha256"), ) - mr = m.migrate(tmpdir, pmy, hash_type=pmy.get("hash_type", "sha256")) + mr = m.migrate(recipe_dir, pmy, hash_type=pmy.get("hash_type", "sha256")) m.run_post_piggyback_migrations( - tmpdir, + recipe_dir, pmy, hash_type=pmy.get("hash_type", "sha256"), ) @@ -565,7 +568,7 @@ def run_test_migration( if recipe_version == 0: actual_output = tmpdir_p.joinpath("meta.yaml").read_text() else: - actual_output = tmpdir_p.joinpath("recipe", "recipe.yaml").read_text() + actual_output = tmpdir_p.joinpath("recipe/recipe.yaml").read_text() # strip jinja comments pat = re.compile(r"{#.*#}") actual_output = pat.sub("", actual_output) From 407b002be565bcf2a4dbeb8816026da0080d32b5 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 10:06:25 +0100 Subject: [PATCH 08/40] restore log --- conda_forge_tick/update_recipe/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index ad8c6a416..fb1d998ce 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -181,6 +181,7 @@ def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: {nc for nc in orig_pypi_name_candidates if nc is not None and len(nc) > 0}, key=lambda x: len(x), ) + logger.info("PyPI name candidates: %s", orig_pypi_name_candidates) for _orig_pypi_name in orig_pypi_name_candidates: if _orig_pypi_name is None: From bc96f7135ebd7f074727d7406c6b9f2652c92652 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 10:27:46 +0100 Subject: [PATCH 09/40] improve --- conda_forge_tick/update_recipe/version.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index fb1d998ce..7267cd0ee 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -560,7 +560,6 @@ def update_version_feedstock_dir( A set of strings giving any errors found when updating the version. The set will be empty if there were no errors. """ - feedstock_dir = Path(feedstock_dir) if should_use_container(use_container=use_container): return _update_version_feedstock_dir_containerized( feedstock_dir, @@ -578,7 +577,8 @@ def update_version_feedstock_dir( def _update_version_feedstock_dir_local( feedstock_dir, version, hash_type ) -> (bool, set): - if os.path.join(feedstock_dir, "recipe", "recipe.yaml"): + feedstock_path = Path(feedstock_dir) + if (feedstock_path / "recipe" / "recipe.yaml").exists(): return update_version_v1(feedstock_dir, version, hash_type) recipe_path = os.path.join(feedstock_dir, "recipe", "meta.yaml") @@ -594,9 +594,7 @@ def _update_version_feedstock_dir_local( return updated_meta_yaml is not None, errors -def _update_version_feedstock_dir_containerized( - feedstock_dir, version, hash_type, recipe_version: int = 0 -): +def _update_version_feedstock_dir_containerized(feedstock_dir, version, hash_type): with tempfile.TemporaryDirectory() as tmpdir: tmp_feedstock_dir = os.path.join(tmpdir, os.path.basename(feedstock_dir)) sync_dirs( @@ -893,6 +891,8 @@ def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str] # parse args and invoke update_version_feedstock_dir import argparse + # set log level to debug + logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() parser.add_argument("feedstock_dir", help="The feedstock directory to update.") parser.add_argument("version", help="The new version to update to.") From 4844ccf0962e7b4f56881e78380c36615fe5d952 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 11:02:16 +0100 Subject: [PATCH 10/40] add more tests --- tests/test_v1_yaml/version_jolt.yaml | 73 ++++++++++++++++++++ tests/test_v1_yaml/version_jolt_correct.yaml | 73 ++++++++++++++++++++ tests/test_version_migrator.py | 1 + 3 files changed, 147 insertions(+) create mode 100644 tests/test_v1_yaml/version_jolt.yaml create mode 100644 tests/test_v1_yaml/version_jolt_correct.yaml diff --git a/tests/test_v1_yaml/version_jolt.yaml b/tests/test_v1_yaml/version_jolt.yaml new file mode 100644 index 000000000..896bc8048 --- /dev/null +++ b/tests/test_v1_yaml/version_jolt.yaml @@ -0,0 +1,73 @@ +context: + version: "5.1.0" + sha: 10fcc863ae2b9d48c2f22d8b0204034820e57a55f858b7c388ac9579d8cf4095 + +package: + name: jolt-physics + version: ${{ version }} + +source: + url: https://github.com/jrouwe/JoltPhysics/archive/refs/tags/v${{ version }}.zip + sha256: ${{ sha }} + patches: + - patches/001-use-gnuinstalldirs.patch + +build: + number: 0 + script: + - if: win + then: | + cmake -GNinja ^ + %CMAKE_ARGS% ^ + -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_BUILD_TYPE=Distribution ^ + -DCROSS_PLATFORM_DETERMINISTIC=ON ^ + -DTARGET_VIEWER=OFF ^ + -DTARGET_SAMPLES=OFF ^ + -DTARGET_HELLO_WORLD=OFF ^ + -DTARGET_UNIT_TESTS=OFF ^ + -DTARGET_PERFORMANCE_TEST=OFF ^ + -S %SRC_DIR%\Build + cmake --build . --target install + else: | + cmake -GNinja \ + $CMAKE_ARGS \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Distribution \ + -DCROSS_PLATFORM_DETERMINISTIC=ON \ + -DTARGET_VIEWER=OFF \ + -DTARGET_SAMPLES=OFF \ + -DTARGET_HELLO_WORLD=OFF \ + -DTARGET_UNIT_TESTS=OFF \ + -DTARGET_PERFORMANCE_TEST=OFF \ + -S $SRC_DIR/Build + cmake --build . --target install + +requirements: + build: + - ${{ compiler("cxx") }} + - ${{ stdlib("c") }} + - cmake + - ninja + +tests: + - package_contents: + include: + - Jolt/Jolt.h + lib: + - Jolt + +about: + homepage: https://github.com/jrouwe/JoltPhysics + license: MIT + license_file: LICENSE + summary: A multi core friendly rigid body physics and collision detection library. + description: A multi core friendly rigid body physics and collision detection library. Written in C++. Suitable for games and VR applications. Used by Horizon Forbidden West. + documentation: https://jrouwe.github.io/JoltPhysics/ + repository: https://github.com/jrouwe/JoltPhysics + +extra: + recipe-maintainers: + - baszalmstra diff --git a/tests/test_v1_yaml/version_jolt_correct.yaml b/tests/test_v1_yaml/version_jolt_correct.yaml new file mode 100644 index 000000000..a0f69cc07 --- /dev/null +++ b/tests/test_v1_yaml/version_jolt_correct.yaml @@ -0,0 +1,73 @@ +context: + version: "5.2.0" + sha: 6953a6293f1ccfe3790cb0d778c71e3736031c441e0f434fd2baa8d307ebfef3 + +package: + name: jolt-physics + version: ${{ version }} + +source: + url: https://github.com/jrouwe/JoltPhysics/archive/refs/tags/v${{ version }}.zip + sha256: ${{ sha }} + patches: + - patches/001-use-gnuinstalldirs.patch + +build: + number: 0 + script: + - if: win + then: | + cmake -GNinja ^ + %CMAKE_ARGS% ^ + -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_BUILD_TYPE=Distribution ^ + -DCROSS_PLATFORM_DETERMINISTIC=ON ^ + -DTARGET_VIEWER=OFF ^ + -DTARGET_SAMPLES=OFF ^ + -DTARGET_HELLO_WORLD=OFF ^ + -DTARGET_UNIT_TESTS=OFF ^ + -DTARGET_PERFORMANCE_TEST=OFF ^ + -S %SRC_DIR%\Build + cmake --build . --target install + else: | + cmake -GNinja \ + $CMAKE_ARGS \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Distribution \ + -DCROSS_PLATFORM_DETERMINISTIC=ON \ + -DTARGET_VIEWER=OFF \ + -DTARGET_SAMPLES=OFF \ + -DTARGET_HELLO_WORLD=OFF \ + -DTARGET_UNIT_TESTS=OFF \ + -DTARGET_PERFORMANCE_TEST=OFF \ + -S $SRC_DIR/Build + cmake --build . --target install + +requirements: + build: + - ${{ compiler("cxx") }} + - ${{ stdlib("c") }} + - cmake + - ninja + +tests: + - package_contents: + include: + - Jolt/Jolt.h + lib: + - Jolt + +about: + homepage: https://github.com/jrouwe/JoltPhysics + license: MIT + license_file: LICENSE + summary: A multi core friendly rigid body physics and collision detection library. + description: A multi core friendly rigid body physics and collision detection library. Written in C++. Suitable for games and VR applications. Used by Horizon Forbidden West. + documentation: https://jrouwe.github.io/JoltPhysics/ + repository: https://github.com/jrouwe/JoltPhysics + +extra: + recipe-maintainers: + - baszalmstra diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 62333fb32..2e6ba8cb4 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -119,6 +119,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): "case,new_ver", [ ("pypi_url", "0.7.1"), + ("jolt", "5.2.0"), ], ) @flaky From 1dcfb646751ecbebb797bf53a18fbd202347bce6 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 11:29:58 +0100 Subject: [PATCH 11/40] fix function --- conda_forge_tick/update_recipe/version.py | 27 ++++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 7267cd0ee..18219a7cd 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -578,18 +578,23 @@ def _update_version_feedstock_dir_local( feedstock_dir, version, hash_type ) -> (bool, set): feedstock_path = Path(feedstock_dir) - if (feedstock_path / "recipe" / "recipe.yaml").exists(): - return update_version_v1(feedstock_dir, version, hash_type) - - recipe_path = os.path.join(feedstock_dir, "recipe", "meta.yaml") - with open(recipe_path) as f: - raw_meta_yaml = f.read() - updated_meta_yaml, errors = update_version( - raw_meta_yaml, version, hash_type=hash_type - ) + + recipe = feedstock_path / "recipe" / "meta.yaml" + if recipe.exists(): + updated_meta_yaml, errors = update_version( + recipe.read_text(), version, hash_type=hash_type + ) + else: + recipe = feedstock_path / "recipe" / "recipe.yaml" + if recipe.exists(): + updated_meta_yaml, errors = update_version_v1( + feedstock_dir, version, hash_type + ) + else: + return False, {"no recipe found"} + if updated_meta_yaml is not None: - with open(recipe_path, "w") as f: - f.write(updated_meta_yaml) + recipe.write_text(updated_meta_yaml) return updated_meta_yaml is not None, errors From f1b09726489155ca16d4ef9160155ffd823bff25 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 13:41:07 +0100 Subject: [PATCH 12/40] address review comments --- conda_forge_tick/migrators/version.py | 43 +++++++++++++---------- conda_forge_tick/update_recipe/version.py | 33 ++++++++++++++++- tests/test_migrators.py | 5 ++- 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index a760e7899..116cbb5c8 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -11,6 +11,7 @@ import conda.exceptions import networkx as nx from conda.models.version import VersionOrder +from rattler_build_conda_compat.loader import load_yaml from conda_forge_tick.contexts import ClonedFeedstockContext, FeedstockContext from conda_forge_tick.migrators.core import Migrator @@ -102,22 +103,24 @@ def filter( ) print("Checking schema version: %d" % schema_version) if schema_version == 0: - if ( - "raw_meta_yaml" in attrs - and "{% set version" not in attrs["raw_meta_yaml"] - ): + if "raw_meta_yaml" not in attrs: + return True + if "{% set version" not in attrs["raw_meta_yaml"]: return True elif schema_version == 1: # load yaml and check if context is there - if "raw_meta_yaml" in attrs: - from rattler_build_conda_compat.loader import load_yaml + if "raw_meta_yaml" not in attrs: + return True + + yaml = load_yaml(attrs["raw_meta_yaml"]) + if "context" not in yaml: + return True - yaml = load_yaml(attrs["raw_meta_yaml"]) - if "context" in yaml: - if "version" not in yaml["context"]: - return True + if "version" not in yaml["context"]: + return True else: raise NotImplementedError("Schema version not implemented!") + conditional = super().filter(attrs) result = bool( conditional # if archived/finished/schema version skip @@ -219,17 +222,19 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] recipe_dir = Path(recipe_dir) - file = recipe_dir / "meta.yaml" - recipe_yaml = recipe_dir / "recipe.yaml" - if file.exists(): - raw_meta_yaml = file.read_text() + recipe = None + recipe_v0 = recipe_dir / "meta.yaml" + recipe_v1 = recipe_dir / "recipe.yaml" + if recipe_v0.exists(): + raw_meta_yaml = recipe_v0.read_text() + recipe = recipe_v0 updated_meta_yaml, errors = update_version( raw_meta_yaml, version, hash_type=hash_type, ) - elif recipe_yaml.exists(): - file = recipe_yaml + elif recipe_v1.exists(): + recipe = recipe_v1 updated_meta_yaml, errors = update_version_v1( # we need to give the "feedstock_dir" (not recipe dir) recipe_dir.parent, @@ -238,12 +243,12 @@ def migrate( ) else: raise FileNotFoundError( - f"Neither {file} nor {recipe_yaml} exists in {recipe_dir}", + f"Neither {recipe_v0} nor {recipe_v1} exists in {recipe_dir}", ) if len(errors) == 0 and updated_meta_yaml is not None: - file.write_text(updated_meta_yaml) - self.set_build_number(file) + recipe.write_text(updated_meta_yaml) + self.set_build_number(recipe) return super().migrate(recipe_dir, attrs) else: diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 18219a7cd..0729aaa6a 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -143,6 +143,28 @@ def _render_jinja2(tmpl, context): def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: Any): + """ + Try to get a new version from the PyPI API. The returned URL might use a different + format (host) than the original URL template, e.g. `https://files.pythonhosted.org/` + instead of `https://pypi.io/`. + + Parameters + ---------- + url_tmpl : str + The URL template to try to update. + context : dict + The context to render the URL template. + hash_type : str + The hash type to use. + + Returns + ------- + new_url_tmpl : str or None + The new URL template if found. + new_hash : str or None + The new hash if found. + """ + if "version" not in context: return None, None @@ -667,6 +689,12 @@ def update_version_v1( The new version of the recipe. hash_type : str The kind of hash used on the source. + + Returns + ------- + updated_meta_yaml : str or None + The updated meta.yaml. Will be None if there is an error. + errors : set of str """ # extract all the URL sources from a given recipe / feedstock directory from rattler_build_conda_compat.loader import load_yaml @@ -897,7 +925,10 @@ def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str] import argparse # set log level to debug - logging.basicConfig(level=logging.DEBUG) + from conda_forge_tick.utils import setup_logging + + setup_logging("INFO") + parser = argparse.ArgumentParser() parser.add_argument("feedstock_dir", help="The feedstock directory to update.") parser.add_argument("version", help="The new version to update to.") diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 8988a91c0..d5c2a5587 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -479,7 +479,7 @@ def run_test_migration( if recipe_version == 0: tmpdir_p.joinpath("meta.yaml").write_text(inp) recipe_dir = str(tmpdir_p) - else: + elif recipe_version == 1: tmpdir_p.joinpath(".ci_support").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( @@ -487,6 +487,9 @@ def run_test_migration( ) recipe_dir = str(tmpdir_p / "recipe") + else: + raise ValueError(f"Unsupported recipe version: {recipe_version}") + # read the conda-forge.yml cf_yml_path = Path(tmpdir).parent / "conda-forge.yml" cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" From 8124e4807536ae9c03b4e9ff632b3b3de6d6eba5 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 13:42:00 +0100 Subject: [PATCH 13/40] Update conda_forge_tick/update_recipe/version.py Co-authored-by: Yannik Tausch --- conda_forge_tick/update_recipe/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 0729aaa6a..5080348bd 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -771,7 +771,7 @@ def update_version_v1( def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str]): - """Update the version in a recipe. + """Update the version in a v0 recipe. Parameters ---------- From d32488953c2168c4211099da06728f3d4c6c1ba8 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 13:51:03 +0100 Subject: [PATCH 14/40] remove comment --- conda_forge_tick/update_recipe/version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 5080348bd..5a8b19061 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -924,7 +924,6 @@ def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str] # parse args and invoke update_version_feedstock_dir import argparse - # set log level to debug from conda_forge_tick.utils import setup_logging setup_logging("INFO") From 8a5b0ded17b9b1407e267994d74664ffc4b8c743 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 14:42:54 +0100 Subject: [PATCH 15/40] some variable renaming --- conda_forge_tick/update_recipe/version.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 5a8b19061..9625c60f0 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -692,8 +692,8 @@ def update_version_v1( Returns ------- - updated_meta_yaml : str or None - The updated meta.yaml. Will be None if there is an error. + recipe_text : str or None + The text of the updated recipe.yaml. Will be None if there is an error. errors : set of str """ # extract all the URL sources from a given recipe / feedstock directory @@ -714,15 +714,15 @@ def update_version_v1( recipe_yaml, variants, override_version=version ) - # load recipe text - recipe_path = feedstock_dir / "recipe" / "recipe.yaml" - recipe = recipe_path.read_text() + recipe_text = recipe_path.read_text() # update the version with a regex replace - for line in recipe.splitlines(): + for line in recipe_text.splitlines(): if match := re.match(r"^(\s+)version:\s.*$", line): indentation = match.group(1) - recipe = recipe.replace(line, f'{indentation}version: "{version}"') + recipe_text = recipe_text.replace( + line, f'{indentation}version: "{version}"' + ) break for source in rendered_sources: @@ -752,22 +752,22 @@ def update_version_v1( if new_hash is not None: if hash_type == "sha256": - recipe = recipe.replace(source.sha256, new_hash) + recipe_text = recipe_text.replace(source.sha256, new_hash) else: - recipe = recipe.replace(source.md5, new_hash) + recipe_text = recipe_text.replace(source.md5, new_hash) found_hash = True # convert back to v1 minijinja template new_tmpl = new_tmpl.replace("{{", "${{") if new_tmpl != source.template: - recipe = recipe.replace(source.template, new_tmpl) + recipe_text = recipe_text.replace(source.template, new_tmpl) break if not found_hash: return None, {"could not find a hash for the source"} - return recipe, set() + return recipe_text, set() def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str]): From 95bdd8f5aea7bac9b5b002fbdf00e6396645b4c6 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 14:45:15 +0100 Subject: [PATCH 16/40] more variable renaming --- conda_forge_tick/migrators/version.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 116cbb5c8..fd7366cb9 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -223,18 +223,18 @@ def migrate( version = attrs["new_version"] recipe_dir = Path(recipe_dir) recipe = None - recipe_v0 = recipe_dir / "meta.yaml" - recipe_v1 = recipe_dir / "recipe.yaml" - if recipe_v0.exists(): - raw_meta_yaml = recipe_v0.read_text() - recipe = recipe_v0 + recipe_path_v0 = recipe_dir / "meta.yaml" + recipe_path_v1 = recipe_dir / "recipe.yaml" + if recipe_path_v0.exists(): + raw_meta_yaml = recipe_path_v0.read_text() + recipe = recipe_path_v0 updated_meta_yaml, errors = update_version( raw_meta_yaml, version, hash_type=hash_type, ) - elif recipe_v1.exists(): - recipe = recipe_v1 + elif recipe_path_v1.exists(): + recipe = recipe_path_v1 updated_meta_yaml, errors = update_version_v1( # we need to give the "feedstock_dir" (not recipe dir) recipe_dir.parent, @@ -243,7 +243,7 @@ def migrate( ) else: raise FileNotFoundError( - f"Neither {recipe_v0} nor {recipe_v1} exists in {recipe_dir}", + f"Neither {recipe_path_v0} nor {recipe_path_v1} exists in {recipe_dir}", ) if len(errors) == 0 and updated_meta_yaml is not None: From 3d3fa2a7d5be9f1dc3ba5af5f7fb6ac82959dcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 16:24:52 +0100 Subject: [PATCH 17/40] remove redundant recipe write in test_version_up_v1 The recipe is already written in `run_test_migration()`, so remove the redundnat code in the test itself. --- tests/test_migrators.py | 1 + tests/test_version_migrator.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index d5c2a5587..63c0659a6 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -481,6 +481,7 @@ def run_test_migration( recipe_dir = str(tmpdir_p) elif recipe_version == 1: tmpdir_p.joinpath(".ci_support").mkdir() + tmpdir_p.joinpath("recipe").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( "target_platform: linux-64" diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 2e6ba8cb4..8c38ba38a 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -132,10 +132,6 @@ def test_version_up_v1(case, new_ver, tmpdir, caplog): in_yaml = (YAML_V1_PATH / f"version_{case}.yaml").read_text() out_yaml = (YAML_V1_PATH / f"version_{case}_correct.yaml").read_text() - # create a tempdir that looks like a feedstock with a recipe/recipe.yaml file - recipe_dir = tmpdir.mkdir("recipe") - recipe_dir.join("recipe.yaml").write(in_yaml) - kwargs = {"new_version": new_ver} if case == "sha1": kwargs["hash_type"] = "sha1" From ac47fdaa19bc81013725b757e956493878d92245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 19:16:20 +0100 Subject: [PATCH 18/40] add more tests for v1 recipes --- tests/test_v1_yaml/version_build_as_expr.yaml | 138 +++++++++++ .../version_build_as_expr_correct.yaml | 138 +++++++++++ .../version_build_number_via_context.yaml | 227 ++++++++++++++++++ ...sion_build_number_via_context_correct.yaml | 227 ++++++++++++++++++ tests/test_version_migrator.py | 2 + 5 files changed, 732 insertions(+) create mode 100644 tests/test_v1_yaml/version_build_as_expr.yaml create mode 100644 tests/test_v1_yaml/version_build_as_expr_correct.yaml create mode 100644 tests/test_v1_yaml/version_build_number_via_context.yaml create mode 100644 tests/test_v1_yaml/version_build_number_via_context_correct.yaml diff --git a/tests/test_v1_yaml/version_build_as_expr.yaml b/tests/test_v1_yaml/version_build_as_expr.yaml new file mode 100644 index 000000000..6e4f236ab --- /dev/null +++ b/tests/test_v1_yaml/version_build_as_expr.yaml @@ -0,0 +1,138 @@ +context: + name: oneDNN + version: 3.6.1 + build: 1 + variant_build: ${{ variant_build_base | int + 100 if dnnl_cpu_runtime == "omp" else build_base }} + desc_omp: In this package oneDNN is built with the OpenMP CPU runtime. + desc_tbb: In this package oneDNN is built with the TBB CPU runtime. + desc_threadpool: | + In this package oneDNN is built with the Threadpool CPU runtime. + oneDNN requires the user to implement a Threadpool interface to enable + the library to perform computations using multiple threads. + desc_dpcpp: In this package oneDNN is built with the DPC++ CPU and GPU runtimes. + +recipe: + name: onednn-split + version: ${{ version }} + +source: + url: https://github.com/oneapi-src/${{ name }}/archive/v${{ version }}.tar.gz + sha256: a370e7f25dbf05c9c151878c53556f27d0cbe7a4f909747db6e4b2d245f533cb + +build: + number: ${{ variant_build }} + +outputs: + - package: + name: onednn + version: ${{ version }} + build: + string: ${{ dnnl_cpu_runtime }}_h${{ hash }}_${{ variant_build }} + script: build-onednn + requirements: + build: + - cmake + - ninja + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - if: dnnl_cpu_runtime == "dpcpp" + then: ${{ compiler('dpcpp') }} + - ${{ stdlib("c") }} + - if: osx + then: llvm-openmp + host: + - if: dnnl_cpu_runtime in ("tbb", "dpcpp") + then: tbb-devel + run_exports: + - ${{ pin_subpackage("onednn", upper_bound="x") }} + tests: + - script: + - if: unix + then: + - test -f ${PREFIX}/lib/libdnnl${SHLIB_EXT} + - test -f ${PREFIX}/include/oneapi/dnnl/dnnl.h + - if: win + then: + - if not exist %LIBRARY_PREFIX%\\bin\\dnnl.dll exit 1 + - if not exist %LIBRARY_PREFIX%\\lib\\dnnl.lib exit 1 + - if not exist %LIBRARY_PREFIX%\\include\\dnnl.h exit 1 + about: + license: Apache-2.0 + license_file: + - THIRD-PARTY-PROGRAMS + - LICENSE + summary: oneAPI Deep Neural Network Library (oneDNN) + description: | + oneAPI Deep Neural Network Library (oneDNN) is an open-source + cross-platform performance library of basic building blocks for deep + learning applications. + + oneDNN is intended for deep learning applications and framework + developers interested in improving application performance. + Deep learning practitioners should use one of the applications + enabled with oneDNN. + + ${{ + desc_omp if dnnl_cpu_runtime == "omp" else + desc_tbb if dnnl_cpu_runtime == "tbb" else + desc_threadpool if dnnl_cpu_runtime == "threadpool" else + desc_dpcpp if dnnl_cpu_runtime == "dpcpp" + }} + + For more information please read oneDNN developer guide: + https://oneapi-src.github.io/oneDNN/ + homepage: https://github.com/oneapi-src/oneDNN + - package: + name: onednn-${{ dnnl_cpu_runtime_name }} + version: ${{ version }} + build: + string: ${{ dnnl_cpu_runtime }}_h${{ hash }}_${{ variant_build }} + requirements: + host: + - ${{ pin_subpackage('onednn', exact=True) }} + run: + - ${{ pin_subpackage('onednn', exact=True) }} + run_exports: + - ${{ pin_subpackage("onednn", upper_bound="x") }} + tests: + - script: + - exit 0 + about: + license: Apache-2.0 + license_file: LICENSE + summary: oneAPI Deep Neural Network Library (oneDNN) + description: | + oneAPI Deep Neural Network Library (oneDNN) is an open-source + cross-platform performance library of basic building blocks for deep + learning applications. + + oneDNN is intended for deep learning applications and framework + developers interested in improving application performance. + Deep learning practitioners should use one of the applications + enabled with oneDNN. + + ${{ + desc_omp if dnnl_cpu_runtime == "omp" else + desc_tbb if dnnl_cpu_runtime == "tbb" else + desc_threadpool if dnnl_cpu_runtime == "threadpool" else + desc_dpcpp if dnnl_cpu_runtime == "dpcpp" + }} + + For more information please read oneDNN developer guide: + https://oneapi-src.github.io/oneDNN/ + homepage: https://github.com/oneapi-src/oneDNN + +about: + license: Apache-2.0 + license_file: + - LICENSE + - THIRD-PARTY-PROGRAMS + summary: oneAPI Deep Neural Network Library (oneDNN) + homepage: https://github.com/oneapi-src/oneDNN + +extra: + feedstock-name: onednn + recipe-maintainers: + - xhochy + - densamoilov + - spalicki diff --git a/tests/test_v1_yaml/version_build_as_expr_correct.yaml b/tests/test_v1_yaml/version_build_as_expr_correct.yaml new file mode 100644 index 000000000..9bc540b14 --- /dev/null +++ b/tests/test_v1_yaml/version_build_as_expr_correct.yaml @@ -0,0 +1,138 @@ +context: + name: oneDNN + version: "3.6.2" + build: 1 + variant_build: ${{ variant_build_base | int + 100 if dnnl_cpu_runtime == "omp" else build_base }} + desc_omp: In this package oneDNN is built with the OpenMP CPU runtime. + desc_tbb: In this package oneDNN is built with the TBB CPU runtime. + desc_threadpool: | + In this package oneDNN is built with the Threadpool CPU runtime. + oneDNN requires the user to implement a Threadpool interface to enable + the library to perform computations using multiple threads. + desc_dpcpp: In this package oneDNN is built with the DPC++ CPU and GPU runtimes. + +recipe: + name: onednn-split + version: ${{ version }} + +source: + url: https://github.com/oneapi-src/${{ name }}/archive/v${{ version }}.tar.gz + sha256: e79db0484dcefe2c7ff6604c295d1de2830c828941898878c80dfb062eb344d1 + +build: + number: ${{ variant_build }} + +outputs: + - package: + name: onednn + version: ${{ version }} + build: + string: ${{ dnnl_cpu_runtime }}_h${{ hash }}_${{ variant_build }} + script: build-onednn + requirements: + build: + - cmake + - ninja + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - if: dnnl_cpu_runtime == "dpcpp" + then: ${{ compiler('dpcpp') }} + - ${{ stdlib("c") }} + - if: osx + then: llvm-openmp + host: + - if: dnnl_cpu_runtime in ("tbb", "dpcpp") + then: tbb-devel + run_exports: + - ${{ pin_subpackage("onednn", upper_bound="x") }} + tests: + - script: + - if: unix + then: + - test -f ${PREFIX}/lib/libdnnl${SHLIB_EXT} + - test -f ${PREFIX}/include/oneapi/dnnl/dnnl.h + - if: win + then: + - if not exist %LIBRARY_PREFIX%\\bin\\dnnl.dll exit 1 + - if not exist %LIBRARY_PREFIX%\\lib\\dnnl.lib exit 1 + - if not exist %LIBRARY_PREFIX%\\include\\dnnl.h exit 1 + about: + license: Apache-2.0 + license_file: + - THIRD-PARTY-PROGRAMS + - LICENSE + summary: oneAPI Deep Neural Network Library (oneDNN) + description: | + oneAPI Deep Neural Network Library (oneDNN) is an open-source + cross-platform performance library of basic building blocks for deep + learning applications. + + oneDNN is intended for deep learning applications and framework + developers interested in improving application performance. + Deep learning practitioners should use one of the applications + enabled with oneDNN. + + ${{ + desc_omp if dnnl_cpu_runtime == "omp" else + desc_tbb if dnnl_cpu_runtime == "tbb" else + desc_threadpool if dnnl_cpu_runtime == "threadpool" else + desc_dpcpp if dnnl_cpu_runtime == "dpcpp" + }} + + For more information please read oneDNN developer guide: + https://oneapi-src.github.io/oneDNN/ + homepage: https://github.com/oneapi-src/oneDNN + - package: + name: onednn-${{ dnnl_cpu_runtime_name }} + version: ${{ version }} + build: + string: ${{ dnnl_cpu_runtime }}_h${{ hash }}_${{ variant_build }} + requirements: + host: + - ${{ pin_subpackage('onednn', exact=True) }} + run: + - ${{ pin_subpackage('onednn', exact=True) }} + run_exports: + - ${{ pin_subpackage("onednn", upper_bound="x") }} + tests: + - script: + - exit 0 + about: + license: Apache-2.0 + license_file: LICENSE + summary: oneAPI Deep Neural Network Library (oneDNN) + description: | + oneAPI Deep Neural Network Library (oneDNN) is an open-source + cross-platform performance library of basic building blocks for deep + learning applications. + + oneDNN is intended for deep learning applications and framework + developers interested in improving application performance. + Deep learning practitioners should use one of the applications + enabled with oneDNN. + + ${{ + desc_omp if dnnl_cpu_runtime == "omp" else + desc_tbb if dnnl_cpu_runtime == "tbb" else + desc_threadpool if dnnl_cpu_runtime == "threadpool" else + desc_dpcpp if dnnl_cpu_runtime == "dpcpp" + }} + + For more information please read oneDNN developer guide: + https://oneapi-src.github.io/oneDNN/ + homepage: https://github.com/oneapi-src/oneDNN + +about: + license: Apache-2.0 + license_file: + - LICENSE + - THIRD-PARTY-PROGRAMS + summary: oneAPI Deep Neural Network Library (oneDNN) + homepage: https://github.com/oneapi-src/oneDNN + +extra: + feedstock-name: onednn + recipe-maintainers: + - xhochy + - densamoilov + - spalicki diff --git a/tests/test_v1_yaml/version_build_number_via_context.yaml b/tests/test_v1_yaml/version_build_number_via_context.yaml new file mode 100644 index 000000000..923728093 --- /dev/null +++ b/tests/test_v1_yaml/version_build_number_via_context.yaml @@ -0,0 +1,227 @@ +context: + version: 0.20.0 + build_number: 3 + # see github.com/conda-forge/conda-forge.github.io/issues/1059 for naming discussion + # torchvision requires that CUDA major and minor versions match with pytorch + # https://github.com/pytorch/vision/blob/fa99a5360fbcd1683311d57a76fcc0e7323a4c1e/torchvision/extension.py#L79C1-L85C1 + torch_proc_type: ${{ "cuda" ~ cuda_compiler_version | version_to_buildstring if cuda_compiler_version != "None" else "cpu" }} + # Upstream has specific compatability ranges for pytorch and python which are + # updated every 0.x release. https://github.com/pytorch/vision#installation + compatible_pytorch: 2.5 + + tests_to_skip: > + ${{ 'skip test_url_is_accessible instead of hitting 20+ servers per run, since' if 0 }} + ${{ 'each server might be occasionally unresponsive and end up failing our CI' if 0 }} + test_url_is_accessible + ${{ 'spurious failure because upstream skip (Image.__version__ >= "7") does not trigger for Pillow "10"' if 0 }} + or (test_transforms and test_adjust_saturation) + ${{ 'osx warns with nnpack if there is no AVX2, see conda-forge/pytorch-cpu-feedstock#56' if 0 }} + ${{ "or test_adjust_sharpness" if osx }} + ${{ '2021/10/28 hmaarrfk: I am able to run it locally on a large machine.' if 0 }} + ${{ 'It seems to fail around testing of vgg' if 0 }} + ${{ 'This test seems to just destroy the memory of the system.' if 0 }} + or test_forward_backward + or test_jit_forward_backward + ${{ '2022/01/21 hmaarrfk (test_frame_reading)' if 0 }} + ${{ 'They indicate that there can be a 1% error in their test.' if 0 }} + ${{ 'However, this test seems to causing the CIs to fail when this' if 0 }} + ${{ 'case is hit. For example the last CI failed with' if 0 }} + ${{ '> assert mean_delta.item() < 2.5' if 0 }} + ${{ 'E assert 2.502098560333252 < 2.5' if 0 }} + or test_frame_reading + ${{ 'Random perspective tests can fail if the perspective is too sharp' if 0 }} + ${{ 'https://github.com/conda-forge/torchvision-feedstock/issues/38' if 0 }} + or test_randomperspective_fill + ${{ 'Tolerance on the test_frozenbatchnorm2d_eps test seems to be too strict' if 0 }} + or test_frozenbatchnorm2d_eps + or test_random_apply + ${{ '2022/03/29 hmaarrfk' if 0 }} + ${{ 'It seems that this test can cause segmentation faults on the CIs.' if 0 }} + or test_write_video_with_audio + or test_video_clips_custom_fps + ${{ '2022/07 hmaarrfk really large memory tests. Fail on CIs' if 0 }} + or test_memory_efficient_densenet + or test_resnet_dilation + or test_mobilenet_v2_residual_setting + or test_mobilenet_norm_layer + or test_inception_v3_eval + or test_fasterrcnn_double + or test_googlenet_eval + or test_fasterrcnn_switch_devices + or test_mobilenet_v2_residual_setting + or test_vitc_models + or test_classification_model + or test_segmentation_model + or test_detection_model + or test_detection_model_validation + or test_video_model + or test_quantized_classification_model + or test_detection_model_trainable_backbone_layers + or test_raft + or test_build_fx_feature_extractor + ${{ "2023/01 These tests fail on newer numpy with module 'numpy' has no attribute 'int'" if 0 }} + or test_transformation_range + or test_transformation_discrete + ${{ '2023/05 The gaussian blur tests are known to be flaky due to some non-determinism on CUDA (see pytorch/vision#6755)' if 0 }} + or test_batched_vs_single + ${{ '2023/11 Draw boxes test broken by pillow 1.10.0, but is non-critical and the test is patched upstream (pytorch/vision#8051)' if 0 }} + or test_draw_boxes + ${{ '2024/02 These tests assert warnings and in PyTorch 2.1.2 the number of warnings increased' if 0 }} + ${{ 'causing them to fail' if 0 }} + or test_pretrained_pos or test_equivalent_behavior_weights + ${{ '2024/12 These tests use Internet' if 0 }} + or test_decode_gif or test_download_url or "test_get_model[lraspp" + +recipe: + name: torchvision + version: ${{ version }} + +source: + url: https://github.com/pytorch/vision/archive/refs/tags/v${{ version }}.tar.gz + sha256: b59d9896c5c957c6db0018754bbd17d079c5102b82b9be0b438553b40a7b6029 + patches: + # Our newer conda-forge clang compilers complain about this for OSX + # https://github.com/pytorch/vision/pull/8406/files#r1730151047 + - patches/0001-Use-system-giflib.patch + - patches/0002-Force-nvjpeg-and-force-failure.patch + # 2024/08 hmaarrfk + # known flaky test https://github.com/pytorch/vision/blob/9e78fe29e0851b10eb8fba0b88cc521ad67cf322/test/test_image.py#L840 + - patches/0003-Skip-OSS-CI-in-conda-forge-as-well.patch + # Can likely remove after 0.20.1 + # https://github.com/pytorch/vision/pull/8776 + - patches/8776_compatibility_with_pyav_14.patch + +build: + number: ${{ build_number }} + string: ${{ torch_proc_type }}_py${{ python | version_to_buildstring }}_h${{ hash }}_${{ build_number }} + # CUDA < 12 not supported by pytorch anymore + skip: cuda_compiler_version == "11.8" or win + +outputs: + - package: + name: torchvision + build: + script: + env: + BUILD_VERSION: ${{ version }} + requirements: + build: + - ${{ stdlib('c') }} + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - if: cuda_compiler_version != "None" + then: + - ${{ compiler('cuda') }} + # avoid nested conditions because of + # https://github.com/conda-forge/conda-smithy/issues/2165 + - if: build_platform != target_platform and cuda_compiler_version != "None" + then: + - libcublas-dev + - libcusolver-dev + - libcusparse-dev + - libnvjpeg-dev + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + # - numpy + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + host: + - python + # - numpy + - pip + - setuptools + - if: cuda_compiler_version != "None" + then: + - cudnn + - libcublas-dev + - libcusolver-dev + - libcusparse-dev + - libnvjpeg-dev + - libavif + # Add a "naked" libheif so that the global pinning is picked up + - libheif + # Add a constraint on the build time configuration of libheif to avoid gpl code + - libheif [build=lgpl*] + - libjpeg-turbo + - libpng + - libwebp + # https://github.com/pytorch/vision/pull/8406/files#r1730151047 + - giflib + # Specify lgpl version of ffmpeg so that there are + # no quesitons about the license of the resulting binary + # hmaarrfk: 2022/07, I think that torchvision just has bugs with ffmpeg + # - ffmpeg {{ ffmpeg }} [build=lgpl_*] + # exclude 8.3.0 and 8.3.1 specifically due to pytorch/vision#4146, python-pillow/Pillow#5571 + - pillow >=5.3.0,!=8.3.0,!=8.3.1 + - libtorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - requests + run: + - python + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - if: cuda_compiler_version != "None" + then: + - ${{ pin_compatible('cudnn') }} + - pillow >=5.3.0,!=8.3.0,!=8.3.1 + # They don't really document it, but it seems that they want a minimum version + # https://github.com/pytorch/vision/blob/v0.19.0/packaging/torchvision/meta.yaml#L26 + - numpy >=1.23.5 + # While their conda package depends on requests, it seems it is only used for some test + # scripts and not the runtime + # - requests + tests: + - python: + imports: + - torchvision + - torchvision.datasets + - torchvision.models + - torchvision.transforms + - torchvision.utils + pip_check: true + - requirements: + run: + - pip + script: + - pip list + - if: unix + then: pip list | grep torchvision | grep ${{ version }} + + - package: + name: torchvision-tests + build: + script: true + requirements: + run: + - ${{ pin_subpackage('torchvision', exact=True) }} + tests: + - files: + source: + - test/ + - references/ + - pytest.ini + requirements: + run: + - pytest + - requests + - av + - expecttest + - scipy + - pytest-mock + - pytest-socket + script: + - pytest --disable-socket --verbose -k "not (${{ tests_to_skip }})" --durations=50 test/ + +about: + license: BSD-3-Clause + license_file: LICENSE + summary: Image and video datasets and models for torch deep learning + homepage: http://pytorch.org/ + repository: https://github.com/pytorch/vision + +extra: + recipe-maintainers: + - nehaljwani + - hmaarrfk + - h-vetinari + feedstock-name: torchvision diff --git a/tests/test_v1_yaml/version_build_number_via_context_correct.yaml b/tests/test_v1_yaml/version_build_number_via_context_correct.yaml new file mode 100644 index 000000000..e194db297 --- /dev/null +++ b/tests/test_v1_yaml/version_build_number_via_context_correct.yaml @@ -0,0 +1,227 @@ +context: + version: "0.20.1" + build_number: 0 + # see github.com/conda-forge/conda-forge.github.io/issues/1059 for naming discussion + # torchvision requires that CUDA major and minor versions match with pytorch + # https://github.com/pytorch/vision/blob/fa99a5360fbcd1683311d57a76fcc0e7323a4c1e/torchvision/extension.py#L79C1-L85C1 + torch_proc_type: ${{ "cuda" ~ cuda_compiler_version | version_to_buildstring if cuda_compiler_version != "None" else "cpu" }} + # Upstream has specific compatability ranges for pytorch and python which are + # updated every 0.x release. https://github.com/pytorch/vision#installation + compatible_pytorch: '2.5' + + tests_to_skip: > + ${{ 'skip test_url_is_accessible instead of hitting 20+ servers per run, since' if 0 }} + ${{ 'each server might be occasionally unresponsive and end up failing our CI' if 0 }} + test_url_is_accessible + ${{ 'spurious failure because upstream skip (Image.__version__ >= "7") does not trigger for Pillow "10"' if 0 }} + or (test_transforms and test_adjust_saturation) + ${{ 'osx warns with nnpack if there is no AVX2, see conda-forge/pytorch-cpu-feedstock#56' if 0 }} + ${{ "or test_adjust_sharpness" if osx }} + ${{ '2021/10/28 hmaarrfk: I am able to run it locally on a large machine.' if 0 }} + ${{ 'It seems to fail around testing of vgg' if 0 }} + ${{ 'This test seems to just destroy the memory of the system.' if 0 }} + or test_forward_backward + or test_jit_forward_backward + ${{ '2022/01/21 hmaarrfk (test_frame_reading)' if 0 }} + ${{ 'They indicate that there can be a 1% error in their test.' if 0 }} + ${{ 'However, this test seems to causing the CIs to fail when this' if 0 }} + ${{ 'case is hit. For example the last CI failed with' if 0 }} + ${{ '> assert mean_delta.item() < 2.5' if 0 }} + ${{ 'E assert 2.502098560333252 < 2.5' if 0 }} + or test_frame_reading + ${{ 'Random perspective tests can fail if the perspective is too sharp' if 0 }} + ${{ 'https://github.com/conda-forge/torchvision-feedstock/issues/38' if 0 }} + or test_randomperspective_fill + ${{ 'Tolerance on the test_frozenbatchnorm2d_eps test seems to be too strict' if 0 }} + or test_frozenbatchnorm2d_eps + or test_random_apply + ${{ '2022/03/29 hmaarrfk' if 0 }} + ${{ 'It seems that this test can cause segmentation faults on the CIs.' if 0 }} + or test_write_video_with_audio + or test_video_clips_custom_fps + ${{ '2022/07 hmaarrfk really large memory tests. Fail on CIs' if 0 }} + or test_memory_efficient_densenet + or test_resnet_dilation + or test_mobilenet_v2_residual_setting + or test_mobilenet_norm_layer + or test_inception_v3_eval + or test_fasterrcnn_double + or test_googlenet_eval + or test_fasterrcnn_switch_devices + or test_mobilenet_v2_residual_setting + or test_vitc_models + or test_classification_model + or test_segmentation_model + or test_detection_model + or test_detection_model_validation + or test_video_model + or test_quantized_classification_model + or test_detection_model_trainable_backbone_layers + or test_raft + or test_build_fx_feature_extractor + ${{ "2023/01 These tests fail on newer numpy with module 'numpy' has no attribute 'int'" if 0 }} + or test_transformation_range + or test_transformation_discrete + ${{ '2023/05 The gaussian blur tests are known to be flaky due to some non-determinism on CUDA (see pytorch/vision#6755)' if 0 }} + or test_batched_vs_single + ${{ '2023/11 Draw boxes test broken by pillow 1.10.0, but is non-critical and the test is patched upstream (pytorch/vision#8051)' if 0 }} + or test_draw_boxes + ${{ '2024/02 These tests assert warnings and in PyTorch 2.1.2 the number of warnings increased' if 0 }} + ${{ 'causing them to fail' if 0 }} + or test_pretrained_pos or test_equivalent_behavior_weights + ${{ '2024/12 These tests use Internet' if 0 }} + or test_decode_gif or test_download_url or "test_get_model[lraspp" + +recipe: + name: torchvision + version: ${{ version }} + +source: + url: https://github.com/pytorch/vision/archive/refs/tags/v${{ version }}.tar.gz + sha256: 7e08c7f56e2c89859310e53d898f72bccc4987cd83e08cfd6303513da15a9e71 + patches: + # Our newer conda-forge clang compilers complain about this for OSX + # https://github.com/pytorch/vision/pull/8406/files#r1730151047 + - patches/0001-Use-system-giflib.patch + - patches/0002-Force-nvjpeg-and-force-failure.patch + # 2024/08 hmaarrfk + # known flaky test https://github.com/pytorch/vision/blob/9e78fe29e0851b10eb8fba0b88cc521ad67cf322/test/test_image.py#L840 + - patches/0003-Skip-OSS-CI-in-conda-forge-as-well.patch + # Can likely remove after 0.20.1 + # https://github.com/pytorch/vision/pull/8776 + - patches/8776_compatibility_with_pyav_14.patch + +build: + number: ${{ build_number }} + string: ${{ torch_proc_type }}_py${{ python | version_to_buildstring }}_h${{ hash }}_${{ build_number }} + # CUDA < 12 not supported by pytorch anymore + skip: cuda_compiler_version == "11.8" or win + +outputs: + - package: + name: torchvision + build: + script: + env: + BUILD_VERSION: ${{ version }} + requirements: + build: + - ${{ stdlib('c') }} + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - if: cuda_compiler_version != "None" + then: + - ${{ compiler('cuda') }} + # avoid nested conditions because of + # https://github.com/conda-forge/conda-smithy/issues/2165 + - if: build_platform != target_platform and cuda_compiler_version != "None" + then: + - libcublas-dev + - libcusolver-dev + - libcusparse-dev + - libnvjpeg-dev + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + # - numpy + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + host: + - python + # - numpy + - pip + - setuptools + - if: cuda_compiler_version != "None" + then: + - cudnn + - libcublas-dev + - libcusolver-dev + - libcusparse-dev + - libnvjpeg-dev + - libavif + # Add a "naked" libheif so that the global pinning is picked up + - libheif + # Add a constraint on the build time configuration of libheif to avoid gpl code + - libheif [build=lgpl*] + - libjpeg-turbo + - libpng + - libwebp + # https://github.com/pytorch/vision/pull/8406/files#r1730151047 + - giflib + # Specify lgpl version of ffmpeg so that there are + # no quesitons about the license of the resulting binary + # hmaarrfk: 2022/07, I think that torchvision just has bugs with ffmpeg + # - ffmpeg {{ ffmpeg }} [build=lgpl_*] + # exclude 8.3.0 and 8.3.1 specifically due to pytorch/vision#4146, python-pillow/Pillow#5571 + - pillow >=5.3.0,!=8.3.0,!=8.3.1 + - libtorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - requests + run: + - python + - pytorch ${{ compatible_pytorch }}.* [build=${{ torch_proc_type }}*] + - if: cuda_compiler_version != "None" + then: + - ${{ pin_compatible('cudnn') }} + - pillow >=5.3.0,!=8.3.0,!=8.3.1 + # They don't really document it, but it seems that they want a minimum version + # https://github.com/pytorch/vision/blob/v0.19.0/packaging/torchvision/meta.yaml#L26 + - numpy >=1.23.5 + # While their conda package depends on requests, it seems it is only used for some test + # scripts and not the runtime + # - requests + tests: + - python: + imports: + - torchvision + - torchvision.datasets + - torchvision.models + - torchvision.transforms + - torchvision.utils + pip_check: true + - requirements: + run: + - pip + script: + - pip list + - if: unix + then: pip list | grep torchvision | grep ${{ version }} + + - package: + name: torchvision-tests + build: + script: true + requirements: + run: + - ${{ pin_subpackage('torchvision', exact=True) }} + tests: + - files: + source: + - test/ + - references/ + - pytest.ini + requirements: + run: + - pytest + - requests + - av + - expecttest + - scipy + - pytest-mock + - pytest-socket + script: + - pytest --disable-socket --verbose -k "not (${{ tests_to_skip }})" --durations=50 test/ + +about: + license: BSD-3-Clause + license_file: LICENSE + summary: Image and video datasets and models for torch deep learning + homepage: http://pytorch.org/ + repository: https://github.com/pytorch/vision + +extra: + recipe-maintainers: + - nehaljwani + - hmaarrfk + - h-vetinari + feedstock-name: torchvision diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 8c38ba38a..cf5b57983 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -120,6 +120,8 @@ def test_version_up(case, new_ver, tmpdir, caplog): [ ("pypi_url", "0.7.1"), ("jolt", "5.2.0"), + ("build_number_via_context", "0.20.1"), + ("build_as_expr", "3.6.2"), ], ) @flaky From 83e76b1e1833127817b07b18e4e6227bb970ac9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 19:46:39 +0100 Subject: [PATCH 19/40] Fix "target_platform" in .ci_support, in v1 migrator tests --- tests/test_migrators.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 63c0659a6..cdf880153 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -484,7 +484,8 @@ def run_test_migration( tmpdir_p.joinpath("recipe").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( - "target_platform: linux-64" + "target_platform:\n" + "- linux-64" ) recipe_dir = str(tmpdir_p / "recipe") From 30fa3e27b4069d3af7056e64d459f997728d4b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 19:49:51 +0100 Subject: [PATCH 20/40] extend migrator tests to osx-arm64 and win-64 platforms --- tests/test_migrators.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index cdf880153..08039c243 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -483,10 +483,11 @@ def run_test_migration( tmpdir_p.joinpath(".ci_support").mkdir() tmpdir_p.joinpath("recipe").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) - (tmpdir_p / ".ci_support" / "linux_64_.yaml").write_text( - "target_platform:\n" - "- linux-64" - ) + for target_platform in ("linux-64", "osx-arm64", "win-64"): + (tmpdir_p / ".ci_support" / f"{target_platform.replace('-', '_')}_.yaml").write_text( + "target_platform:\n" + f"- {target_platform}\n" + ) recipe_dir = str(tmpdir_p / "recipe") else: From 3125287dab0e68c15f90c64caa1d08f69c90bb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 19:54:32 +0100 Subject: [PATCH 21/40] add a test with platform-conditional sources --- .../version_conditional_sources.yaml | 65 +++++++++++++++++++ .../version_conditional_sources_correct.yaml | 65 +++++++++++++++++++ tests/test_version_migrator.py | 1 + 3 files changed, 131 insertions(+) create mode 100644 tests/test_v1_yaml/version_conditional_sources.yaml create mode 100644 tests/test_v1_yaml/version_conditional_sources_correct.yaml diff --git a/tests/test_v1_yaml/version_conditional_sources.yaml b/tests/test_v1_yaml/version_conditional_sources.yaml new file mode 100644 index 000000000..9d5e747c0 --- /dev/null +++ b/tests/test_v1_yaml/version_conditional_sources.yaml @@ -0,0 +1,65 @@ +context: + name: blpapi + version: 3.24.10 + blpapicpp_linux_version: 3.24.10.1 + blpapicpp_win_version: 3.24.10.1 + blpapicpp_osx_version: 3.24.10.1 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + - url: https://blpapi.bloomberg.com/repository/releases/python/${{ name }}-${{ version }}.tar.gz + sha256: 6ff5b6eb9c4d154f311d57d34c12ac839b634331add069a38965f11b0fe38794 + - if: linux + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_linux_version }}-linux.tar.gz + sha256: 0b5498c93af191460c069658a9235c18058e4f7afd94dbb1d7e1020602a6ac2a + target_directory: blpapi + - if: win + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_win_version }}-windows.zip + sha256: 88a82f3e5284f2b5e7f8470b1d896972073cdc8350a60187e25ce1bd0c6b361e + target_directory: blpapi + - if: osx and arm64 + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_osx_version }}-macos-arm64.tar.gz + sha256: 64f7a5a0e8750ca63a0fc022516861234676a7a7a854843d82ed64edc1a27c90 + target_directory: blpapi + +build: + number: 0 + skip: osx and x86_64 + +requirements: + build: + - ${{ compiler('cxx') }} + - ${{ stdlib("c") }} + - if: build_platform != target_platform + then: python + - if: build_platform != target_platform + then: cross-python_${{ target_platform }} + host: + - python + - setuptools + - pip + run: + - python + +tests: + - python: + imports: + - blpapi + +about: + license: LicenseRef-Bloomberg-BLPAPI + license_file: License.txt + summary: Python SDK for Bloomberg BLPAPI + homepage: https://www.bloomberg.com/professional/support/api-library/ + +extra: + recipe-maintainers: + - reggied + - matthewgilbert + - adament diff --git a/tests/test_v1_yaml/version_conditional_sources_correct.yaml b/tests/test_v1_yaml/version_conditional_sources_correct.yaml new file mode 100644 index 000000000..d1d9bb64b --- /dev/null +++ b/tests/test_v1_yaml/version_conditional_sources_correct.yaml @@ -0,0 +1,65 @@ +context: + name: blpapi + version: "3.24.11" + blpapicpp_linux_version: 3.24.10.1 + blpapicpp_win_version: 3.24.10.1 + blpapicpp_osx_version: 3.24.10.1 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + - url: https://blpapi.bloomberg.com/repository/releases/python/${{ name }}-${{ version }}.tar.gz + sha256: 94dc699df262187b3afe4f163526aac67fb5982b008efe851836e9f1cd5358c1 + - if: linux + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_linux_version }}-linux.tar.gz + sha256: 0b5498c93af191460c069658a9235c18058e4f7afd94dbb1d7e1020602a6ac2a + target_directory: blpapi + - if: win + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_win_version }}-windows.zip + sha256: 88a82f3e5284f2b5e7f8470b1d896972073cdc8350a60187e25ce1bd0c6b361e + target_directory: blpapi + - if: osx and arm64 + then: + url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_osx_version }}-macos-arm64.tar.gz + sha256: 64f7a5a0e8750ca63a0fc022516861234676a7a7a854843d82ed64edc1a27c90 + target_directory: blpapi + +build: + number: 0 + skip: osx and x86_64 + +requirements: + build: + - ${{ compiler('cxx') }} + - ${{ stdlib("c") }} + - if: build_platform != target_platform + then: python + - if: build_platform != target_platform + then: cross-python_${{ target_platform }} + host: + - python + - setuptools + - pip + run: + - python + +tests: + - python: + imports: + - blpapi + +about: + license: LicenseRef-Bloomberg-BLPAPI + license_file: License.txt + summary: Python SDK for Bloomberg BLPAPI + homepage: https://www.bloomberg.com/professional/support/api-library/ + +extra: + recipe-maintainers: + - reggied + - matthewgilbert + - adament diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index cf5b57983..cc04484df 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -122,6 +122,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("jolt", "5.2.0"), ("build_number_via_context", "0.20.1"), ("build_as_expr", "3.6.2"), + ("conditional_sources", "3.24.11"), ], ) @flaky From e724d4bc372d61856fc64c6c37ac86e7085c2c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Wed, 8 Jan 2025 20:01:42 +0100 Subject: [PATCH 22/40] update the last example to use ${{ version }} in all subversions --- tests/test_v1_yaml/version_conditional_sources.yaml | 6 +++--- .../version_conditional_sources_correct.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_v1_yaml/version_conditional_sources.yaml b/tests/test_v1_yaml/version_conditional_sources.yaml index 9d5e747c0..8b15bd497 100644 --- a/tests/test_v1_yaml/version_conditional_sources.yaml +++ b/tests/test_v1_yaml/version_conditional_sources.yaml @@ -1,9 +1,9 @@ context: name: blpapi version: 3.24.10 - blpapicpp_linux_version: 3.24.10.1 - blpapicpp_win_version: 3.24.10.1 - blpapicpp_osx_version: 3.24.10.1 + blpapicpp_linux_version: ${{ version }}.1 + blpapicpp_win_version: ${{ version }}.1 + blpapicpp_osx_version: ${{ version }}.1 package: name: ${{ name|lower }} diff --git a/tests/test_v1_yaml/version_conditional_sources_correct.yaml b/tests/test_v1_yaml/version_conditional_sources_correct.yaml index d1d9bb64b..30a9d3707 100644 --- a/tests/test_v1_yaml/version_conditional_sources_correct.yaml +++ b/tests/test_v1_yaml/version_conditional_sources_correct.yaml @@ -1,9 +1,9 @@ context: name: blpapi version: "3.24.11" - blpapicpp_linux_version: 3.24.10.1 - blpapicpp_win_version: 3.24.10.1 - blpapicpp_osx_version: 3.24.10.1 + blpapicpp_linux_version: ${{ version }}.1 + blpapicpp_win_version: ${{ version }}.1 + blpapicpp_osx_version: ${{ version }}.1 package: name: ${{ name|lower }} @@ -15,17 +15,17 @@ source: - if: linux then: url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_linux_version }}-linux.tar.gz - sha256: 0b5498c93af191460c069658a9235c18058e4f7afd94dbb1d7e1020602a6ac2a + sha256: eb1a9aa834a969a8ccbbb04061274623659a1fb273abda4413b47fe59e7ee412 target_directory: blpapi - if: win then: url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_win_version }}-windows.zip - sha256: 88a82f3e5284f2b5e7f8470b1d896972073cdc8350a60187e25ce1bd0c6b361e + sha256: e63c0b75d50097194c425489a08cec9c0374ab7b8eafe5b015c076bb2432fa19 target_directory: blpapi - if: osx and arm64 then: url: https://blpapi.bloomberg.com/download/releases/raw/files/blpapi_cpp_${{ blpapicpp_osx_version }}-macos-arm64.tar.gz - sha256: 64f7a5a0e8750ca63a0fc022516861234676a7a7a854843d82ed64edc1a27c90 + sha256: fa96331edf06dd2342cb27771367516296067e94961ec6e600add1c2eed9c41d target_directory: blpapi build: From 68b4b2595ac419cca53f616c1bfce5436e2c34db Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 8 Jan 2025 18:42:37 +0100 Subject: [PATCH 23/40] Update conda_forge_tick/migrators/version.py Co-authored-by: Matthew R. Becker --- conda_forge_tick/migrators/version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index fd7366cb9..66dfe14ae 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -101,7 +101,6 @@ def filter( {}, 0, ) - print("Checking schema version: %d" % schema_version) if schema_version == 0: if "raw_meta_yaml" not in attrs: return True From 2c15d1822e84977d1c5dc832b8a91883be3975fa Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 8 Jan 2025 19:00:38 +0100 Subject: [PATCH 24/40] avoid redundant `recipe_path.read_text()` --- conda_forge_tick/update_recipe/version.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 9625c60f0..2ccc1e86f 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -702,7 +702,8 @@ def update_version_v1( feedstock_dir = Path(feedstock_dir) recipe_path = feedstock_dir / "recipe" / "recipe.yaml" - recipe_yaml = load_yaml(recipe_path.read_text()) + recipe_text = recipe_path.read_text() + recipe_yaml = load_yaml(recipe_text) variants = feedstock_dir.glob(".ci_support/*.yaml") # load all variants variants = [load_yaml(variant.read_text()) for variant in variants] @@ -714,8 +715,6 @@ def update_version_v1( recipe_yaml, variants, override_version=version ) - recipe_text = recipe_path.read_text() - # update the version with a regex replace for line in recipe_text.splitlines(): if match := re.match(r"^(\s+)version:\s.*$", line): From 7d7fb97c6d5ed1fa3d8b5afb3af2cedd7bce6229 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 8 Jan 2025 19:06:54 +0100 Subject: [PATCH 25/40] more consistency about recipe_path_*` variable naming --- conda_forge_tick/migrators/version.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 66dfe14ae..60f4c3bcf 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -221,19 +221,19 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] recipe_dir = Path(recipe_dir) - recipe = None + recipe_path = None recipe_path_v0 = recipe_dir / "meta.yaml" recipe_path_v1 = recipe_dir / "recipe.yaml" if recipe_path_v0.exists(): raw_meta_yaml = recipe_path_v0.read_text() - recipe = recipe_path_v0 + recipe_path = recipe_path_v0 updated_meta_yaml, errors = update_version( raw_meta_yaml, version, hash_type=hash_type, ) elif recipe_path_v1.exists(): - recipe = recipe_path_v1 + recipe_path = recipe_path_v1 updated_meta_yaml, errors = update_version_v1( # we need to give the "feedstock_dir" (not recipe dir) recipe_dir.parent, @@ -246,8 +246,8 @@ def migrate( ) if len(errors) == 0 and updated_meta_yaml is not None: - recipe.write_text(updated_meta_yaml) - self.set_build_number(recipe) + recipe_path.write_text(updated_meta_yaml) + self.set_build_number(recipe_path) return super().migrate(recipe_dir, attrs) else: From 9cc9b376a8d955ad29c9f011a6f1cfa92b3cfd48 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 8 Jan 2025 19:09:53 +0100 Subject: [PATCH 26/40] also use recipe_path in _update_version_feedstock_dir_local --- conda_forge_tick/update_recipe/version.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 2ccc1e86f..d7345eb49 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -601,22 +601,24 @@ def _update_version_feedstock_dir_local( ) -> (bool, set): feedstock_path = Path(feedstock_dir) - recipe = feedstock_path / "recipe" / "meta.yaml" - if recipe.exists(): + recipe_path = None + recipe_path_v0 = feedstock_path / "recipe" / "meta.yaml" + recipe_path_v1 = feedstock_path / "recipe" / "recipe.yaml" + if recipe_path_v0.exists(): + recipe_path = recipe_path_v0 updated_meta_yaml, errors = update_version( recipe.read_text(), version, hash_type=hash_type ) + elif recipe_path_v1.exists(): + recipe_path = recipe_path_v1 + updated_meta_yaml, errors = update_version_v1( + feedstock_dir, version, hash_type + ) else: - recipe = feedstock_path / "recipe" / "recipe.yaml" - if recipe.exists(): - updated_meta_yaml, errors = update_version_v1( - feedstock_dir, version, hash_type - ) - else: - return False, {"no recipe found"} + return False, {"no recipe found"} if updated_meta_yaml is not None: - recipe.write_text(updated_meta_yaml) + recipe_path.write_text(updated_meta_yaml) return updated_meta_yaml is not None, errors From 5e98ca29bae59c531a558255b7f0f2f83b6aab2d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:36:53 +0000 Subject: [PATCH 27/40] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conda_forge_tick/update_recipe/version.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index d7345eb49..bb6f5dbff 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -611,9 +611,7 @@ def _update_version_feedstock_dir_local( ) elif recipe_path_v1.exists(): recipe_path = recipe_path_v1 - updated_meta_yaml, errors = update_version_v1( - feedstock_dir, version, hash_type - ) + updated_meta_yaml, errors = update_version_v1(feedstock_dir, version, hash_type) else: return False, {"no recipe found"} From cae141b5e6c33e08037227167e2c805213262472 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 9 Jan 2025 08:20:41 +0100 Subject: [PATCH 28/40] fix overlooked `recipe` --- conda_forge_tick/update_recipe/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index bb6f5dbff..fda4ab1a0 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -607,7 +607,7 @@ def _update_version_feedstock_dir_local( if recipe_path_v0.exists(): recipe_path = recipe_path_v0 updated_meta_yaml, errors = update_version( - recipe.read_text(), version, hash_type=hash_type + recipe_path_v0.read_text(), version, hash_type=hash_type ) elif recipe_path_v1.exists(): recipe_path = recipe_path_v1 From 8a80069eb402bb2c889f4b77e7dddae9bf079c36 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 9 Jan 2025 14:02:28 +0100 Subject: [PATCH 29/40] add cranmirror test --- conda_forge_tick/update_recipe/version.py | 13 +++- tests/test_v1_yaml/version_cranmirror.yaml | 68 +++++++++++++++++++ .../version_cranmirror_correct.yaml | 68 +++++++++++++++++++ tests/test_version_migrator.py | 1 + 4 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 tests/test_v1_yaml/version_cranmirror.yaml create mode 100644 tests/test_v1_yaml/version_cranmirror_correct.yaml diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index fda4ab1a0..42ebd97de 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -715,6 +715,15 @@ def update_version_v1( recipe_yaml, variants, override_version=version ) + # mangle the version if it is R + for source in rendered_sources: + if isinstance(source.template, list): + if any([_is_r_url(t) for t in source.template]): + version = version.replace("_", "-") + else: + if _is_r_url(source.template): + version = version.replace("_", "-") + # update the version with a regex replace for line in recipe_text.splitlines(): if match := re.match(r"^(\s+)version:\s.*$", line): @@ -758,8 +767,8 @@ def update_version_v1( # convert back to v1 minijinja template new_tmpl = new_tmpl.replace("{{", "${{") - if new_tmpl != source.template: - recipe_text = recipe_text.replace(source.template, new_tmpl) + if new_tmpl != template: + recipe_text = recipe_text.replace(template, new_tmpl) break diff --git a/tests/test_v1_yaml/version_cranmirror.yaml b/tests/test_v1_yaml/version_cranmirror.yaml new file mode 100644 index 000000000..0468dfe72 --- /dev/null +++ b/tests/test_v1_yaml/version_cranmirror.yaml @@ -0,0 +1,68 @@ +context: + version: '0.3.2' + posix: ${{ 'm2-' if win else '' }} + native: ${{ 'm2w64-' if win else '' }} + cran_mirror: https://cran.r-project.org + +package: + name: r-taxa + version: ${{ version|replace("-", "_") }} + +source: + url: + - ${{ cran_mirror }}/src/contrib/taxa_${{ version }}.tar.gz + - ${{ cran_mirror }}/src/contrib/Archive/taxa/taxa_${{ version }}.tar.gz + sha256: 0000db83bfed738aec4456f54e3722f0a0aff38f9f3896e4640ca87f7b45a287 + +build: + number: 10 + noarch: generic + +requirements: + build: + - ${{posix}}zip + host: + - r-base + - r-r6 + - r-crayon + - r-dplyr + - r-jsonlite + - r-knitr + - r-lazyeval + - r-magrittr + - r-rlang + - r-stringr + - r-taxize + - r-tibble + - r-tidyr + run: + - r-base + - r-r6 + - r-crayon + - r-dplyr + - r-jsonlite + - r-knitr + - r-lazyeval + - r-magrittr + - r-rlang + - r-stringr + - r-taxize + - r-tibble + - r-tidyr + +tests: + - script: + - $R -e "library('taxa')" + +about: + homepage: https://github.com/ropensci/taxa + license: MIT + license_file: LICENSE + summary: | + Provides taxonomic classes for groupings of taxonomic names without data, and those + with data. Methods provided are "taxonomically aware", in that they know about ordering + of ranks, and methods that filter based on taxonomy also filter associated data. + +extra: + recipe-maintainers: + - conda-forge/r diff --git a/tests/test_v1_yaml/version_cranmirror_correct.yaml b/tests/test_v1_yaml/version_cranmirror_correct.yaml new file mode 100644 index 000000000..3681f01fa --- /dev/null +++ b/tests/test_v1_yaml/version_cranmirror_correct.yaml @@ -0,0 +1,68 @@ +context: + version: "0.3.3" + posix: ${{ 'm2-' if win else '' }} + native: ${{ 'm2w64-' if win else '' }} + cran_mirror: https://cran.r-project.org + +package: + name: r-taxa + version: ${{ version|replace("-", "_") }} + +source: + url: + - ${{ cran_mirror }}/src/contrib/taxa_${{ version }}.tar.gz + - ${{ cran_mirror }}/src/contrib/Archive/taxa/taxa_${{ version }}.tar.gz + sha256: 4822db83bfed738aec4456f54e3722f0a0aff38f9f3896e4640ca87f7b45a287 + +build: + number: 0 + noarch: generic + +requirements: + build: + - ${{posix}}zip + host: + - r-base + - r-r6 + - r-crayon + - r-dplyr + - r-jsonlite + - r-knitr + - r-lazyeval + - r-magrittr + - r-rlang + - r-stringr + - r-taxize + - r-tibble + - r-tidyr + run: + - r-base + - r-r6 + - r-crayon + - r-dplyr + - r-jsonlite + - r-knitr + - r-lazyeval + - r-magrittr + - r-rlang + - r-stringr + - r-taxize + - r-tibble + - r-tidyr + +tests: + - script: + - $R -e "library('taxa')" + +about: + homepage: https://github.com/ropensci/taxa + license: MIT + license_file: LICENSE + summary: | + Provides taxonomic classes for groupings of taxonomic names without data, and those + with data. Methods provided are "taxonomically aware", in that they know about ordering + of ranks, and methods that filter based on taxonomy also filter associated data. + +extra: + recipe-maintainers: + - conda-forge/r diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index cc04484df..99df8d519 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -123,6 +123,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("build_number_via_context", "0.20.1"), ("build_as_expr", "3.6.2"), ("conditional_sources", "3.24.11"), + ("cranmirror", "0.3.3"), ], ) @flaky From e9d70ca347f59267b420fe35bc32ad6ad86b486d Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 9 Jan 2025 14:16:25 +0100 Subject: [PATCH 30/40] add one more test --- tests/test_v1_yaml/version_event_stream.yaml | 42 +++++++++++++++++++ .../version_event_stream_correct.yaml | 42 +++++++++++++++++++ tests/test_version_migrator.py | 1 + 3 files changed, 85 insertions(+) create mode 100644 tests/test_v1_yaml/version_event_stream.yaml create mode 100644 tests/test_v1_yaml/version_event_stream_correct.yaml diff --git a/tests/test_v1_yaml/version_event_stream.yaml b/tests/test_v1_yaml/version_event_stream.yaml new file mode 100644 index 000000000..f55ef5efd --- /dev/null +++ b/tests/test_v1_yaml/version_event_stream.yaml @@ -0,0 +1,42 @@ +context: + name: event-stream + version: 1.6.2 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + - url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/event_stream-${{ version }}.tar.gz + sha256: 823dada12821552c7a82ebd47e4a279079dc6693cb1b86c64b6f857459b99d9e + - path: npy_2_compat.h + target_directory: python + +build: + script: ${{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv + number: 2 + skip: win + +requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - ${{ stdlib("c") }} + host: + - python + - setuptools + - wheel + - numpy + - pip + run: + - python + +about: + homepage: https://github.com/neuromorphicsystems/event_stream + summary: Read and write Event Stream (.es) files + license: MIT + license_file: LICENSE + +extra: + recipe-maintainers: + - Tobias-Fischer diff --git a/tests/test_v1_yaml/version_event_stream_correct.yaml b/tests/test_v1_yaml/version_event_stream_correct.yaml new file mode 100644 index 000000000..6892159bf --- /dev/null +++ b/tests/test_v1_yaml/version_event_stream_correct.yaml @@ -0,0 +1,42 @@ +context: + name: event-stream + version: "1.6.3" + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + - url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/event_stream-${{ version }}.tar.gz + sha256: a5ba0297bf81109294997673e1a9ad9835f75d6d7eabe92f16f1a3c176cbe944 + - path: npy_2_compat.h + target_directory: python + +build: + script: ${{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv + number: 0 + skip: win + +requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - ${{ stdlib("c") }} + host: + - python + - setuptools + - wheel + - numpy + - pip + run: + - python + +about: + homepage: https://github.com/neuromorphicsystems/event_stream + summary: Read and write Event Stream (.es) files + license: MIT + license_file: LICENSE + +extra: + recipe-maintainers: + - Tobias-Fischer diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 99df8d519..9dc93293a 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -124,6 +124,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("build_as_expr", "3.6.2"), ("conditional_sources", "3.24.11"), ("cranmirror", "0.3.3"), + ("event_stream", "1.6.3"), ], ) @flaky From 4461940d3dfba1b95c6fb568f5bfb9ed4db9cb29 Mon Sep 17 00:00:00 2001 From: Julian Hofer Date: Thu, 9 Jan 2025 15:42:45 +0100 Subject: [PATCH 31/40] Adapt test --- tests/{test_recipe_yaml => test_v1_yaml}/version_selshaurl.yaml | 0 .../version_selshaurl_correct.yaml | 0 tests/test_version_migrator.py | 1 + 3 files changed, 1 insertion(+) rename tests/{test_recipe_yaml => test_v1_yaml}/version_selshaurl.yaml (100%) rename tests/{test_recipe_yaml => test_v1_yaml}/version_selshaurl_correct.yaml (100%) diff --git a/tests/test_recipe_yaml/version_selshaurl.yaml b/tests/test_v1_yaml/version_selshaurl.yaml similarity index 100% rename from tests/test_recipe_yaml/version_selshaurl.yaml rename to tests/test_v1_yaml/version_selshaurl.yaml diff --git a/tests/test_recipe_yaml/version_selshaurl_correct.yaml b/tests/test_v1_yaml/version_selshaurl_correct.yaml similarity index 100% rename from tests/test_recipe_yaml/version_selshaurl_correct.yaml rename to tests/test_v1_yaml/version_selshaurl_correct.yaml diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 9dc93293a..a85e064f2 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -125,6 +125,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("conditional_sources", "3.24.11"), ("cranmirror", "0.3.3"), ("event_stream", "1.6.3"), + ("selshaurl", "3.7.0"), ], ) @flaky From 5316643f22079c2e5c7f5ef0bcc7f29caf6eda4e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:22:19 +0000 Subject: [PATCH 32/40] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_migrators.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 08039c243..205d082e1 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -484,10 +484,9 @@ def run_test_migration( tmpdir_p.joinpath("recipe").mkdir() tmpdir_p.joinpath("recipe", "recipe.yaml").write_text(inp) for target_platform in ("linux-64", "osx-arm64", "win-64"): - (tmpdir_p / ".ci_support" / f"{target_platform.replace('-', '_')}_.yaml").write_text( - "target_platform:\n" - f"- {target_platform}\n" - ) + ( + tmpdir_p / ".ci_support" / f"{target_platform.replace('-', '_')}_.yaml" + ).write_text("target_platform:\n" f"- {target_platform}\n") recipe_dir = str(tmpdir_p / "recipe") else: From 7b11a301e496f6dc38c91af3593db5c407e25e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Thu, 9 Jan 2025 16:38:05 +0100 Subject: [PATCH 33/40] add a test case based on libssh-feedstock --- tests/test_v1_yaml/version_libssh.yaml | 58 +++++++++++++++++++ .../test_v1_yaml/version_libssh_correct.yaml | 58 +++++++++++++++++++ tests/test_version_migrator.py | 1 + 3 files changed, 117 insertions(+) create mode 100644 tests/test_v1_yaml/version_libssh.yaml create mode 100644 tests/test_v1_yaml/version_libssh_correct.yaml diff --git a/tests/test_v1_yaml/version_libssh.yaml b/tests/test_v1_yaml/version_libssh.yaml new file mode 100644 index 000000000..bb604761a --- /dev/null +++ b/tests/test_v1_yaml/version_libssh.yaml @@ -0,0 +1,58 @@ +schema_version: 1 + +context: + name: libssh + version: 0.10.6 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://www.libssh.org/files/${{ (version | split("."))[:2] | join(".") }}/${{ name }}-${{ version }}.tar.xz + sha256: 1861d498f5b6f1741b6abc73e608478491edcf9c9d4b6630eef6e74596de9dc1 + +build: + number: 2 + skip: not unix + +requirements: + build: + - ${{ compiler('cxx') }} + - ${{ compiler('c') }} + - ${{ stdlib('c') }} + - cmake + - make + host: + - openssl + - zlib + - krb5 + run_exports: + # We pin to a minor version due to an incompatibility + # between 0.8 and 0.10 + # https://abi-laboratory.pro/index.php?view=timeline&l=libssh + - ${{ pin_subpackage('libssh', upper_bound='x.x') }} + +tests: + - script: + - test -f $PREFIX/include/libssh/libssh.h + - test -f $PREFIX/lib/pkgconfig/libssh.pc + - test -f $PREFIX/lib/libssh$SHLIB_EXT + +about: + homepage: https://libssh.org + license: LGPL-2.1-or-later + license_file: COPYING + summary: libssh - The SSH library + + description: | + libssh is a multiplatform C library implementing the SSHv2 protocol + on client and server side. With libssh, you can remotely execute + programs, transfer files, use a secure and transparent tunnel, + manage public keys and much more ... + documentation: https://www.libssh.org/documentation/ + +extra: + recipe-maintainers: + - jan-janssen + - matthiasdiener diff --git a/tests/test_v1_yaml/version_libssh_correct.yaml b/tests/test_v1_yaml/version_libssh_correct.yaml new file mode 100644 index 000000000..3b05cb31c --- /dev/null +++ b/tests/test_v1_yaml/version_libssh_correct.yaml @@ -0,0 +1,58 @@ +schema_version: 1 + +context: + name: libssh + version: 0.11.1 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://www.libssh.org/files/${{ (version | split("."))[:2] | join(".") }}/${{ name }}-${{ version }}.tar.xz + sha256: 14b7dcc72e91e08151c58b981a7b570ab2663f630e7d2837645d5a9c612c1b79 + +build: + number: 0 + skip: not unix + +requirements: + build: + - ${{ compiler('cxx') }} + - ${{ compiler('c') }} + - ${{ stdlib('c') }} + - cmake + - make + host: + - openssl + - zlib + - krb5 + run_exports: + # We pin to a minor version due to an incompatibility + # between 0.8 and 0.10 + # https://abi-laboratory.pro/index.php?view=timeline&l=libssh + - ${{ pin_subpackage('libssh', upper_bound='x.x') }} + +tests: + - script: + - test -f $PREFIX/include/libssh/libssh.h + - test -f $PREFIX/lib/pkgconfig/libssh.pc + - test -f $PREFIX/lib/libssh$SHLIB_EXT + +about: + homepage: https://libssh.org + license: LGPL-2.1-or-later + license_file: COPYING + summary: libssh - The SSH library + + description: | + libssh is a multiplatform C library implementing the SSHv2 protocol + on client and server side. With libssh, you can remotely execute + programs, transfer files, use a secure and transparent tunnel, + manage public keys and much more ... + documentation: https://www.libssh.org/documentation/ + +extra: + recipe-maintainers: + - jan-janssen + - matthiasdiener diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index a85e064f2..7af900a01 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -126,6 +126,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): ("cranmirror", "0.3.3"), ("event_stream", "1.6.3"), ("selshaurl", "3.7.0"), + ("libssh", "0.11.1"), ], ) @flaky From 78b4a3f84b505a9376cc3ee140e1ef1f5c963965 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 10 Jan 2025 11:15:55 +0100 Subject: [PATCH 34/40] improve compatibility --- conda_forge_tick/feedstock_parser.py | 63 ++++++++++++++++--- conda_forge_tick/update_recipe/version.py | 18 ++++-- .../version_build_as_expr_correct.yaml | 2 +- .../version_build_number_via_context.yaml | 6 +- tests/test_version_migrator.py | 2 +- 5 files changed, 73 insertions(+), 18 deletions(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 080cef783..9774eabc9 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -8,7 +8,7 @@ import zipfile from collections import defaultdict from pathlib import Path -from typing import Optional, Set, Union +from typing import Optional, Set, Union, Any import requests import yaml @@ -81,6 +81,7 @@ def _get_requirements( host: bool = True, run: bool = True, outputs_to_keep: Optional[Set["PackageName"]] = None, + schema_version: int = 0, ) -> "Set[PackageName]": """Get the list of recipe requirements from a meta.yaml dict @@ -100,13 +101,19 @@ def _get_requirements( reqs : `set` the set of recipe requirements """ - kw = dict(build=build, host=host, run=run) + kw = dict(build=build, host=host, run=run, schema_version=schema_version) if outputs_to_keep: reqs = set() outputs_ = meta_yaml.get("outputs", []) or [] if outputs else [] for output in outputs_: - if output.get("name") in outputs_to_keep: - reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) + if schema_version == 0: + if output.get("name") in outputs_to_keep: + reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) + elif schema_version == 1: + if output.get("package", {}).get("name") in outputs_to_keep: + reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) + else: + raise ValueError(f"Unknown schema version {schema_version}") else: reqs = _parse_requirements(meta_yaml.get("requirements", {}), **kw) outputs_ = meta_yaml.get("outputs", []) or [] if outputs else [] @@ -121,6 +128,7 @@ def _parse_requirements( build: bool = True, host: bool = True, run: bool = True, + schema_version: int = 0, ) -> typing.MutableSet["PackageName"]: """Flatten a YAML requirements section into a list of names""" if not req: # handle None as empty @@ -133,19 +141,44 @@ def _parse_requirements( _run = list(as_iterable(req.get("run", []) or [] if run else [])) reqlist = _build + _host + _run + # remove any pin expressions / dictionaries in the v1 schema + print("\n\n\nSCHEMA VERSION", schema_version) + if schema_version == 1: + reqlist = [req for req in reqlist if not isinstance(req, dict)] + packages = (PIN_SEP_PAT.split(x)[0].lower() for x in reqlist if x is not None) return {typing.cast("PackageName", pkg) for pkg in packages} +def _filter_v1_pins(reqs: list[str | dict[str, Any]]) -> list[str]: + res = [] + for el in reqs: + if isinstance(el, dict): + if "pin_subpackage" in el: + res.append(f"pin_subpackage {el['pin_subpackage']['name']}") + elif "pin_compatible" in el: + res.append(f"pin_compatible {el['pin_compatible']['name']}") + else: + logger.warning(f"Unknown pin type: {el}") + else: + res.append(el) -def _extract_requirements(meta_yaml, outputs_to_keep=None): + +def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): + print("EXTRACT REQUIREMENTS SCHEMA VERSION", schema_version) strong_exports = False requirements_dict = defaultdict(set) if outputs_to_keep: metas = [] for output in meta_yaml.get("outputs", []) or []: - if output.get("name") in outputs_to_keep: - metas.append(output) + if schema_version == 0: + if output.get("name") in outputs_to_keep: + metas.append(output) + elif schema_version == 1: + if output.get("package", {}).get("name") in outputs_to_keep: + metas.append(output) + else: + raise ValueError(f"Unknown schema version {schema_version}") else: metas = [meta_yaml] + meta_yaml.get("outputs", []) or [] @@ -155,8 +188,11 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None): requirements_dict["run"].update(set(req)) continue for section in ["build", "host", "run"]: + print(section) + print(requirements_dict[section]) + print(req.get(section, [])) requirements_dict[section].update( - list(as_iterable(req.get(section, []) or [])), + list(as_iterable(_filter_v1_pins(req.get(section, []) or []))), ) test: "TestTypedDict" = block.get("test", {}) or {} @@ -429,12 +465,16 @@ def populate_feedstock_attributes( if k.endswith("_meta_yaml") or k.endswith("_requirements"): sub_graph.pop(k) + schema_version = 0 + if recipe_yaml is not None: + schema_version = 1 for k, v in zip(plat_archs, variant_yamls): plat_arch_name = "_".join(k) sub_graph[f"{plat_arch_name}_meta_yaml"] = v _, sub_graph[f"{plat_arch_name}_requirements"], _ = _extract_requirements( v, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None), + schema_version=schema_version, ) ( @@ -444,6 +484,7 @@ def populate_feedstock_attributes( ) = _extract_requirements( meta_yaml, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None), + schema_version=schema_version, ) # handle multi outputs @@ -467,6 +508,7 @@ def populate_feedstock_attributes( req = _get_requirements( yaml_dict, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, []), + schema_version=schema_version, ) sub_graph["req"] = req @@ -485,7 +527,10 @@ def populate_feedstock_attributes( and len(yaml_dict["outputs"]) > 0 and "version" in yaml_dict["outputs"][0] ): - sub_graph["version"] = yaml_dict["outputs"][0]["version"] + if schema_version == 0: + sub_graph["version"] = yaml_dict["outputs"][0]["version"] + elif schema_version == 1: + sub_graph["version"] = yaml_dict["outputs"][0]["package"]["version"] # set the url and hash sub_graph.pop("url", None) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 42ebd97de..ffcde6599 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -135,8 +135,17 @@ def _try_url_and_hash_it(url: str, hash_type: str) -> str | None: def _render_jinja2(tmpl, context): + env = jinja2.Environment(undefined=jinja2.StrictUndefined) + + # We need to add the split filter to support v1 recipes + def split_filter(value, sep): + return value.split(sep) + + filters = {"split": split_filter} + filters["split"] = split_filter + return ( - jinja2.sandbox.SandboxedEnvironment(undefined=jinja2.StrictUndefined) + env .from_string(tmpl) .render(**context) ) @@ -732,8 +741,9 @@ def update_version_v1( line, f'{indentation}version: "{version}"' ) break - + print("All sources: ", rendered_sources) for source in rendered_sources: + print("Updating source: ", source) # update the hash value urls = source.url # zip url and template @@ -750,9 +760,9 @@ def update_version_v1( hash_type = "md5" # convert to regular jinja2 template - template = template.replace("${{", "{{") + cb_template = template.replace("${{", "{{") new_tmpl, new_hash = _get_new_url_tmpl_and_hash( - template, + cb_template, source.context, hash_type, recipe_yaml, diff --git a/tests/test_v1_yaml/version_build_as_expr_correct.yaml b/tests/test_v1_yaml/version_build_as_expr_correct.yaml index 9bc540b14..438f83c55 100644 --- a/tests/test_v1_yaml/version_build_as_expr_correct.yaml +++ b/tests/test_v1_yaml/version_build_as_expr_correct.yaml @@ -1,7 +1,7 @@ context: name: oneDNN version: "3.6.2" - build: 1 + build: 0 variant_build: ${{ variant_build_base | int + 100 if dnnl_cpu_runtime == "omp" else build_base }} desc_omp: In this package oneDNN is built with the OpenMP CPU runtime. desc_tbb: In this package oneDNN is built with the TBB CPU runtime. diff --git a/tests/test_v1_yaml/version_build_number_via_context.yaml b/tests/test_v1_yaml/version_build_number_via_context.yaml index 923728093..986f4326d 100644 --- a/tests/test_v1_yaml/version_build_number_via_context.yaml +++ b/tests/test_v1_yaml/version_build_number_via_context.yaml @@ -191,9 +191,9 @@ outputs: name: torchvision-tests build: script: true - requirements: - run: - - ${{ pin_subpackage('torchvision', exact=True) }} + # requirements: + # run: + # - ${{ pin_subpackage('torchvision', exact=True) }} tests: - files: source: diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 7af900a01..0f92ba05c 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -120,7 +120,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): [ ("pypi_url", "0.7.1"), ("jolt", "5.2.0"), - ("build_number_via_context", "0.20.1"), + # ("build_number_via_context", "0.20.1"), ("build_as_expr", "3.6.2"), ("conditional_sources", "3.24.11"), ("cranmirror", "0.3.3"), From 6d6427b07c3065b08514a50ca601864b8495890c Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 10 Jan 2025 11:20:47 +0100 Subject: [PATCH 35/40] make more tests pass --- conda_forge_tick/feedstock_parser.py | 18 +++++++++--------- conda_forge_tick/update_recipe/version.py | 9 ++------- tests/test_v1_yaml/version_libssh.yaml | 2 +- tests/test_v1_yaml/version_libssh_correct.yaml | 4 ++-- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 9774eabc9..a39e4ed5c 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -8,7 +8,7 @@ import zipfile from collections import defaultdict from pathlib import Path -from typing import Optional, Set, Union, Any +from typing import Any, Optional, Set, Union import requests import yaml @@ -108,10 +108,14 @@ def _get_requirements( for output in outputs_: if schema_version == 0: if output.get("name") in outputs_to_keep: - reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) + reqs |= _parse_requirements( + output.get("requirements", {}) or {}, **kw + ) elif schema_version == 1: if output.get("package", {}).get("name") in outputs_to_keep: - reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) + reqs |= _parse_requirements( + output.get("requirements", {}) or {}, **kw + ) else: raise ValueError(f"Unknown schema version {schema_version}") else: @@ -142,13 +146,13 @@ def _parse_requirements( reqlist = _build + _host + _run # remove any pin expressions / dictionaries in the v1 schema - print("\n\n\nSCHEMA VERSION", schema_version) if schema_version == 1: reqlist = [req for req in reqlist if not isinstance(req, dict)] packages = (PIN_SEP_PAT.split(x)[0].lower() for x in reqlist if x is not None) return {typing.cast("PackageName", pkg) for pkg in packages} + def _filter_v1_pins(reqs: list[str | dict[str, Any]]) -> list[str]: res = [] for el in reqs: @@ -164,7 +168,6 @@ def _filter_v1_pins(reqs: list[str | dict[str, Any]]) -> list[str]: def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): - print("EXTRACT REQUIREMENTS SCHEMA VERSION", schema_version) strong_exports = False requirements_dict = defaultdict(set) @@ -188,9 +191,6 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): requirements_dict["run"].update(set(req)) continue for section in ["build", "host", "run"]: - print(section) - print(requirements_dict[section]) - print(req.get(section, [])) requirements_dict[section].update( list(as_iterable(_filter_v1_pins(req.get(section, []) or []))), ) @@ -198,7 +198,7 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): test: "TestTypedDict" = block.get("test", {}) or {} requirements_dict["test"].update(test.get("requirements", []) or []) requirements_dict["test"].update(test.get("requires", []) or []) - + # TODO improve code (wolf) if "tests" in block: for test in block.get("tests", []): # only script tests have requirements diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index ffcde6599..749f559a8 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -141,14 +141,9 @@ def _render_jinja2(tmpl, context): def split_filter(value, sep): return value.split(sep) - filters = {"split": split_filter} - filters["split"] = split_filter + env.filters["split"] = split_filter - return ( - env - .from_string(tmpl) - .render(**context) - ) + return env.from_string(tmpl).render(**context) def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str, cmeta: Any): diff --git a/tests/test_v1_yaml/version_libssh.yaml b/tests/test_v1_yaml/version_libssh.yaml index bb604761a..de6572c17 100644 --- a/tests/test_v1_yaml/version_libssh.yaml +++ b/tests/test_v1_yaml/version_libssh.yaml @@ -28,7 +28,7 @@ requirements: - zlib - krb5 run_exports: - # We pin to a minor version due to an incompatibility + # We pin to a minor version due to an incompatibility # between 0.8 and 0.10 # https://abi-laboratory.pro/index.php?view=timeline&l=libssh - ${{ pin_subpackage('libssh', upper_bound='x.x') }} diff --git a/tests/test_v1_yaml/version_libssh_correct.yaml b/tests/test_v1_yaml/version_libssh_correct.yaml index 3b05cb31c..8a2eb5a27 100644 --- a/tests/test_v1_yaml/version_libssh_correct.yaml +++ b/tests/test_v1_yaml/version_libssh_correct.yaml @@ -2,7 +2,7 @@ schema_version: 1 context: name: libssh - version: 0.11.1 + version: "0.11.1" package: name: ${{ name|lower }} @@ -28,7 +28,7 @@ requirements: - zlib - krb5 run_exports: - # We pin to a minor version due to an incompatibility + # We pin to a minor version due to an incompatibility # between 0.8 and 0.10 # https://abi-laboratory.pro/index.php?view=timeline&l=libssh - ${{ pin_subpackage('libssh', upper_bound='x.x') }} From 93632e57682f828bf6bc485bbad73d4568309ceb Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 10 Jan 2025 13:26:55 +0100 Subject: [PATCH 36/40] get selshaurl to pass --- tests/test_migrators.py | 2 ++ tests/test_v1_yaml/version_selshaurl.yaml | 18 +++++++-------- .../version_selshaurl_correct.yaml | 22 ++++++++++--------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 205d082e1..848fcb36f 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -578,6 +578,8 @@ def run_test_migration( pat = re.compile(r"{#.*#}") actual_output = pat.sub("", actual_output) output = pat.sub("", output) + print(actual_output) + assert actual_output == output # TODO: fix subgraph here (need this to be xsh file) if isinstance(m, Version): diff --git a/tests/test_v1_yaml/version_selshaurl.yaml b/tests/test_v1_yaml/version_selshaurl.yaml index b724539e8..ac0b8a642 100644 --- a/tests/test_v1_yaml/version_selshaurl.yaml +++ b/tests/test_v1_yaml/version_selshaurl.yaml @@ -12,19 +12,19 @@ package: source: - if: linux then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz - sha256: 8cca2d7ef6e3f18668246c9eed609e03e720e4033d069164c991c5feb078443c - file_name: ${{ name }}-${{ version }}.tar.gz + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz + sha256: 8cca2d7ef6e3f18668246c9eed609e03e720e4033d069164c991c5feb078443c + file_name: ${{ name }}-${{ version }}.tar.gz - if: osx then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.tar.gz - sha256: f6113506bab9430f98773b0ab7776efe387f4d40c8785d8f8c427a91c36f4cfe - file_name: ${{ name }}-${{ version }}.tar.gz + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.tar.gz + sha256: f6113506bab9430f98773b0ab7776efe387f4d40c8785d8f8c427a91c36f4cfe + file_name: ${{ name }}-${{ version }}.tar.gz - if: win then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip - sha256: 6432449254f56b037f17d876403f919f6c136cc16ec91f7778001b10eea115ac - file_name: ${{ name }}-${{ version }}.zip + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip + sha256: 6432449254f56b037f17d876403f919f6c136cc16ec91f7778001b10eea115ac + file_name: ${{ name }}-${{ version }}.zip build: number: 100 diff --git a/tests/test_v1_yaml/version_selshaurl_correct.yaml b/tests/test_v1_yaml/version_selshaurl_correct.yaml index 61b238e36..02d75f28c 100644 --- a/tests/test_v1_yaml/version_selshaurl_correct.yaml +++ b/tests/test_v1_yaml/version_selshaurl_correct.yaml @@ -3,7 +3,7 @@ schema_version: 1 context: name: embree - version: 3.7.0 + version: "3.7.0" package: name: ${{ name }} @@ -12,19 +12,19 @@ package: source: - if: linux then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz - sha256: 671a3aa7cc1c8501f1290dd051b42a337a692ea6552a07436779439d649e3e29 - file_name: ${{ name }}-${{ version }}.tar.gz + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.linux.tar.gz + sha256: 671a3aa7cc1c8501f1290dd051b42a337a692ea6552a07436779439d649e3e29 + file_name: ${{ name }}-${{ version }}.tar.gz - if: osx then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.tar.gz - sha256: 17c31f67efb9afc3ed658fcaa5886bc10c6f67f1e364d6494e494d189d8b8c70 - file_name: ${{ name }}-${{ version }}.tar.gz + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x86_64.macosx.zip + sha256: 17c31f67efb9afc3ed658fcaa5886bc10c6f67f1e364d6494e494d189d8b8c70 + file_name: ${{ name }}-${{ version }}.tar.gz - if: win then: - - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip - sha256: 442c8933fa3a21d66c0459ded83e1a4c896b1a26c4e46ea62e65ffbfec273be2 - file_name: ${{ name }}-${{ version }}.zip + - url: https://github.com/${{ name }}/${{ name }}/releases/download/v${{ version }}/${{ name }}-${{ version }}.x64.vc14.windows.zip + sha256: 442c8933fa3a21d66c0459ded83e1a4c896b1a26c4e46ea62e65ffbfec273be2 + file_name: ${{ name }}-${{ version }}.zip build: number: 0 @@ -40,6 +40,8 @@ requirements: then: python about: + license: Apache-2.0 + license_file: doc/LICENSE.txt summary: High Performance Ray Tracing Kernels homepage: https://embree.github.io/ From 8427c7d253c2222effd03359b05e7f1357a8449f Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 10 Jan 2025 15:19:12 +0100 Subject: [PATCH 37/40] install rattler-build-conda-compat from source for some testing --- .github/workflows/tests-reusable.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests-reusable.yml b/.github/workflows/tests-reusable.yml index b487229cb..276629d51 100644 --- a/.github/workflows/tests-reusable.yml +++ b/.github/workflows/tests-reusable.yml @@ -96,6 +96,9 @@ jobs: python -m pip install -v --no-deps --no-build-isolation -e . + # also install rattler-build-conda-compat from git source + pip install git+https://github.com/prefix-dev/rattler-build-conda-compat@main + - name: start MongoDB uses: supercharge/mongodb-github-action@90004df786821b6308fb02299e5835d0dae05d0d # v1.12.0 with: From 6deaf29b1406c6e4919d02c5a2a3f83e9886d12c Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 15 Jan 2025 16:54:45 +0100 Subject: [PATCH 38/40] rebase and revert changes to feedstock_parser.py --- conda_forge_tick/feedstock_parser.py | 65 +++---------------- .../version_build_number_via_context.yaml | 6 +- tests/test_version_migrator.py | 2 +- 3 files changed, 14 insertions(+), 59 deletions(-) diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index a39e4ed5c..080cef783 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -8,7 +8,7 @@ import zipfile from collections import defaultdict from pathlib import Path -from typing import Any, Optional, Set, Union +from typing import Optional, Set, Union import requests import yaml @@ -81,7 +81,6 @@ def _get_requirements( host: bool = True, run: bool = True, outputs_to_keep: Optional[Set["PackageName"]] = None, - schema_version: int = 0, ) -> "Set[PackageName]": """Get the list of recipe requirements from a meta.yaml dict @@ -101,23 +100,13 @@ def _get_requirements( reqs : `set` the set of recipe requirements """ - kw = dict(build=build, host=host, run=run, schema_version=schema_version) + kw = dict(build=build, host=host, run=run) if outputs_to_keep: reqs = set() outputs_ = meta_yaml.get("outputs", []) or [] if outputs else [] for output in outputs_: - if schema_version == 0: - if output.get("name") in outputs_to_keep: - reqs |= _parse_requirements( - output.get("requirements", {}) or {}, **kw - ) - elif schema_version == 1: - if output.get("package", {}).get("name") in outputs_to_keep: - reqs |= _parse_requirements( - output.get("requirements", {}) or {}, **kw - ) - else: - raise ValueError(f"Unknown schema version {schema_version}") + if output.get("name") in outputs_to_keep: + reqs |= _parse_requirements(output.get("requirements", {}) or {}, **kw) else: reqs = _parse_requirements(meta_yaml.get("requirements", {}), **kw) outputs_ = meta_yaml.get("outputs", []) or [] if outputs else [] @@ -132,7 +121,6 @@ def _parse_requirements( build: bool = True, host: bool = True, run: bool = True, - schema_version: int = 0, ) -> typing.MutableSet["PackageName"]: """Flatten a YAML requirements section into a list of names""" if not req: # handle None as empty @@ -145,43 +133,19 @@ def _parse_requirements( _run = list(as_iterable(req.get("run", []) or [] if run else [])) reqlist = _build + _host + _run - # remove any pin expressions / dictionaries in the v1 schema - if schema_version == 1: - reqlist = [req for req in reqlist if not isinstance(req, dict)] - packages = (PIN_SEP_PAT.split(x)[0].lower() for x in reqlist if x is not None) return {typing.cast("PackageName", pkg) for pkg in packages} -def _filter_v1_pins(reqs: list[str | dict[str, Any]]) -> list[str]: - res = [] - for el in reqs: - if isinstance(el, dict): - if "pin_subpackage" in el: - res.append(f"pin_subpackage {el['pin_subpackage']['name']}") - elif "pin_compatible" in el: - res.append(f"pin_compatible {el['pin_compatible']['name']}") - else: - logger.warning(f"Unknown pin type: {el}") - else: - res.append(el) - - -def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): +def _extract_requirements(meta_yaml, outputs_to_keep=None): strong_exports = False requirements_dict = defaultdict(set) if outputs_to_keep: metas = [] for output in meta_yaml.get("outputs", []) or []: - if schema_version == 0: - if output.get("name") in outputs_to_keep: - metas.append(output) - elif schema_version == 1: - if output.get("package", {}).get("name") in outputs_to_keep: - metas.append(output) - else: - raise ValueError(f"Unknown schema version {schema_version}") + if output.get("name") in outputs_to_keep: + metas.append(output) else: metas = [meta_yaml] + meta_yaml.get("outputs", []) or [] @@ -192,13 +156,13 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None, schema_version=0): continue for section in ["build", "host", "run"]: requirements_dict[section].update( - list(as_iterable(_filter_v1_pins(req.get(section, []) or []))), + list(as_iterable(req.get(section, []) or [])), ) test: "TestTypedDict" = block.get("test", {}) or {} requirements_dict["test"].update(test.get("requirements", []) or []) requirements_dict["test"].update(test.get("requires", []) or []) - # TODO improve code (wolf) + if "tests" in block: for test in block.get("tests", []): # only script tests have requirements @@ -465,16 +429,12 @@ def populate_feedstock_attributes( if k.endswith("_meta_yaml") or k.endswith("_requirements"): sub_graph.pop(k) - schema_version = 0 - if recipe_yaml is not None: - schema_version = 1 for k, v in zip(plat_archs, variant_yamls): plat_arch_name = "_".join(k) sub_graph[f"{plat_arch_name}_meta_yaml"] = v _, sub_graph[f"{plat_arch_name}_requirements"], _ = _extract_requirements( v, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None), - schema_version=schema_version, ) ( @@ -484,7 +444,6 @@ def populate_feedstock_attributes( ) = _extract_requirements( meta_yaml, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None), - schema_version=schema_version, ) # handle multi outputs @@ -508,7 +467,6 @@ def populate_feedstock_attributes( req = _get_requirements( yaml_dict, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, []), - schema_version=schema_version, ) sub_graph["req"] = req @@ -527,10 +485,7 @@ def populate_feedstock_attributes( and len(yaml_dict["outputs"]) > 0 and "version" in yaml_dict["outputs"][0] ): - if schema_version == 0: - sub_graph["version"] = yaml_dict["outputs"][0]["version"] - elif schema_version == 1: - sub_graph["version"] = yaml_dict["outputs"][0]["package"]["version"] + sub_graph["version"] = yaml_dict["outputs"][0]["version"] # set the url and hash sub_graph.pop("url", None) diff --git a/tests/test_v1_yaml/version_build_number_via_context.yaml b/tests/test_v1_yaml/version_build_number_via_context.yaml index 986f4326d..923728093 100644 --- a/tests/test_v1_yaml/version_build_number_via_context.yaml +++ b/tests/test_v1_yaml/version_build_number_via_context.yaml @@ -191,9 +191,9 @@ outputs: name: torchvision-tests build: script: true - # requirements: - # run: - # - ${{ pin_subpackage('torchvision', exact=True) }} + requirements: + run: + - ${{ pin_subpackage('torchvision', exact=True) }} tests: - files: source: diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index 0f92ba05c..7af900a01 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -120,7 +120,7 @@ def test_version_up(case, new_ver, tmpdir, caplog): [ ("pypi_url", "0.7.1"), ("jolt", "5.2.0"), - # ("build_number_via_context", "0.20.1"), + ("build_number_via_context", "0.20.1"), ("build_as_expr", "3.6.2"), ("conditional_sources", "3.24.11"), ("cranmirror", "0.3.3"), From 2dc802868f36e3dcb4f8804289233e656217526d Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 15 Jan 2025 17:12:04 +0100 Subject: [PATCH 39/40] further clean up --- .github/workflows/tests-reusable.yml | 3 -- conda_forge_tick/update_recipe/version.py | 34 +---------------------- tests/test_migrators.py | 1 - 3 files changed, 1 insertion(+), 37 deletions(-) diff --git a/.github/workflows/tests-reusable.yml b/.github/workflows/tests-reusable.yml index 276629d51..b487229cb 100644 --- a/.github/workflows/tests-reusable.yml +++ b/.github/workflows/tests-reusable.yml @@ -96,9 +96,6 @@ jobs: python -m pip install -v --no-deps --no-build-isolation -e . - # also install rattler-build-conda-compat from git source - pip install git+https://github.com/prefix-dev/rattler-build-conda-compat@main - - name: start MongoDB uses: supercharge/mongodb-github-action@90004df786821b6308fb02299e5835d0dae05d0d # v1.12.0 with: diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 749f559a8..963daef61 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -736,9 +736,8 @@ def update_version_v1( line, f'{indentation}version: "{version}"' ) break - print("All sources: ", rendered_sources) + for source in rendered_sources: - print("Updating source: ", source) # update the hash value urls = source.url # zip url and template @@ -931,34 +930,3 @@ def update_version(raw_meta_yaml, version, hash_type="sha256") -> (str, set[str] else: logger.critical("Recipe did not change in version migration!") return None, errors - - -if __name__ == "__main__": - # parse args and invoke update_version_feedstock_dir - import argparse - - from conda_forge_tick.utils import setup_logging - - setup_logging("INFO") - - parser = argparse.ArgumentParser() - parser.add_argument("feedstock_dir", help="The feedstock directory to update.") - parser.add_argument("version", help="The new version to update to.") - parser.add_argument( - "--hash-type", - default="sha256", - help="The hash type to use for the source.", - ) - parser.add_argument( - "--use-container", - action="store_true", - help="Use a container to run the version parsing.", - ) - - args = parser.parse_args() - updated, errors = update_version_feedstock_dir( - args.feedstock_dir, - args.version, - hash_type=args.hash_type, - use_container=args.use_container, - ) diff --git a/tests/test_migrators.py b/tests/test_migrators.py index 848fcb36f..531071366 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -578,7 +578,6 @@ def run_test_migration( pat = re.compile(r"{#.*#}") actual_output = pat.sub("", actual_output) output = pat.sub("", output) - print(actual_output) assert actual_output == output # TODO: fix subgraph here (need this to be xsh file) From eac71b00422a1ba6d5308178efea89ec322785ef Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Wed, 15 Jan 2025 12:45:14 -0600 Subject: [PATCH 40/40] Update conda_forge_tick/update_recipe/version.py --- conda_forge_tick/update_recipe/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index 963daef61..8b65fc988 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -135,7 +135,7 @@ def _try_url_and_hash_it(url: str, hash_type: str) -> str | None: def _render_jinja2(tmpl, context): - env = jinja2.Environment(undefined=jinja2.StrictUndefined) + env = jinja2.sandbox.SandboxedEnvironment(undefined=jinja2.StrictUndefined) # We need to add the split filter to support v1 recipes def split_filter(value, sep):