Skip to content

Commit

Permalink
Merge branch 'main' into submodulev3
Browse files Browse the repository at this point in the history
  • Loading branch information
@adam2392
adam2392 committed Jan 18, 2024
2 parents b838238 + 95b2c9d commit a6a62e0
Show file tree
Hide file tree
Showing 154 changed files with 4,637 additions and 2,435 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/update-lock-files.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@ jobs:
matrix:
include:
- name: main
update_script_args: "--skip-build 'scipy-dev|^pymin_conda_forge$|pypy'"
update_script_args: "--select-tag main-ci"
additional_commit_message: "[doc build]"
- name: scipy-dev
update_script_args: "--select-build scipy_dev"
update_script_args: "--select-tag scipy-dev"
additional_commit_message: "[scipy-dev]"
- name: cirrus-arm
update_script_args: "--select-build '^pymin_conda_forge$'"
update_script_args: "--select-tag arm"
additional_commit_message: "[cirrus arm]"
- name: pypy
update_script_args: "--select-build pypy"
update_script_args: "--select-tag pypy"
additional_commit_message: "[pypy]"

steps:
Expand Down
173 changes: 117 additions & 56 deletions build_tools/update_environments_and_lock_files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import json
import logging
import re
import shlex
import subprocess
import sys
from importlib.metadata import version
Expand Down Expand Up @@ -88,9 +87,11 @@ def remove_from(alist, to_remove):
return [each for each in alist if each not in to_remove]


conda_build_metadata_list = [
build_metadata_list = [
{
"build_name": "pylatest_conda_forge_mkl_linux-64",
"name": "pylatest_conda_forge_mkl_linux-64",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "conda-forge",
Expand All @@ -108,7 +109,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pylatest_conda_forge_mkl_osx-64",
"name": "pylatest_conda_forge_mkl_osx-64",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "osx-64",
"channel": "conda-forge",
Expand All @@ -122,7 +125,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pylatest_conda_mkl_no_openmp",
"name": "pylatest_conda_mkl_no_openmp",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "osx-64",
"channel": "defaults",
Expand All @@ -136,7 +141,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pymin_conda_defaults_openblas",
"name": "pymin_conda_defaults_openblas",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "defaults",
Expand All @@ -152,7 +159,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pymin_conda_forge_openblas_ubuntu_2204",
"name": "pymin_conda_forge_openblas_ubuntu_2204",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "conda-forge",
Expand All @@ -167,7 +176,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pylatest_pip_openblas_pandas",
"name": "pylatest_pip_openblas_pandas",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "defaults",
Expand All @@ -182,7 +193,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pylatest_pip_scipy_dev",
"name": "pylatest_pip_scipy_dev",
"type": "conda",
"tag": "scipy-dev",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "defaults",
Expand Down Expand Up @@ -219,7 +232,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pypy3",
"name": "pypy3",
"type": "conda",
"tag": "pypy",
"folder": "build_tools/azure",
"platform": "linux-64",
"channel": "conda-forge",
Expand All @@ -236,7 +251,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pymin_conda_forge_mkl",
"name": "pymin_conda_forge_mkl",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/azure",
"platform": "win-64",
"channel": "conda-forge",
Expand All @@ -250,7 +267,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "doc_min_dependencies",
"name": "doc_min_dependencies",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/circle",
"platform": "linux-64",
"channel": "conda-forge",
Expand Down Expand Up @@ -288,7 +307,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "doc",
"name": "doc",
"type": "conda",
"tag": "main-ci",
"folder": "build_tools/circle",
"platform": "linux-64",
"channel": "conda-forge",
Expand All @@ -313,7 +334,9 @@ def remove_from(alist, to_remove):
},
},
{
"build_name": "pymin_conda_forge",
"name": "pymin_conda_forge",
"type": "conda",
"tag": "arm",
"folder": "build_tools/cirrus",
"platform": "linux-aarch64",
"channel": "conda-forge",
Expand All @@ -324,12 +347,10 @@ def remove_from(alist, to_remove):
"python": "3.9",
},
},
]


pip_build_metadata_list = [
{
"build_name": "debian_atlas_32bit",
"name": "debian_atlas_32bit",
"type": "pip",
"tag": "main-ci",
"folder": "build_tools/azure",
"pip_dependencies": [
"cython",
Expand All @@ -350,7 +371,9 @@ def remove_from(alist, to_remove):
"python_version": "3.9.2",
},
{
"build_name": "ubuntu_atlas",
"name": "ubuntu_atlas",
"type": "pip",
"tag": "main-ci",
"folder": "build_tools/azure",
"pip_dependencies": [
"cython",
Expand Down Expand Up @@ -444,7 +467,7 @@ def get_conda_environment_content(build_metadata):

def write_conda_environment(build_metadata):
content = get_conda_environment_content(build_metadata)
build_name = build_metadata["build_name"]
build_name = build_metadata["name"]
folder_path = Path(build_metadata["folder"])
output_path = folder_path / f"{build_name}_environment.yml"
logger.debug(output_path)
Expand All @@ -457,15 +480,25 @@ def write_all_conda_environments(build_metadata_list):


def conda_lock(environment_path, lock_file_path, platform):
command = (
f"conda-lock lock --mamba --kind explicit --platform {platform} "
f"--file {environment_path} --filename-template {lock_file_path}"
execute_command(
[
"conda-lock",
"lock",
"--mamba",
"--kind",
"explicit",
"--platform",
platform,
"--file",
str(environment_path),
"--filename-template",
str(lock_file_path),
]
)
execute_command(shlex.split(command))


def create_conda_lock_file(build_metadata):
build_name = build_metadata["build_name"]
build_name = build_metadata["name"]
folder_path = Path(build_metadata["folder"])
environment_path = folder_path / f"{build_name}_environment.yml"
platform = build_metadata["platform"]
Expand All @@ -479,7 +512,7 @@ def create_conda_lock_file(build_metadata):

def write_all_conda_lock_files(build_metadata_list):
for build_metadata in build_metadata_list:
logger.info(f"# Locking dependencies for {build_metadata['build_name']}")
logger.info(f"# Locking dependencies for {build_metadata['name']}")
create_conda_lock_file(build_metadata)


Expand All @@ -495,7 +528,7 @@ def get_pip_requirements_content(build_metadata):


def write_pip_requirements(build_metadata):
build_name = build_metadata["build_name"]
build_name = build_metadata["name"]
content = get_pip_requirements_content(build_metadata)
folder_path = Path(build_metadata["folder"])
output_path = folder_path / f"{build_name}_requirements.txt"
Expand All @@ -509,26 +542,41 @@ def write_all_pip_requirements(build_metadata_list):


def pip_compile(pip_compile_path, requirements_path, lock_file_path):
command = f"{pip_compile_path} --upgrade {requirements_path} -o {lock_file_path}"
execute_command(shlex.split(command))
execute_command(
[
str(pip_compile_path),
"--upgrade",
str(requirements_path),
"-o",
str(lock_file_path),
]
)


def write_pip_lock_file(build_metadata):
build_name = build_metadata["build_name"]
build_name = build_metadata["name"]
python_version = build_metadata["python_version"]
environment_name = f"pip-tools-python{python_version}"
# To make sure that the Python used to create the pip lock file is the same
# as the one used during the CI build where the lock file is used, we first
# create a conda environment with the correct Python version and
# pip-compile and run pip-compile in this environment

command = (
"conda create -c conda-forge -n"
f" pip-tools-python{python_version} python={python_version} pip-tools -y"
execute_command(
[
"conda",
"create",
"-c",
"conda-forge",
"-n",
f"pip-tools-python{python_version}",
f"python={python_version}",
"pip-tools",
"-y",
]
)
execute_command(shlex.split(command))

json_output = execute_command(shlex.split("conda info --json"))
json_output = execute_command(["conda", "info", "--json"])
conda_info = json.loads(json_output)
environment_folder = [
each for each in conda_info["envs"] if each.endswith(environment_name)
Expand All @@ -544,7 +592,7 @@ def write_pip_lock_file(build_metadata):

def write_all_pip_lock_files(build_metadata_list):
for build_metadata in build_metadata_list:
logger.info(f"# Locking dependencies for {build_metadata['build_name']}")
logger.info(f"# Locking dependencies for {build_metadata['name']}")
write_pip_lock_file(build_metadata)


Expand Down Expand Up @@ -583,7 +631,7 @@ def check_conda_version():
"--select-build",
default="",
help=(
"Regex to restrict the builds we want to update environment and lock files. By"
"Regex to filter the builds we want to update environment and lock files. By"
" default all the builds are selected."
),
)
Expand All @@ -592,6 +640,14 @@ def check_conda_version():
default=None,
help="Regex to skip some builds from the builds selected by --select-build",
)
@click.option(
"--select-tag",
default=None,
help=(
"Tag to filter the builds, e.g. 'main-ci' or 'scipy-dev'. "
"This is an additional filtering on top of --select-build."
),
)
@click.option(
"-v",
"--verbose",
Expand All @@ -604,7 +660,7 @@ def check_conda_version():
is_flag=True,
help="Print output of commands executed by the script",
)
def main(verbose, very_verbose, select_build, skip_build):
def main(select_build, skip_build, select_tag, verbose, very_verbose):
if verbose:
logger.setLevel(logging.DEBUG)
if very_verbose:
Expand All @@ -613,36 +669,41 @@ def main(verbose, very_verbose, select_build, skip_build):
check_conda_lock_version()
check_conda_version()

filtered_conda_build_metadata_list = [
each
for each in conda_build_metadata_list
if re.search(select_build, each["build_name"])
filtered_build_metadata_list = [
each for each in build_metadata_list if re.search(select_build, each["name"])
]
if select_tag is not None:
filtered_build_metadata_list = [
each for each in build_metadata_list if each["tag"] == select_tag
]
if skip_build is not None:
filtered_conda_build_metadata_list = [
filtered_build_metadata_list = [
each
for each in filtered_conda_build_metadata_list
if not re.search(skip_build, each["build_name"])
for each in filtered_build_metadata_list
if not re.search(skip_build, each["name"])
]

selected_build_info = "\n".join(
f" - {each['name']}, type: {each['type']}, tag: {each['tag']}"
for each in filtered_build_metadata_list
)
selected_build_message = (
f"# {len(filtered_build_metadata_list)} selected builds\n{selected_build_info}"
)
logger.info(selected_build_message)

filtered_conda_build_metadata_list = [
each for each in filtered_build_metadata_list if each["type"] == "conda"
]
if filtered_conda_build_metadata_list:
logger.info("# Writing conda environments")
write_all_conda_environments(filtered_conda_build_metadata_list)
logger.info("# Writing conda lock files")
write_all_conda_lock_files(filtered_conda_build_metadata_list)

filtered_pip_build_metadata_list = [
each
for each in pip_build_metadata_list
if re.search(select_build, each["build_name"])
each for each in filtered_build_metadata_list if each["type"] == "pip"
]
if skip_build is not None:
filtered_pip_build_metadata_list = [
each
for each in filtered_pip_build_metadata_list
if not re.search(skip_build, each["build_name"])
]

if filtered_pip_build_metadata_list:
logger.info("# Writing pip requirements")
write_all_pip_requirements(filtered_pip_build_metadata_list)
Expand Down
Loading

0 comments on commit a6a62e0

Please sign in to comment.