Skip to content

Commit

Permalink
BLD Add Meson support (scikit-learn#28040)
Browse files Browse the repository at this point in the history
Co-authored-by: Adrin Jalali <adrin.jalali@gmail.com>
Co-authored-by: Olivier Grisel <olivier.grisel@ensta.org>
Co-authored-by: Guillaume Lemaitre <g.lemaitre58@gmail.com>
Co-authored-by: Julien Jerphanion <git@jjerphan.xyz>
  • Loading branch information
5 people authored Jan 23, 2024
1 parent 07007b3 commit 56625c9
Show file tree
Hide file tree
Showing 34 changed files with 1,100 additions and 3 deletions.
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,14 @@ in: inplace # just a shortcut
inplace:
$(PYTHON) setup.py build_ext -i

dev-meson:
# Temporary script to try the experimental meson build. Once meson is
# accepted as the default build tool, this will go away.
python build_tools/build-meson-editable-install.py

clean-meson:
pip uninstall -y scikit-learn

test-code: in
$(PYTEST) --showlocals -v sklearn --durations=20
test-sphinxext:
Expand Down
1 change: 1 addition & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ jobs:
DISTRIB: 'conda'
LOCK_FILE: './build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock'
COVERAGE: 'true'
BUILD_WITH_MESON: 'true'
SKLEARN_TESTS_GLOBAL_RANDOM_SEED: '42' # default global random seed

# Check compilation with Ubuntu 22.04 LTS (Jammy Jellyfish) and scipy from conda-forge
Expand Down
4 changes: 3 additions & 1 deletion build_tools/azure/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,10 @@ scikit_learn_install() {
export LDFLAGS="$LDFLAGS -Wl,--sysroot=/"
fi

if [[ "$BUILD_WITH_MESON" == "true" ]]; then
make dev-meson
# TODO use a specific variable for this rather than using a particular build ...
if [[ "$DISTRIB" == "conda-pip-latest" ]]; then
elif [[ "$DISTRIB" == "conda-pip-latest" ]]; then
# Check that pip can automatically build scikit-learn with the build
# dependencies specified in pyproject.toml using an isolated build
# environment:
Expand Down
10 changes: 8 additions & 2 deletions build_tools/azure/pylatest_conda_forge_mkl_linux-64_conda.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generated by conda-lock.
# platform: linux-64
# input_hash: 0e751f4212c4e51710aad471314a8b385a5e12fe3536c2a766f949da61eabb88
# input_hash: 0cee038efd0cc93a79f66b1cdbbc359ac52521c98df956b3e4042575e89711f5
@EXPLICIT
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2023.11.17-hbcca054_0.conda#01ffc8d36f9eba0ce0b3c1955fa780ee
Expand All @@ -20,7 +20,7 @@ https://conda.anaconda.org/conda-forge/linux-64/alsa-lib-1.2.10-hd590300_0.conda
https://conda.anaconda.org/conda-forge/linux-64/attr-2.5.1-h166bdaf_1.tar.bz2#d9c69a24ad678ffce24c6543a0176b00
https://conda.anaconda.org/conda-forge/linux-64/aws-c-common-0.9.0-hd590300_0.conda#71b89db63b5b504e7afc8ad901172e1e
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda#69b8b6202a07720f448be700e300ccf4
https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.24.0-hd590300_0.conda#f5842b88e9cbfa177abfaeacd457a45d
https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.25.0-hd590300_0.conda#89e40af02dd3a0846c0c1131c5126706
https://conda.anaconda.org/conda-forge/linux-64/gettext-0.21.1-h27087fc_0.tar.bz2#14947d8770185e5153fdd04d4673ed37
https://conda.anaconda.org/conda-forge/linux-64/gflags-2.2.2-he1b5a44_1004.tar.bz2#cddaf2c63ea4a5901cf09524c490ecdc
https://conda.anaconda.org/conda-forge/linux-64/graphite2-1.3.13-h58526e2_1001.tar.bz2#8c54672728e8ec6aa6db90cf2806d220
Expand Down Expand Up @@ -50,6 +50,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda#
https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.9.4-hcb278e6_0.conda#318b08df404f9c9be5712aaa5a6f0bb0
https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.3-h59595ed_0.conda#bdadff838d5437aea83607ced8b37f75
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4-h59595ed_2.conda#7dbaa197d7ba6032caf7ae7f32c1efa0
https://conda.anaconda.org/conda-forge/linux-64/ninja-1.11.1-h924138e_0.conda#73a4953a2d9c115bdc10ff30a52f675f
https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda#da0ec11a6454ae19bff5b02ed881a2b1
https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.0-hd590300_1.conda#603827b39ea2b835268adb8c821b8570
https://conda.anaconda.org/conda-forge/linux-64/pixman-0.43.0-h59595ed_0.conda#6b4b43013628634b6cfdee6b74fd696b
Expand Down Expand Up @@ -159,6 +160,7 @@ https://conda.anaconda.org/conda-forge/noarch/toml-0.10.2-pyhd8ed1ab_0.tar.bz2#f
https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2#5844808ffab9ebdb694585b50ba02a96
https://conda.anaconda.org/conda-forge/linux-64/tornado-6.3.3-py311h459d7ec_1.conda#a700fcb5cedd3e72d0c75d095c7a6eda
https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.9.0-pyha770c72_0.conda#a92a6440c3fe7052d63244f3aba2a4a7
https://conda.anaconda.org/conda-forge/noarch/wheel-0.42.0-pyhd8ed1ab_0.conda#1cdea58981c5cbc17b51973bcaddcea7
https://conda.anaconda.org/conda-forge/linux-64/xcb-util-image-0.4.0-h8ee46fc_1.conda#9d7bcddf49cbf727730af10e71022c73
https://conda.anaconda.org/conda-forge/linux-64/xkeyboard-config-2.40-hd590300_0.conda#07c15d846a2e4d673da22cbd85fdb6d2
https://conda.anaconda.org/conda-forge/linux-64/xorg-libxext-1.3.4-h0b41bf4_2.conda#82b6df12252e6f32402b96dacc656fec
Expand All @@ -173,9 +175,12 @@ https://conda.anaconda.org/conda-forge/noarch/joblib-1.3.2-pyhd8ed1ab_0.conda#4d
https://conda.anaconda.org/conda-forge/linux-64/libclang-15.0.7-default_hb11cfb5_4.conda#c90f4cbb57839c98fef8f830e4b9972f
https://conda.anaconda.org/conda-forge/linux-64/libgoogle-cloud-2.12.0-hac9eb74_1.conda#0dee716254497604762957076ac76540
https://conda.anaconda.org/conda-forge/linux-64/libxkbcommon-1.6.0-h5d7e998_0.conda#d8edd0e29db6fb6b6988e1a28d35d994
https://conda.anaconda.org/conda-forge/noarch/meson-1.3.1-pyhd8ed1ab_0.conda#54744574be599bff37ee4c3624ed02d2
https://conda.anaconda.org/conda-forge/linux-64/mkl-2022.2.1-h84fe81f_16997.conda#a7ce56d5757f5b57e7daabe703ade5bb
https://conda.anaconda.org/conda-forge/linux-64/pillow-10.2.0-py311ha6c5da5_0.conda#a5ccd7f2271f28b7d2de0b02b64e3796
https://conda.anaconda.org/conda-forge/noarch/pip-23.3.2-pyhd8ed1ab_0.conda#8591c748f98dcc02253003533bc2e4b1
https://conda.anaconda.org/conda-forge/linux-64/pulseaudio-client-16.1-hb77b528_5.conda#ac902ff3c1c6d750dd0dfc93a974ab74
https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.7.1-pyhd8ed1ab_0.conda#dcb27826ffc94d5f04e241322239983b
https://conda.anaconda.org/conda-forge/noarch/pytest-7.4.4-pyhd8ed1ab_0.conda#a9d145de8c5f064b5fa68fb34725d9f4
https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.8.2-pyhd8ed1ab_0.tar.bz2#dd999d1cc9f79e67dbb855c8924c7984
https://conda.anaconda.org/conda-forge/linux-64/sip-6.7.12-py311hb755f60_0.conda#02336abab4cb5dd794010ef53c54bd09
Expand All @@ -184,6 +189,7 @@ https://conda.anaconda.org/conda-forge/linux-64/blas-1.0-mkl.tar.bz2#349aef876b1
https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.22.8-h98fc4e7_1.conda#1b52a89485ab573a5bb83a5225ff706e
https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-8.3.0-h3d44ed6_0.conda#5a6f6c00ef982a9bc83558d9ac8f64a0
https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-16_linux64_mkl.tar.bz2#85f61af03fd291dae33150ffe89dc09a
https://conda.anaconda.org/conda-forge/noarch/meson-python-0.15.0-pyh0c530f3_0.conda#3bc64565ca78ce3bb80248d09926d8f9
https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.12.2-py311hb755f60_5.conda#e4d262cc3600e70b505a6761d29f6207
https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.1.0-pyhd8ed1ab_0.conda#06eb685a3a0b146347a58dda979485da
https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.5.0-pyhd8ed1ab_0.conda#d5f595da2daead898ca958ac62f0307b
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ dependencies:
- pytest-cov
- coverage
- ccache
- meson-python
- pip
- pytorch=1.13
- pytorch-cpu
- polars
Expand Down
45 changes: 45 additions & 0 deletions build_tools/build-meson-editable-install.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import re
import shlex
import subprocess
from pathlib import Path


def main():
pyproject_path = Path("pyproject.toml")

if not pyproject_path.exists():
raise SystemExit(
"Can not find pyproject.toml. You should run this script from the"
" scikit-learn root folder."
)

old_pyproject_content = pyproject_path.read_text(encoding="utf-8")
if 'build-backend = "mesonpy"' not in old_pyproject_content:
new_pyproject_content = re.sub(
r"\[build-system\]",
r'[build-system]\nbuild-backend = "mesonpy"',
old_pyproject_content,
)
pyproject_path.write_text(new_pyproject_content, encoding="utf-8")

command = shlex.split(
"pip install --editable . --verbose --no-build-isolation "
"--config-settings editable-verbose=true"
)

exception = None
try:
subprocess.check_call(command)
except Exception as e:
exception = e
finally:
pyproject_path.write_text(old_pyproject_content, encoding="utf-8")

if exception is not None:
raise RuntimeError(
"There was some error when running the script"
) from exception


if __name__ == "__main__":
main()
2 changes: 2 additions & 0 deletions build_tools/update_environments_and_lock_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ def remove_from(alist, to_remove):
"channel": "conda-forge",
"conda_dependencies": common_dependencies + [
"ccache",
"meson-python",
"pip",
"pytorch",
"pytorch-cpu",
"polars",
Expand Down
82 changes: 82 additions & 0 deletions doc/developers/advanced_installation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,88 @@ It is however preferred to use pip.
On Unix-like systems, you can equivalently type ``make in`` from the top-level
folder. Have a look at the ``Makefile`` for additional utilities.

.. _building_with_meson:

Building with Meson
-------------------

Support for Meson is experimental, in scikit-learn 1.5.0.dev0.
`Open an issue <https://github.com/scikit-learn/scikit-learn/issues/new>`__ if
you encounter any problems!

Make sure you have `meson-python` and `ninja` installed, either with `conda`:

.. code-block:: bash
conda install -c conda-forge meson-python ninja -y
or with pip:

.. code-block:: bash
pip install meson-python ninja
Simplest way to build with Meson
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

To build scikit-learn, the simplest way is to run:

.. code-block:: bash
make dev-meson
You need to do it once after this you can run your code that imports `sklearn`
and it will recompile as needed.

In case you want to go back to using setuptools:

.. code-block:: bash
make clean-meson
More advanced way to build with Meson
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

If you can not use `make`, want to do it yourself or understand what goes in
behind the scenes, you can edit `pyproject.toml` and make sure `build-backend`
is set to `"mesonpy"`

.. code-block:: toml
[build-system]
build-backend = "mesonpy"
Build with the following `pip` command:

.. code-block:: bash
pip install --editable . \
--verbose --no-build-isolation \
--config-settings editable-verbose=true
If you want to go back to using `setuptools`:

.. code-block:: bash
pip uninstall -y scikit-learn
Note `--config-settings editable-verbose=true` is advised to avoid surprises.
meson-python implements editable install by recompiling when doing `import
sklearn`. Even changing python files involves copying files to the Meson build
directory. You will see the meson output when that happens, rather than
potentially waiting a while and wondering what is taking so long. Bonus: that
means you only have to do the `pip install` once, after that your code will
recompile when doing `import sklearn`.

Other places that may be worth looking at:

- `pandas setup doc
<https://pandas.pydata.org/docs/development/contributing_environment.html#step-3-build-and-install-pandas>`_:
pandas has a similar setup as ours (no spin or dev.py)
- `scipy Meson doc
<https://scipy.github.io/devdocs/building/understanding_meson.html>`_ gives
more background about how Meson works behind the scenes

.. _platform_specific_instructions:

Platform-specific instructions
Expand Down
10 changes: 10 additions & 0 deletions doc/whats_new/v1.5.rst
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,16 @@ Version 1.5.0

.. include:: changelog_legend.inc

Support for building with Meson
-------------------------------

Meson is now supported as a build backend, see :ref:`Building with Meson
<building_with_meson>` for more details.

:pr:`28040` by :user:`Loïc Estève <lesteve>`

TODO Fill more details before the 1.5 release, when the Meson story has settled down.

Changelog
---------

Expand Down
53 changes: 53 additions & 0 deletions meson.build
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
project(
'scikit-learn',
'c', 'cpp', 'cython',
version: run_command('sklearn/_build_utils/version.py', check: true).stdout().strip(),
license: 'BSD-3',
meson_version: '>= 1.1.0',
default_options: [
'buildtype=debugoptimized',
'c_std=c99',
'cpp_std=c++14',
],
)

cc = meson.get_compiler('c')
cpp = meson.get_compiler('cpp')

# Check compiler is recent enough (see "Toolchain Roadmap" for details)
if cc.get_id() == 'gcc'
if not cc.version().version_compare('>=8.0')
error('scikit-learn requires GCC >= 8.0')
endif
elif cc.get_id() == 'msvc'
if not cc.version().version_compare('>=19.20')
error('scikit-learn requires at least vc142 (default with Visual Studio 2019) ' + \
'when building with MSVC')
endif
endif

_global_c_args = cc.get_supported_arguments(
'-Wno-unused-but-set-variable',
'-Wno-unused-function',
'-Wno-conversion',
'-Wno-misleading-indentation',
)
add_project_arguments(_global_c_args, language : 'c')

# We need -lm for all C code (assuming it uses math functions, which is safe to
# assume for scikit-learn). For C++ it isn't needed, because libstdc++/libc++ is
# guaranteed to depend on it.
m_dep = cc.find_library('m', required : false)
if m_dep.found()
add_project_link_arguments('-lm', language : 'c')
endif

tempita = files('sklearn/_build_utils/tempita.py')

py = import('python').find_installation(pure: false)

# Copy all the .py files to the install dir, rather than using
# py.install_sources and needing to list them explicitely one by one
install_subdir('sklearn', install_dir: py.get_install_dir())

subdir('sklearn')
7 changes: 7 additions & 0 deletions sklearn/__check_build/meson.build
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
py.extension_module(
'_check_build',
'_check_build.pyx',
cython_args: cython_args,
install: true,
subdir: 'sklearn/__check_build',
)
8 changes: 8 additions & 0 deletions sklearn/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,14 @@
"show_versions",
]

_BUILT_WITH_MESON = False
try:
import sklearn._built_with_meson # noqa: F401

_BUILT_WITH_MESON = True
except ModuleNotFoundError:
pass


def setup_module(module):
"""Fixture for the tests to assure globally controllable seeding of RNGs"""
Expand Down
57 changes: 57 additions & 0 deletions sklearn/_build_utils/tempita.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import argparse
import os

from Cython import Tempita as tempita

# XXX: If this import ever fails (does it really?), vendor either
# cython.tempita or numpy/npy_tempita.


def process_tempita(fromfile, outfile=None):
"""Process tempita templated file and write out the result.
The template file is expected to end in `.c.tp` or `.pyx.tp`:
E.g. processing `template.c.in` generates `template.c`.
"""
with open(fromfile, "r", encoding="utf-8") as f:
template_content = f.read()

template = tempita.Template(template_content)
content = template.substitute()

with open(outfile, "w", encoding="utf-8") as f:
f.write(content)


def main():
parser = argparse.ArgumentParser()
parser.add_argument("infile", type=str, help="Path to the input file")
parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory")
parser.add_argument(
"-i",
"--ignore",
type=str,
help=(
"An ignored input - may be useful to add a "
"dependency between custom targets"
),
)
args = parser.parse_args()

if not args.infile.endswith(".tp"):
raise ValueError(f"Unexpected extension: {args.infile}")

if not args.outdir:
raise ValueError("Missing `--outdir` argument to tempita.py")

outdir_abs = os.path.join(os.getcwd(), args.outdir)
outfile = os.path.join(
outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0]
)

process_tempita(args.infile, outfile)


if __name__ == "__main__":
main()
Loading

0 comments on commit 56625c9

Please sign in to comment.