diff --git a/README.rst b/README.rst index ece3a0772d1..457665de976 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,13 @@ For running the client: $ sudo pip install -r conans/requirements.txt + +In OSX you should also install: + +:: + + $ sudo pip install -r conans/requirements_osx.txt + Server: :: @@ -85,16 +92,19 @@ Server: $ sudo apt-get install python-dev $ sudo pip install -r conans/requirements_server.txt -Development: +Development (for running the tests): :: $ sudo pip install -r conans/requirements_dev.txt + +If you are in Windows, using ``sudo`` is not required. + Running the tests ~~~~~~~~~~~~~~~~~~ -Make sure that the Python requirements have been installed. +Make sure that the Python requirements for testing have been installed, as explained above. Before you can run the tests, you need to set a few environment variables first. @@ -108,7 +118,7 @@ On Windows it would be (while being in the conan root directory): :: - $ export PYTHONPATH=. + $ set PYTHONPATH=. Ensure that your ``cmake`` has version 2.8 or later. You can see the version with the following command: @@ -163,6 +173,16 @@ A few minutes later it should print ``OK``: OK +To run specific tests, you can specify the test name too, something like: + +:: + + $ nosetests conans.test.integration.flat_requirements_test --nocapture + + +The ``--nocapture`` argument can be useful to see some output that otherwise is captured by nosetests. + + Create a launcher ~~~~~~~~~~~~~~~~~ diff --git a/conans/__init__.py b/conans/__init__.py index 7818586d786..bd8392e0b44 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -15,4 +15,5 @@ COMPLEX_SEARCH_CAPABILITY = "complex_search" SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.23.2' +__version__ = '0.24.0' + diff --git a/conans/client/command.py b/conans/client/command.py index cdb7ce7dd1d..20f619e16ba 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1,40 +1,19 @@ import argparse -import hashlib import inspect import os -import requests import sys -import conans -from conans import __version__ as CLIENT_VERSION, tools -from conans.client.client_cache import ClientCache -from conans.client.conf import MIN_SERVER_COMPATIBLE_VERSION, ConanClientConfigParser -from conans.client.manager import ConanManager -from conans.client.migrations import ClientMigrator -from conans.client.output import ConanOutput, Color -from conans.client.printer import Printer -from conans.client.remote_manager import RemoteManager -from conans.client.remote_registry import RemoteRegistry -from conans.client.rest.auth_manager import ConanApiAuthManager -from conans.client.rest.rest_client import RestApiClient -from conans.client.rest.version_checker import VersionCheckerRequester -from conans.client.runner import ConanRunner -from conans.client.store.localdb import LocalDB -from conans.client.userio import UserIO +from conans import __version__ as CLIENT_VERSION +from conans.client.conan_api import (Conan, default_manifest_folder) +from conans.client.conan_command_output import CommandOutputer +from conans.client.output import Color + from conans.errors import ConanException -from conans.model.ref import ConanFileReference, is_a_reference -from conans.model.version import Version -from conans.paths import CONANFILE, conan_expand_user -from conans.search.search import DiskSearchManager, DiskSearchAdapter +from conans.model.ref import ConanFileReference +from conans.paths import CONANFILE from conans.util.config_parser import get_bool_from_text -from conans.util.env_reader import get_env -from conans.util.files import rmdir, save_files, exception_message_safe -from conans.util.log import logger, configure_logger -from conans.util.tracer import log_command, log_exception -from conans.model.profile import Profile -from conans.client.command_profile_args import profile_from_args -from conans.client.new import get_files -from conans.client.loader import load_consumer_conanfile +from conans.util.log import logger +from conans.util.files import exception_message_safe class Extender(argparse.Action): @@ -69,17 +48,11 @@ class Command(object): collaborators. It can also show help of the tool """ - def __init__(self, client_cache, user_io, runner, remote_manager, search_manager): - assert isinstance(user_io, UserIO) - assert isinstance(client_cache, ClientCache) + def __init__(self, conan_api, client_cache, user_io): + assert isinstance(conan_api, Conan) + self._conan = conan_api self._client_cache = client_cache self._user_io = user_io - self._runner = runner - self._manager = ConanManager(client_cache, user_io, runner, remote_manager, search_manager) - - @property - def client_cache(self): - return self._client_cache def new(self, *args): """Creates a new package recipe template with a 'conanfile.py'. @@ -102,17 +75,30 @@ def new(self, *args): parser.add_argument("-b", "--bare", action='store_true', default=False, help='Create the minimum package recipe, without build() or package()' 'methods. Useful in combination with "package_files" command') + parser.add_argument("-cis", "--ci_shared", action='store_true', default=False, + help='Package will have a "shared" option to be used in CI') + parser.add_argument("-cilg", "--ci_travis_gcc", action='store_true', default=False, + help='Generate travis-ci files for linux gcc') + parser.add_argument("-cilc", "--ci_travis_clang", action='store_true', default=False, + help='Generate travis-ci files for linux clang') + parser.add_argument("-cio", "--ci_travis_osx", action='store_true', default=False, + help='Generate travis-ci files for OSX apple-clang') + parser.add_argument("-ciw", "--ci_appveyor_win", action='store_true', default=False, + help='Generate appveyor files for Appveyor Visual Studio') + parser.add_argument("-gi", "--gitignore", action='store_true', default=False, + help='Generate a .gitignore with the known patterns to excluded') + parser.add_argument("-ciu", "--ci_upload_url", + help='Define URL of the repository to upload') args = parser.parse_args(*args) - log_command("new", vars(args)) - - root_folder = os.getcwd() - files = get_files(args.name, header=args.header, pure_c=args.pure_c, test=args.test, - exports_sources=args.sources, bare=args.bare) - - save_files(root_folder, files) - for f in sorted(files): - self._user_io.out.success("File saved: %s" % f) + self._conan.new(args.name, header=args.header, pure_c=args.pure_c, test=args.test, + exports_sources=args.sources, bare=args.bare, + visual_versions=args.ci_appveyor_win, + linux_gcc_versions=args.ci_travis_gcc, + linux_clang_versions=args.ci_travis_clang, + gitignore=args.gitignore, + osx_clang_versions=args.ci_travis_osx, shared=args.ci_shared, + upload_url=args.ci_upload_url) def test_package(self, *args): """ Export, build package and test it with a consumer project. @@ -120,6 +106,7 @@ def test_package(self, *args): located in a subfolder, named 'test_package` by default. It must 'require' the package under testing. """ + parser = argparse.ArgumentParser(description=self.test_package.__doc__, prog="conan test_package") parser.add_argument("path", nargs='?', default="", help='path to conanfile file, ' @@ -136,86 +123,20 @@ def test_package(self, *args): _add_common_install_arguments(parser, build_help=_help_build_policies) args = parser.parse_args(*args) - log_command("test_package", vars(args)) - - current_path = os.getcwd() - root_folder = os.path.normpath(os.path.join(current_path, args.path)) - if args.test_folder: - test_folder_name = args.test_folder - test_folder = os.path.join(root_folder, test_folder_name) - test_conanfile = os.path.join(test_folder, "conanfile.py") - if not os.path.exists(test_conanfile): - raise ConanException("test folder '%s' not available, " - "or it doesn't have a conanfile.py" % args.test_folder) - else: - for name in ["test_package", "test"]: - test_folder_name = name - test_folder = os.path.join(root_folder, test_folder_name) - test_conanfile = os.path.join(test_folder, "conanfile.py") - if os.path.exists(test_conanfile): - break - else: - raise ConanException("test folder 'test_package' not available, " - "or it doesn't have a conanfile.py") - - options = args.options or [] - settings = args.settings or [] - - sha = hashlib.sha1("".join(options + settings).encode()).hexdigest() - build_folder = os.path.join(test_folder, "build", sha) - rmdir(build_folder) - # shutil.copytree(test_folder, build_folder) - profile = profile_from_args(args, current_path, self._client_cache.profiles_path) - conanfile = load_consumer_conanfile(test_conanfile, "", - self._client_cache.settings, self._runner, - self._user_io.out) - try: - # convert to list from ItemViews required for python3 - if hasattr(conanfile, "requirements"): - conanfile.requirements() - reqs = list(conanfile.requires.items()) - first_dep = reqs[0][1].conan_reference - except Exception: - raise ConanException("Unable to retrieve first requirement of test conanfile.py") - - # Forcing an export! - if not args.not_export: - self._user_io.out.info("Exporting package recipe") - user_channel = "%s/%s" % (first_dep.user, first_dep.channel) - self._manager.export(user_channel, root_folder, keep_source=args.keep_source) - - lib_to_test = first_dep.name + "*" - # Get False or a list of patterns to check - if args.build is None and lib_to_test: # Not specified, force build the tested library - args.build = [lib_to_test] - - manifests = _parse_manifests_arguments(args, root_folder, current_path) - manifest_folder, manifest_interactive, manifest_verify = manifests - self._manager.install(reference=test_folder, - current_path=build_folder, - manifest_folder=manifest_folder, - manifest_verify=manifest_verify, - manifest_interactive=manifest_interactive, - remote=args.remote, - profile=profile, - build_modes=args.build, - update=args.update, - generators=["txt"] - ) - - test_conanfile = os.path.join(test_folder, CONANFILE) - self._manager.build(test_conanfile, test_folder, build_folder, test=True) + return self._conan.test_package(args.path, args.profile, args.settings, args.options, + args.env, args.scope, args.test_folder, args.not_export, + args.build, args.keep_source, args.verify, args.manifests, + args.manifests_interactive, args.remote, args.update) # Alias to test def test(self, *args): - """ (deprecated). Alias to test_package, use it instead - """ - self.test_package(*args) + """ (deprecated). Alias to test_package, use it instead """ + return self.test_package(*args) def package_files(self, *args): """Creates a package binary from given precompiled artifacts in user folder, skipping - the package recipe build() and package() methods + the package recipe build() and package() methods """ parser = argparse.ArgumentParser(description=self.package_files.__doc__, prog="conan package_files") parser.add_argument("reference", @@ -234,27 +155,18 @@ def package_files(self, *args): action='store_true', help='Overwrite existing package if existing') args = parser.parse_args(*args) - args.env = None - args.scope = None - log_command("package_files", vars(args)) - reference = ConanFileReference.loads(args.reference) - current_path = os.getcwd() - package_folder = args.package_folder or current_path - if not os.path.isabs(package_folder): - package_folder = os.path.join(current_path, package_folder) - - profile = profile_from_args(args, current_path, self._client_cache.profiles_path) - self._manager.package_files(reference=reference, package_folder=package_folder, - profile=profile, force=args.force) + return self._conan.package_files(reference=args.reference, package_folder=args.package_folder, + profile_name=args.profile, force=args.force, settings=args.settings, + options=args.options) def install(self, *args): """Installs the requirements specified in a 'conanfile.py' or 'conanfile.txt'. - It can also be used to install a concrete recipe/package specified by the reference parameter. - If the recipe is not found in the local cache it will retrieve the recipe from a remote, - looking for it sequentially in the available configured remotes. - When the recipe has been downloaded it will try to download a binary package matching - the specified settings, only from the remote from which the recipe was retrieved. - If no binary package is found you can build the package from sources using the '--build' option. + It can also be used to install a concrete recipe/package specified by the reference parameter. + If the recipe is not found in the local cache it will retrieve the recipe from a remote, + looking for it sequentially in the available configured remotes. + When the recipe has been downloaded it will try to download a binary package matching + the specified settings, only from the remote from which the recipe was retrieved. + If no binary package is found you can build the package from sources using the '--build' option. """ parser = argparse.ArgumentParser(description=self.install.__doc__, prog="conan install") parser.add_argument("reference", nargs='?', default="", @@ -278,38 +190,14 @@ def install(self, *args): _add_common_install_arguments(parser, build_help=_help_build_policies) args = parser.parse_args(*args) - log_command("install", vars(args)) - self._user_io.out.werror_active = args.werror - current_path = os.getcwd() - try: - reference = ConanFileReference.loads(args.reference) - except: - reference = os.path.normpath(os.path.join(current_path, args.reference)) - - if args.all or args.package: # Install packages without settings (fixed ids or all) - if args.all: - args.package = [] - if not args.reference or not isinstance(reference, ConanFileReference): - raise ConanException("Invalid package recipe reference. " - "e.g., MyPackage/1.2@user/channel") - self._manager.download(reference, args.package, remote=args.remote) - else: # Classic install, package chosen with settings and options - manifests = _parse_manifests_arguments(args, reference, current_path) - manifest_folder, manifest_interactive, manifest_verify = manifests - profile = profile_from_args(args, current_path, self._client_cache.profiles_path) - self._manager.install(reference=reference, - current_path=current_path, - remote=args.remote, - profile=profile, - build_modes=args.build, - filename=args.file, - update=args.update, - manifest_folder=manifest_folder, - manifest_verify=manifest_verify, - manifest_interactive=manifest_interactive, - generators=args.generator, - no_imports=args.no_imports) + return self._conan.install(reference=args.reference, package=args.package, settings=args.settings, + options=args.options, + env=args.env, scope=args.scope, all=args.all, remote=args.remote, werror=args.werror, + verify=args.verify, manifests=args.manifests, + manifests_interactive=args.manifests_interactive, + build=args.build, profile_name=args.profile, update=args.update, + generator=args.generator, no_imports=args.no_imports, filename=args.file) def config(self, *args): """Manages conan.conf information @@ -327,17 +215,16 @@ def config(self, *args): args = parser.parse_args(*args) - config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) if args.subcommand == "set": try: key, value = args.item.split("=", 1) except: - raise ConanException("Please specify key=value") - config_parser.set_item(key.strip(), value.strip()) + self._raise_exception_printing("Please specify key=value") + return self._conan.config_set(key, value) elif args.subcommand == "get": - self._user_io.out.info(config_parser.get_item(args.item)) + return self._conan.config_get(args.item) elif args.subcommand == "rm": - config_parser.rm_item(args.item) + return self._conan.config_rm(args.item) def info(self, *args): """Prints information about a package recipe's dependency graph. @@ -368,45 +255,64 @@ def info(self, *args): parser.add_argument("--build_order", "-bo", help='given a modified reference, return an ordered list to build (CI)', nargs=1, action=Extender) + parser.add_argument("--json", "-j", nargs='?', const="1", type=str, + help='Only with --build_order option, return the information in a json. e.j' + ' --json=/path/to/filename.json or --json to output the json') parser.add_argument("--graph", "-g", help='Creates file with project dependencies graph. It will generate ' 'a DOT or HTML file depending on the filename extension') + parser.add_argument("--cwd", "-c", help='Use this directory as the current directory') build_help = 'given a build policy (same install command "build" parameter), return an ordered list of ' \ 'packages that would be built from sources in install command (simulation)' _add_common_install_arguments(parser, build_help=build_help) args = parser.parse_args(*args) - log_command("info", vars(args)) - current_path = os.getcwd() - try: - reference = ConanFileReference.loads(args.reference) - except: - reference = os.path.normpath(os.path.join(current_path, args.reference)) - - if args.only == ["None"]: - args.only = [] - - if args.only and args.paths and (set(args.only) - set(path_only_options)): - raise ConanException("Invalid --only value '%s' with --path specified, allowed values: [%s]." - "" % (args.only, str_path_only_options)) - elif args.only and not args.paths and (set(args.only) - set(info_only_options)): - raise ConanException("Invalid --only value '%s', allowed values: [%s].\n" - "Use --only=None to show only the references." % (args.only, str_only_options)) - - profile = profile_from_args(args, current_path, self._client_cache.profiles_path) - self._manager.info(reference=reference, - current_path=current_path, - remote=args.remote, - profile=profile, - info=args.only, - package_filter=args.package_filter, - check_updates=args.update, - filename=args.file, - build_order=args.build_order, - build_modes=args.build, - graph_filename=args.graph, - show_paths=args.paths) + outputer = CommandOutputer(self._user_io, self._client_cache) + # BUILD ORDER ONLY + if args.build_order: + ret = self._conan.info_build_order(args.reference, settings=args.settings, options=args.options, + env=args.env, scope=args.scope, profile_name=args.profile, + filename=args.file, remote=args.remote, build_order=args.build_order, + check_updates=args.update, cwd=args.cwd) + if args.json: + json_arg = True if args.json == "1" else args.json + outputer.json_build_order(ret, json_arg, args.cwd) + else: + outputer.build_order(ret) + + # INSTALL SIMULATION, NODES TO INSTALL + elif args.build is not None: + nodes, _ = self._conan.info_nodes_to_build(args.reference, build_modes=args.build, + settings=args.settings, + options=args.options, env=args.env, scope=args.scope, + profile_name=args.profile, filename=args.file, remote=args.remote, + check_updates=args.update, cwd=args.cwd) + outputer.nodes_to_build(nodes) + # INFO ABOUT DEPS OF CURRENT PROJECT OR REFERENCE + else: + data = self._conan.info_get_graph(args.reference, remote=args.remote, settings=args.settings, + options=args.options, env=args.env, scope=args.scope, + profile_name=args.profile, update=args.update, + filename=args.file, cwd=args.cwd) + deps_graph, graph_updates_info, project_reference = data + only = args.only + if args.only == ["None"]: + only = [] + if only and args.paths and (set(only) - set(path_only_options)): + self._raise_exception_printing("Invalid --only value '%s' with --path specified, allowed values: [%s]." + % (only, str_path_only_options)) + elif only and not args.paths and (set(only) - set(info_only_options)): + self._raise_exception_printing("Invalid --only value '%s', allowed values: [%s].\n" + "Use --only=None to show only the references." % + (only, str_only_options)) + + if args.graph: + outputer.info_graph(args.graph, deps_graph, project_reference, args.cwd) + else: + outputer.info(deps_graph, graph_updates_info, only, args.remote, args.package_filter, args.paths, + project_reference) + return def build(self, *args): """ Utility command to run your current project 'conanfile.py' build() method. @@ -422,22 +328,7 @@ def build(self, *args): parser.add_argument("--file", "-f", help="specify conanfile filename") parser.add_argument("--source_folder", "-sf", help="local folder containing the sources") args = parser.parse_args(*args) - log_command("build", vars(args)) - current_path = os.getcwd() - if args.path: - root_path = os.path.abspath(args.path) - else: - root_path = current_path - - build_folder = current_path - source_folder = args.source_folder or root_path - if not os.path.isabs(source_folder): - source_folder = os.path.normpath(os.path.join(current_path, source_folder)) - - if args.file and args.file.endswith(".txt"): - raise ConanException("A conanfile.py is needed to call 'conan build'") - conanfile_path = os.path.join(root_path, args.file or CONANFILE) - self._manager.build(conanfile_path, source_folder, build_folder) + return self._conan.build(path=args.path, source_folder=args.source_folder, filename=args.file) def package(self, *args): """ Calls your conanfile.py 'package' method for a specific package recipe. @@ -471,24 +362,8 @@ def package(self, *args): parser.add_argument("--source_folder", "-sf", help="local folder containing the sources") args = parser.parse_args(*args) - log_command("package", vars(args)) - - current_path = os.getcwd() - try: - reference = ConanFileReference.loads(args.reference) - self._manager.package(reference, args.package) - except: - if "@" in args.reference: - raise - recipe_folder = args.reference - if not os.path.isabs(recipe_folder): - recipe_folder = os.path.normpath(os.path.join(current_path, recipe_folder)) - build_folder = args.build_folder or current_path - if not os.path.isabs(build_folder): - build_folder = os.path.normpath(os.path.join(current_path, build_folder)) - package_folder = current_path - source_folder = args.source_folder or recipe_folder - self._manager.local_package(package_folder, recipe_folder, build_folder, source_folder) + return self._conan.package(reference=args.reference, package=args.package, build_folder=args.build_folder, + source_folder=args.source_folder) def source(self, *args): """ Calls your conanfile.py 'source()' method to configure the source directory. @@ -507,10 +382,7 @@ def source(self, *args): " do nothing.") args = parser.parse_args(*args) - log_command("source", vars(args)) - - current_path, reference = _get_reference(args) - self._manager.source(current_path, reference, args.force) + return self._conan.source(args.reference, args.force) def imports(self, *args): """ Execute the 'imports' stage of a conanfile.txt or a conanfile.py. @@ -531,18 +403,7 @@ def imports(self, *args): help="Undo imports. Remove imported files") args = parser.parse_args(*args) - log_command("imports", vars(args)) - - if args.undo: - if not os.path.isabs(args.reference): - current_path = os.path.normpath(os.path.join(os.getcwd(), args.reference)) - else: - current_path = args.reference - self._manager.imports_undo(current_path) - else: - dest_folder = args.dest - current_path, reference = _get_reference(args) - self._manager.imports(current_path, reference, args.file, dest_folder) + return self._conan.imports(args.reference, args.undo, args.dest, args.file) def export(self, *args): """ Copies the package recipe (conanfile.py and associated files) to your local cache. @@ -560,11 +421,7 @@ def export(self, *args): 'Use for testing purposes only') parser.add_argument("--file", "-f", help="specify conanfile filename") args = parser.parse_args(*args) - log_command("export", vars(args)) - - current_path = os.path.abspath(args.path or os.getcwd()) - keep_source = args.keep_source - self._manager.export(args.user, current_path, keep_source, args.file) + return self._conan.export(user=args.user, path=args.path, keep_source=args.keep_source, filename=args.file) def remove(self, *args): """Remove any package recipe or binary matching a pattern. @@ -592,18 +449,16 @@ def remove(self, *args): 'reference: MyPackage/1.2' '@user/channel') args = parser.parse_args(*args) - log_command("remove", vars(args)) - - reference = _check_query_parameter_and_get_reference(args) + reference = self._check_query_parameter_and_get_reference(args.pattern, args.query) if args.packages is not None and args.query: - raise ConanException("'-q' and '-p' parameters can't be used at the same time") + self._raise_exception_printing("'-q' and '-p' parameters can't be used at the same time") if args.builds is not None and args.query: - raise ConanException("'-q' and '-b' parameters can't be used at the same time") + self._raise_exception_printing("'-q' and '-b' parameters can't be used at the same time") - self._manager.remove(reference or args.pattern, package_ids_filter=args.packages, build_ids=args.builds, - src=args.src, force=args.force, remote=args.remote, packages_query=args.query) + return self._conan.remove(pattern=reference or args.pattern, query=args.query, packages=args.packages, builds=args.builds, + src=args.src, force=args.force, remote=args.remote) def copy(self, *args): """ Copy conan recipes and packages to another user/channel. @@ -626,15 +481,8 @@ def copy(self, *args): default=False, help='Override destination packages and the package recipe') args = parser.parse_args(*args) - log_command("copy", vars(args)) - - reference = ConanFileReference.loads(args.reference) - new_ref = ConanFileReference.loads("%s/%s@%s" % (reference.name, - reference.version, - args.user_channel)) - if args.all: - args.package = [] - self._manager.copy(reference, args.package, new_ref.user, new_ref.channel, args.force) + return self._conan.copy(reference=args.reference, user_channel=args.user_channel, force=args.force, + all=args.all, package=args.package) def user(self, *parameters): """ Update your cached user name (and auth token) to avoid it being requested later. @@ -652,14 +500,7 @@ def user(self, *parameters): parser.add_argument('-c', '--clean', default=False, action='store_true', help='Remove user and tokens for all remotes') args = parser.parse_args(*parameters) # To enable -h - log_command("user", vars(args)) - - if args.clean: - localdb = LocalDB(self._client_cache.localdb) - localdb.init(clean=True) - self._user_io.out.success("Deleted user data") - return - self._manager.user(args.remote, args.name, args.password) + return self._conan.user(name=args.name, clean=args.clean, remote=args.remote, password=args.password) def search(self, *args): """ Search package recipes and binaries in the local cache or in a remote server. @@ -674,6 +515,8 @@ def search(self, *args): parser.add_argument('--case-sensitive', default=False, action='store_true', help='Make a case-sensitive search') parser.add_argument('-r', '--remote', help='Remote origin') + parser.add_argument('--raw', default=False, action='store_true', + help='Make a case-sensitive search') parser.add_argument('-q', '--query', default=None, help='Packages query: "os=Windows AND ' '(arch=x86 OR compiler=gcc)".' ' The "pattern" parameter ' @@ -681,21 +524,31 @@ def search(self, *args): 'reference: MyPackage/1.2' '@user/channel') args = parser.parse_args(*args) - log_command("search", vars(args)) + outputer = CommandOutputer(self._user_io, self._client_cache) - reference = _check_query_parameter_and_get_reference(args) + try: + reference = ConanFileReference.loads(args.pattern) + except: + reference = None - self._manager.search(reference or args.pattern, - args.remote, - ignorecase=not args.case_sensitive, - packages_query=args.query) + if reference: + ret = self._conan.search_packages(reference, query=args.query, remote=args.remote) + ordered_packages, reference, recipe_hash, packages_query = ret + outputer.print_search_packages(ordered_packages, reference, recipe_hash, + packages_query) + else: + refs = self._conan.search_recipes(args.pattern, remote=args.remote, + case_sensitive=args.case_sensitive) + self._check_query_parameter_and_get_reference(args.pattern, args.query) + outputer.print_search_references(refs, args.pattern, args.raw) def upload(self, *args): """ Uploads a package recipe and the generated binary packages to a specified remote """ parser = argparse.ArgumentParser(description=self.upload.__doc__, prog="conan upload") - parser.add_argument('pattern', help='Pattern or package recipe reference, e.g., "openssl/*", "MyPackage/1.2@user/channel"') + parser.add_argument('pattern', help='Pattern or package recipe reference, e.g., "openssl/*", ' + '"MyPackage/1.2@user/channel"') # TODO: packageparser.add_argument('package', help='user name') parser.add_argument("--package", "-p", default=None, help='package ID to upload') parser.add_argument("--remote", "-r", help='upload to this specific remote') @@ -707,22 +560,17 @@ def upload(self, *args): default=False, help='Do not check conan recipe date, override remote with local') parser.add_argument('--confirm', '-c', default=False, - action='store_true', help='If pattern is given upload all matching recipes without confirmation') + action='store_true', help='If pattern is given upload all matching recipes without ' + 'confirmation') parser.add_argument('--retry', default=2, type=int, help='In case of fail retries to upload again the specified times') parser.add_argument('--retry_wait', default=5, type=int, help='Waits specified seconds before retry again') args = parser.parse_args(*args) - log_command("upload", vars(args)) - - if args.package and not is_a_reference(args.pattern): - raise ConanException("-p parameter only allowed with a valid recipe reference, not with a pattern") - - self._manager.upload(args.pattern, args.package, - args.remote, all_packages=args.all, - force=args.force, confirm=args.confirm, retry=args.retry, - retry_wait=args.retry_wait, skip_upload=args.skip_upload) + return self._conan.upload(pattern=args.pattern, package=args.package, remote=args.remote, all=args.all, + force=args.force, confirm=args.confirm, retry=args.retry, retry_wait=args.retry_wait, + skip_upload=args.skip_upload) def remote(self, *args): """ Handles the remote list and the package recipes associated to a remote. @@ -737,6 +585,8 @@ def remote(self, *args): parser_add.add_argument('url', help='url of the remote') parser_add.add_argument('verify_ssl', help='Verify SSL certificated. Default True', default="True", nargs="?") + parser_add.add_argument("-i", "--insert", nargs="?", const=0, type=int, + help="insert remote at specific index") parser_rm = subparsers.add_parser('remove', help='remove a remote') parser_rm.add_argument('remote', help='name of the remote') parser_upd = subparsers.add_parser('update', help='update the remote url') @@ -758,29 +608,33 @@ def remote(self, *args): parser_pupd.add_argument('reference', help='package recipe reference') parser_pupd.add_argument('remote', help='name of the remote') args = parser.parse_args(*args) - log_command("remote", vars(args)) - registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + reference = args.reference if hasattr(args, 'reference') else None + + try: + verify_ssl = get_bool_from_text(args.verify_ssl) if hasattr(args, 'verify_ssl') else False + except ConanException as exc: + self._raise_exception_printing(str(exc)) + + remote = args.remote if hasattr(args, 'remote') else None + url = args.url if hasattr(args, 'url') else None + if args.subcommand == "list": - for r in registry.remotes: - self._user_io.out.info("%s: %s [Verify SSL: %s]" % (r.name, r.url, r.verify_ssl)) + return self._conan.remote_list() elif args.subcommand == "add": - verify = get_bool_from_text(args.verify_ssl) - registry.add(args.remote, args.url, verify) + return self._conan.remote_add(remote, url, verify_ssl, args.insert) elif args.subcommand == "remove": - registry.remove(args.remote) + return self._conan.remote_remove(remote) elif args.subcommand == "update": - verify = get_bool_from_text(args.verify_ssl) - registry.update(args.remote, args.url, verify) + return self._conan.remote_update(remote, url, verify_ssl) elif args.subcommand == "list_ref": - for ref, remote in registry.refs.items(): - self._user_io.out.info("%s: %s" % (ref, remote)) + return self._conan.remote_list_ref() elif args.subcommand == "add_ref": - registry.add_ref(args.reference, args.remote) + return self._conan.remote_add_ref(reference, remote) elif args.subcommand == "remove_ref": - registry.remove_ref(args.reference) + return self._conan.remote_remove_ref(reference) elif args.subcommand == "update_ref": - registry.update_ref(args.reference, args.remote) + return self._conan.remote_update_ref(reference, remote) def profile(self, *args): """ List profiles in the '.conan/profiles' folder, or show profile details. @@ -798,19 +652,10 @@ def profile(self, *args): ' a profile file in any location.') parser_show.add_argument('profile', help='name of the profile') args = parser.parse_args(*args) - log_command("profile", vars(args)) - if args.subcommand == "list": - folder = self._client_cache.profiles_path - if os.path.exists(folder): - profiles = [name for name in os.listdir(folder) if not os.path.isdir(name)] - for p in sorted(profiles): - self._user_io.out.info(p) - else: - self._user_io.out.info("No profiles defined") - elif args.subcommand == "show": - p = Profile.read_file(args.profile, os.getcwd(), self._client_cache.profiles_path) - Printer(self._user_io.out).print_profile(args.profile, p) + profile = args.profile if hasattr(args, 'profile') else None + + return self._conan.profile(args.subcommand, profile) def _show_help(self): """ prints a summary of all commands @@ -837,6 +682,22 @@ def _commands(self): result[method_name] = method return result + def _check_query_parameter_and_get_reference(self, pattern, query): + reference = None + if pattern: + try: + reference = ConanFileReference.loads(pattern) + except ConanException: + if query is not None: + msg = "-q parameter only allowed with a valid recipe reference as search pattern. e.j conan search " \ + "MyPackage/1.2@user/channel -q \"os=Windows\"" + self._raise_exception_printing(msg) + return reference + + def _raise_exception_printing(self, msg): + self._user_io.out.error(msg) + raise ConanException(msg) + def run(self, *args): """HIDDEN: entry point for executing commands, dispatcher to class methods @@ -858,79 +719,32 @@ def run(self, *args): except IndexError as exc: # No parameters self._show_help() return False - with tools.environment_append(self.client_cache.conan_config.env_vars): - method(args[0][1:]) + method(args[0][1:]) except (KeyboardInterrupt, SystemExit) as exc: logger.error(exc) errors = True except ConanException as exc: - # import traceback - # logger.debug(traceback.format_exc()) errors = True msg = exception_message_safe(exc) self._user_io.out.error(msg) - try: - log_exception(exc, msg) - except: - pass except Exception as exc: - # import traceback - # print(traceback.format_exc()) + import traceback + print(traceback.format_exc()) msg = exception_message_safe(exc) - try: - log_exception(exc, msg) - except: - pass - raise exc + self._user_io.out.error(msg) return errors -def _check_query_parameter_and_get_reference(args): - reference = None - if args.pattern: - try: - reference = ConanFileReference.loads(args.pattern) - except ConanException: - if args.query is not None: - raise ConanException("-q parameter only allowed with a valid recipe " - "reference as search pattern. e.j conan search " - "MyPackage/1.2@user/channel -q \"os=Windows\"") - return reference - - -def _parse_manifests_arguments(args, reference, current_path): - if args.manifests and args.manifests_interactive: - raise ConanException("Do not specify both manifests and " - "manifests-interactive arguments") - if args.verify and (args.manifests or args.manifests_interactive): - raise ConanException("Do not specify both 'verify' and " - "'manifests' or 'manifests-interactive' arguments") - manifest_folder = args.verify or args.manifests or args.manifests_interactive - if manifest_folder: - if not os.path.isabs(manifest_folder): - if isinstance(reference, ConanFileReference): - manifest_folder = os.path.join(current_path, manifest_folder) - else: - manifest_folder = os.path.join(reference, manifest_folder) - manifest_verify = args.verify is not None - manifest_interactive = args.manifests_interactive is not None - else: - manifest_verify = manifest_interactive = False - - return manifest_folder, manifest_interactive, manifest_verify - - def _add_manifests_arguments(parser): - default_manifest_folder = '.conan_manifests' parser.add_argument("--manifests", "-m", const=default_manifest_folder, nargs="?", help='Install dependencies manifests in folder for later verify.' - ' Default folder is .conan_manifests, but can be changed') + ' Default folder is .conan_manifests, but can be changed') parser.add_argument("--manifests-interactive", "-mi", const=default_manifest_folder, nargs="?", help='Install dependencies manifests in folder for later verify, ' - 'asking user for confirmation. ' - 'Default folder is .conan_manifests, but can be changed') + 'asking user for confirmation. ' + 'Default folder is .conan_manifests, but can be changed') parser.add_argument("--verify", "-v", const=default_manifest_folder, nargs="?", help='Verify dependencies manifests against stored ones') @@ -966,97 +780,12 @@ def _add_common_install_arguments(parser, build_help): ''' -def _get_reference(args): - current_path = os.getcwd() - try: - reference = ConanFileReference.loads(args.reference) - except: - if "@" in args.reference: - raise - if not os.path.isabs(args.reference): - reference = os.path.normpath(os.path.join(current_path, args.reference)) - else: - reference = args.reference - return current_path, reference - - -def migrate_and_get_client_cache(base_folder, out, storage_folder=None): - # Init paths - client_cache = ClientCache(base_folder, storage_folder, out) - - # Migration system - migrator = ClientMigrator(client_cache, Version(CLIENT_VERSION), out) - migrator.migrate() - - return client_cache - - -def get_command(): - - def instance_remote_manager(client_cache): - requester = requests.Session() - requester.proxies = client_cache.conan_config.proxies - # Verify client version against remotes - version_checker_requester = VersionCheckerRequester(requester, Version(CLIENT_VERSION), - Version(MIN_SERVER_COMPATIBLE_VERSION), - out) - # To handle remote connections - put_headers = client_cache.read_put_headers() - rest_api_client = RestApiClient(out, requester=version_checker_requester, put_headers=put_headers) - # To store user and token - localdb = LocalDB(client_cache.localdb) - # Wraps RestApiClient to add authentication support (same interface) - auth_manager = ConanApiAuthManager(rest_api_client, user_io, localdb) - # Handle remote connections - remote_manager = RemoteManager(client_cache, auth_manager, out) - return remote_manager - - use_color = get_env("CONAN_COLOR_DISPLAY", 1) - if use_color and hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): - import colorama - colorama.init() - color = True - else: - color = False - out = ConanOutput(sys.stdout, color) - user_io = UserIO(out=out) - - user_folder = os.getenv("CONAN_USER_HOME", conan_expand_user("~")) - - try: - client_cache = migrate_and_get_client_cache(user_folder, out) - except Exception as e: - out.error(str(e)) - sys.exit(True) - - with tools.environment_append(client_cache.conan_config.env_vars): - # Adjust CONAN_LOGGING_LEVEL with the env readed - conans.util.log.logger = configure_logger() - - # Get the new command instance after migrations have been done - remote_manager = instance_remote_manager(client_cache) - - # Get a search manager - search_adapter = DiskSearchAdapter() - search_manager = DiskSearchManager(client_cache, search_adapter) - command = Command(client_cache, user_io, get_conan_runner(), remote_manager, search_manager) - - return command - - -def get_conan_runner(): - print_commands_to_output = get_env("CONAN_PRINT_RUN_COMMANDS", False) - generate_run_log_file = get_env("CONAN_LOG_RUN_TO_FILE", False) - log_run_to_output = get_env("CONAN_LOG_RUN_TO_OUTPUT", True) - runner = ConanRunner(print_commands_to_output, generate_run_log_file, log_run_to_output) - return runner - - def main(args): """ main entry point of the conan application, using a Command to parse parameters """ - command = get_command() + conan_api = Conan.factory() + command = Command(conan_api, conan_api._client_cache, conan_api._user_io) current_dir = os.getcwd() try: import signal diff --git a/conans/client/command_profile_args.py b/conans/client/command_profile_args.py deleted file mode 100644 index f89a9b15015..00000000000 --- a/conans/client/command_profile_args.py +++ /dev/null @@ -1,74 +0,0 @@ -from conans.model.profile import Profile -from conans.errors import ConanException -from collections import defaultdict, OrderedDict -from conans.model.env_info import EnvValues -from conans.model.options import OptionsValues -from conans.model.scope import Scopes - - -def profile_from_args(args, cwd, default_folder): - """ Return a Profile object, as the result of merging a potentially existing Profile - file and the args command-line arguments - """ - file_profile = Profile.read_file(args.profile, cwd, default_folder) - args_profile = _profile_parse_args(args.settings, args.options, args.env, args.scope) - - if file_profile: - file_profile.update(args_profile) - return file_profile - else: - return args_profile - - -def _profile_parse_args(settings, options, envs, scopes): - """ return a Profile object result of parsing raw data - """ - def _get_tuples_list_from_extender_arg(items): - if not items: - return [] - # Validate the pairs - for item in items: - chunks = item.split("=", 1) - if len(chunks) != 2: - raise ConanException("Invalid input '%s', use 'name=value'" % item) - return [(item[0], item[1]) for item in [item.split("=", 1) for item in items]] - - def _get_simple_and_package_tuples(items): - """Parse items like "thing:item=value or item2=value2 and returns a tuple list for - the simple items (name, value) and a dict for the package items - {package: [(item, value)...)], ...} - """ - simple_items = [] - package_items = defaultdict(list) - tuples = _get_tuples_list_from_extender_arg(items) - for name, value in tuples: - if ":" in name: # Scoped items - tmp = name.split(":", 1) - ref_name = tmp[0] - name = tmp[1] - package_items[ref_name].append((name, value)) - else: - simple_items.append((name, value)) - return simple_items, package_items - - def _get_env_values(env, package_env): - env_values = EnvValues() - for name, value in env: - env_values.add(name, EnvValues.load_value(value)) - for package, data in package_env.items(): - for name, value in data: - env_values.add(name, EnvValues.load_value(value), package) - return env_values - - result = Profile() - options = _get_tuples_list_from_extender_arg(options) - result.options = OptionsValues(options) - env, package_env = _get_simple_and_package_tuples(envs) - env_values = _get_env_values(env, package_env) - result.env_values = env_values - settings, package_settings = _get_simple_and_package_tuples(settings) - result.settings = OrderedDict(settings) - for pkg, values in package_settings.items(): - result.package_settings[pkg] = OrderedDict(values) - result.scopes = Scopes.from_list(scopes) if scopes else Scopes() - return result diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py new file mode 100644 index 00000000000..d2c17042841 --- /dev/null +++ b/conans/client/conan_api.py @@ -0,0 +1,678 @@ +import hashlib +import os +import sys + +import requests +from collections import defaultdict, OrderedDict + +import conans +from conans import __version__ as CLIENT_VERSION, tools +from conans.client.client_cache import ClientCache +from conans.client.conf import MIN_SERVER_COMPATIBLE_VERSION, ConanClientConfigParser +from conans.client.loader import load_consumer_conanfile +from conans.client.manager import ConanManager +from conans.client.migrations import ClientMigrator +from conans.client.output import ConanOutput +from conans.client.printer import Printer +from conans.client.profile_loader import read_profile +from conans.client.remote_manager import RemoteManager +from conans.client.remote_registry import RemoteRegistry +from conans.client.rest.auth_manager import ConanApiAuthManager +from conans.client.rest.rest_client import RestApiClient +from conans.client.rest.version_checker import VersionCheckerRequester +from conans.client.runner import ConanRunner +from conans.client.store.localdb import LocalDB +from conans.client.userio import UserIO +from conans.errors import ConanException +from conans.model.env_info import EnvValues +from conans.model.options import OptionsValues +from conans.model.profile import Profile +from conans.model.ref import ConanFileReference, is_a_reference +from conans.model.scope import Scopes +from conans.model.version import Version +from conans.paths import CONANFILE, conan_expand_user +from conans.search.search import DiskSearchManager, DiskSearchAdapter +from conans.util.env_reader import get_env +from conans.util.files import rmdir, save_files, exception_message_safe +from conans.util.log import configure_logger +from conans.util.tracer import log_command, log_exception + + +default_manifest_folder = '.conan_manifests' + + +def get_basic_requester(client_cache): + requester = requests.Session() + requester.proxies = client_cache.conan_config.proxies + return requester + + +def api_method(f): + def wrapper(*args, **kwargs): + the_self = args[0] + try: + log_command(f.__name__, kwargs) + with tools.environment_append(the_self._client_cache.conan_config.env_vars): + # Patch the globals in tools + return f(*args, **kwargs) + except Exception as exc: + msg = exception_message_safe(exc) + try: + log_exception(exc, msg) + except: + pass + raise + + return wrapper + + +def prepare_cwd(cwd): + if cwd: + if os.path.isabs(cwd): + return cwd + else: + return os.path.abspath(cwd) + else: + return os.getcwd() + + +class ConanAPIV1(object): + + @staticmethod + def factory(): + """Factory""" + + def instance_remote_manager(client_cache): + requester = get_basic_requester(client_cache) + # Verify client version against remotes + version_checker_requester = VersionCheckerRequester(requester, Version(CLIENT_VERSION), + Version(MIN_SERVER_COMPATIBLE_VERSION), + out) + # To handle remote connections + put_headers = client_cache.read_put_headers() + rest_api_client = RestApiClient(out, requester=version_checker_requester, put_headers=put_headers) + # To store user and token + localdb = LocalDB(client_cache.localdb) + # Wraps RestApiClient to add authentication support (same interface) + auth_manager = ConanApiAuthManager(rest_api_client, user_io, localdb) + # Handle remote connections + remote_manager = RemoteManager(client_cache, auth_manager, out) + return remote_manager + + use_color = get_env("CONAN_COLOR_DISPLAY", 1) + if use_color and hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + import colorama + colorama.init() + color = True + else: + color = False + out = ConanOutput(sys.stdout, color) + user_io = UserIO(out=out) + + user_folder = os.getenv("CONAN_USER_HOME", conan_expand_user("~")) + + try: + client_cache = migrate_and_get_client_cache(user_folder, out) + except Exception as e: + out.error(str(e)) + raise + + with tools.environment_append(client_cache.conan_config.env_vars): + # Adjust CONAN_LOGGING_LEVEL with the env readed + conans.util.log.logger = configure_logger() + + # Get the new command instance after migrations have been done + remote_manager = instance_remote_manager(client_cache) + + # Get a search manager + search_adapter = DiskSearchAdapter() + search_manager = DiskSearchManager(client_cache, search_adapter) + conan = Conan(client_cache, user_io, get_conan_runner(), remote_manager, search_manager) + + return conan + + def __init__(self, client_cache, user_io, runner, remote_manager, search_manager): + assert isinstance(user_io, UserIO) + assert isinstance(client_cache, ClientCache) + self._client_cache = client_cache + self._user_io = user_io + self._runner = runner + self._manager = ConanManager(client_cache, user_io, runner, remote_manager, search_manager) + # Patch the tools module with a good requester and user_io + tools._global_requester = get_basic_requester(self._client_cache) + tools._global_output = self._user_io.out + + @api_method + def new(self, name, header=False, pure_c=False, test=False, exports_sources=False, bare=False, cwd=None, + visual_versions=None, linux_gcc_versions=None, linux_clang_versions=None, osx_clang_versions=None, + shared=None, upload_url=None, gitignore=None): + from conans.client.new import get_files + cwd = prepare_cwd(cwd) + files = get_files(name, header=header, pure_c=pure_c, test=test, + exports_sources=exports_sources, bare=bare, + visual_versions=visual_versions, + linux_gcc_versions=linux_gcc_versions, + linux_clang_versions=linux_clang_versions, + osx_clang_versions=osx_clang_versions, shared=shared, + upload_url=upload_url, gitignore=gitignore) + + save_files(cwd, files) + for f in sorted(files): + self._user_io.out.success("File saved: %s" % f) + + @api_method + def test_package(self, path=None, profile_name=None, settings=None, options=None, env=None, scope=None, + test_folder=None, not_export=False, build=None, keep_source=False, + verify=default_manifest_folder, manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder, + remote=None, update=False, cwd=None): + settings = settings or [] + options = options or [] + env = env or [] + cwd = prepare_cwd(cwd) + + root_folder = os.path.normpath(os.path.join(cwd, path)) + if test_folder: + test_folder_name = test_folder + test_folder = os.path.join(root_folder, test_folder_name) + test_conanfile = os.path.join(test_folder, "conanfile.py") + if not os.path.exists(test_conanfile): + raise ConanException("test folder '%s' not available, " + "or it doesn't have a conanfile.py" % test_folder) + else: + for name in ["test_package", "test"]: + test_folder_name = name + test_folder = os.path.join(root_folder, test_folder_name) + test_conanfile = os.path.join(test_folder, "conanfile.py") + if os.path.exists(test_conanfile): + break + else: + raise ConanException("test folder 'test_package' not available, " + "or it doesn't have a conanfile.py") + + options = options or [] + settings = settings or [] + + sha = hashlib.sha1("".join(options + settings).encode()).hexdigest() + build_folder = os.path.join(test_folder, "build", sha) + rmdir(build_folder) + # shutil.copytree(test_folder, build_folder) + + profile = profile_from_args(profile_name, settings, options, env, scope, cwd, + self._client_cache.profiles_path) + conanfile = load_consumer_conanfile(test_conanfile, "", + self._client_cache.settings, self._runner, + self._user_io.out) + try: + # convert to list from ItemViews required for python3 + if hasattr(conanfile, "requirements"): + conanfile.requirements() + reqs = list(conanfile.requires.items()) + first_dep = reqs[0][1].conan_reference + except Exception: + raise ConanException("Unable to retrieve first requirement of test conanfile.py") + + # Forcing an export! + if not not_export: + self._user_io.out.info("Exporting package recipe") + user_channel = "%s/%s" % (first_dep.user, first_dep.channel) + self._manager.export(user_channel, root_folder, keep_source=keep_source) + + lib_to_test = first_dep.name + # Get False or a list of patterns to check + if build is None and lib_to_test: # Not specified, force build the tested library + build = [lib_to_test] + + manifests = _parse_manifests_arguments(verify, manifests, manifests_interactive, + root_folder, cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + self._manager.install(reference=test_folder, + current_path=build_folder, + manifest_folder=manifest_folder, + manifest_verify=manifest_verify, + manifest_interactive=manifest_interactive, + remote=remote, + profile=profile, + build_modes=build, + update=update, + generators=["txt"] + ) + + test_conanfile = os.path.join(test_folder, CONANFILE) + self._manager.build(test_conanfile, test_folder, build_folder, test=True) + + # Alias to test + @api_method + def test(self, *args): + self.test_package(*args) + + @api_method + def package_files(self, reference, package_folder=None, profile_name=None, + force=False, settings=None, options=None, cwd=None): + + cwd = prepare_cwd(cwd) + + reference = ConanFileReference.loads(reference) + package_folder = package_folder or cwd + if not os.path.isabs(package_folder): + package_folder = os.path.join(cwd, package_folder) + profile = profile_from_args(profile_name, settings, options, env=None, + scope=None, cwd=cwd, default_folder=self._client_cache.profiles_path) + self._manager.package_files(reference=reference, package_folder=package_folder, + profile=profile, force=force) + + @api_method + def install(self, reference="", package=None, settings=None, options=None, env=None, scope=None, all=False, + remote=None, werror=False, verify=default_manifest_folder, manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder, build=None, profile_name=None, + update=False, generator=None, no_imports=False, filename=None, cwd=None): + + self._user_io.out.werror_active = werror + cwd = prepare_cwd(cwd) + + try: + ref = ConanFileReference.loads(reference) + except: + ref = os.path.normpath(os.path.join(cwd, reference)) + + if all or package: # Install packages without settings (fixed ids or all) + if all: + package = [] + if not reference or not isinstance(ref, ConanFileReference): + raise ConanException("Invalid package recipe reference. " + "e.g., MyPackage/1.2@user/channel") + self._manager.download(ref, package, remote=remote) + else: # Classic install, package chosen with settings and options + manifests = _parse_manifests_arguments(verify, manifests, manifests_interactive, cwd, + self._client_cache.profiles_path) + manifest_folder, manifest_interactive, manifest_verify = manifests + profile = profile_from_args(profile_name, settings, options, env, scope, cwd, + self._client_cache.profiles_path) + + self._manager.install(reference=ref, + current_path=cwd, + remote=remote, + profile=profile, + build_modes=build, + filename=filename, + update=update, + manifest_folder=manifest_folder, + manifest_verify=manifest_verify, + manifest_interactive=manifest_interactive, + generators=generator, + no_imports=no_imports) + + @api_method + def config_get(self, item): + config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) + self._user_io.out.info(config_parser.get_item(item)) + return config_parser.get_item(item) + + @api_method + def config_set(self, item, value): + config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) + config_parser.set_item(item, value) + + @api_method + def config_rm(self, item): + config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) + config_parser.rm_item(item) + + @api_method + def info_build_order(self, reference, settings=None, options=None, env=None, scope=None, profile_name=None, + filename=None, remote=None, build_order=None, check_updates=None, cwd=None): + + current_path = prepare_cwd(cwd) + try: + reference = ConanFileReference.loads(reference) + except: + reference = os.path.normpath(os.path.join(current_path, reference)) + + profile = profile_from_args(profile_name, settings, options, env, scope, cwd, self._client_cache.profiles_path) + graph = self._manager.info_build_order(reference, profile, filename, build_order, remote, check_updates, cwd=cwd) + return graph + + @api_method + def info_nodes_to_build(self, reference, build_modes, settings=None, options=None, env=None, scope=None, + profile_name=None, filename=None, remote=None, check_updates=None, cwd=None): + + current_path = prepare_cwd(cwd) + try: + reference = ConanFileReference.loads(reference) + except: + reference = os.path.normpath(os.path.join(current_path, reference)) + + profile = profile_from_args(profile_name, settings, options, env, scope, cwd, self._client_cache.profiles_path) + ret = self._manager.info_nodes_to_build(reference, profile, filename, build_modes, remote, check_updates, cwd) + ref_list, project_reference = ret + return ref_list, project_reference + + @api_method + def info_get_graph(self, reference, remote=None, settings=None, options=None, env=None, scope=None, + profile_name=None, update=False, filename=None, cwd=None): + + current_path = prepare_cwd(cwd) + try: + reference = ConanFileReference.loads(reference) + except: + reference = os.path.normpath(os.path.join(current_path, reference)) + + profile = profile_from_args(profile_name, settings, options, env, scope, current_path, + self._client_cache.profiles_path) + ret = self._manager.info_get_graph(reference=reference, current_path=current_path, remote=remote, + profile=profile, check_updates=update, filename=filename) + deps_graph, graph_updates_info, project_reference = ret + return deps_graph, graph_updates_info, project_reference + + @api_method + def build(self, path="", source_folder=None, filename=None, cwd=None): + + current_path = prepare_cwd(cwd) + if path: + root_path = os.path.abspath(path) + else: + root_path = current_path + + build_folder = current_path + source_folder = source_folder or root_path + if not os.path.isabs(source_folder): + source_folder = os.path.normpath(os.path.join(current_path, source_folder)) + + if filename and filename.endswith(".txt"): + raise ConanException("A conanfile.py is needed to call 'conan build'") + conanfile_path = os.path.join(root_path, filename or CONANFILE) + self._manager.build(conanfile_path, source_folder, build_folder) + + @api_method + def package(self, reference="", package=None, build_folder=None, source_folder=None, cwd=None): + + current_path = prepare_cwd(cwd) + try: + self._manager.package(ConanFileReference.loads(reference), package) + except: + if "@" in reference: + raise + recipe_folder = reference + if not os.path.isabs(recipe_folder): + recipe_folder = os.path.normpath(os.path.join(current_path, recipe_folder)) + build_folder = build_folder or current_path + if not os.path.isabs(build_folder): + build_folder = os.path.normpath(os.path.join(current_path, build_folder)) + package_folder = current_path + source_folder = source_folder or recipe_folder + self._manager.local_package(package_folder, recipe_folder, build_folder, source_folder) + + @api_method + def source(self, reference, force=False, cwd=None): + cwd = prepare_cwd(cwd) + current_path, reference = _get_reference(reference, cwd) + self._manager.source(current_path, reference, force) + + @api_method + def imports(self, reference, undo=False, dest=None, filename=None, cwd=None): + cwd = prepare_cwd(cwd) + + if undo: + if not os.path.isabs(reference): + current_path = os.path.normpath(os.path.join(cwd, reference)) + else: + current_path = reference + self._manager.imports_undo(current_path) + else: + cwd = prepare_cwd(cwd) + current_path, reference = _get_reference(reference, cwd) + self._manager.imports(current_path, reference, filename, dest) + + @api_method + def export(self, user, path=None, keep_source=False, filename=None, cwd=None): + cwd = prepare_cwd(cwd) + current_path = os.path.abspath(path or cwd) + self._manager.export(user, current_path, keep_source, filename=filename) + + @api_method + def remove(self, pattern, query=None, packages=None, builds=None, src=False, force=False, + remote=None): + self._manager.remove(pattern, package_ids_filter=packages, build_ids=builds, + src=src, force=force, remote=remote, packages_query=query) + + @api_method + def copy(self, reference="", user_channel="", force=False, all=False, package=None): + reference = ConanFileReference.loads(reference) + new_ref = ConanFileReference.loads("%s/%s@%s" % (reference.name, + reference.version, + user_channel)) + if all: + package = [] + self._manager.copy(reference, package, new_ref.user, new_ref.channel, force) + + @api_method + def user(self, name=None, clean=False, remote=None, password=None): + if clean: + localdb = LocalDB(self._client_cache.localdb) + localdb.init(clean=True) + self._user_io.out.success("Deleted user data") + return + self._manager.user(remote, name, password) + + @api_method + def search_recipes(self, pattern, remote=None, case_sensitive=False): + refs = self._manager.search_recipes(pattern, remote, ignorecase=not case_sensitive) + return refs + + @api_method + def search_packages(self, reference, query=None, remote=None): + ret = self._manager.search_packages(reference, remote, packages_query=query) + return ret + + @api_method + def upload(self, pattern, package=None, remote=None, all=False, force=False, confirm=False, + retry=2, retry_wait=5, + skip_upload=False): + """ Uploads a package recipe and the generated binary packages to a specified remote + """ + if package and not is_a_reference(pattern): + raise ConanException("-p parameter only allowed with a valid recipe reference, " + "not with a pattern") + + self._manager.upload(pattern, package, + remote, all_packages=all, + force=force, confirm=confirm, retry=retry, + retry_wait=retry_wait, skip_upload=skip_upload) + + @api_method + def remote_list(self): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + for r in registry.remotes: + self._user_io.out.info("%s: %s [Verify SSL: %s]" % (r.name, r.url, r.verify_ssl)) + + @api_method + def remote_add(self, remote, url, verify_ssl=True, insert=None): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.add(remote, url, verify_ssl, insert) + + @api_method + def remote_remove(self, remote): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.remove(remote) + + @api_method + def remote_update(self, remote, url, verify_ssl=True): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.update(remote, url, verify_ssl) + + @api_method + def remote_list_ref(self): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + for ref, remote in registry.refs.items(): + self._user_io.out.info("%s: %s" % (ref, remote)) + + @api_method + def remote_add_ref(self, reference, remote): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.add_ref(reference, remote) + + @api_method + def remote_remove_ref(self, reference): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.remove_ref(reference) + + @api_method + def remote_update_ref(self, reference, remote): + registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) + return registry.update_ref(reference, remote) + + @api_method + def profile(self, subcommand, profile=None, cwd=None): + cwd = prepare_cwd(cwd) + if subcommand == "list": + folder = self._client_cache.profiles_path + if os.path.exists(folder): + profiles = [name for name in os.listdir(folder) if not os.path.isdir(name)] + for p in sorted(profiles): + self._user_io.out.info(p) + else: + self._user_io.out.info("No profiles defined") + elif subcommand == "show": + p, _ = read_profile(profile, os.getcwd(), self._client_cache.profiles_path) + Printer(self._user_io.out).print_profile(profile, p) + + +Conan = ConanAPIV1 + + +def _check_query_parameter_and_get_reference(query, pattern): + reference = None + if pattern: + try: + reference = ConanFileReference.loads(pattern) + except ConanException: + if query is not None: + raise ConanException("-q parameter only allowed with a valid recipe " + "reference as search pattern. e.j conan search " + "MyPackage/1.2@user/channel -q \"os=Windows\"") + return reference + + +def _parse_manifests_arguments(verify, manifests, manifests_interactive, reference, current_path): + if manifests and manifests_interactive: + raise ConanException("Do not specify both manifests and " + "manifests-interactive arguments") + if verify and (manifests or manifests_interactive): + raise ConanException("Do not specify both 'verify' and " + "'manifests' or 'manifests-interactive' arguments") + manifest_folder = verify or manifests or manifests_interactive + if manifest_folder: + if not os.path.isabs(manifest_folder): + if isinstance(reference, ConanFileReference): + manifest_folder = os.path.join(current_path, manifest_folder) + else: + manifest_folder = os.path.join(reference, manifest_folder) + manifest_verify = verify is not None + manifest_interactive = manifests_interactive is not None + else: + manifest_verify = manifest_interactive = False + + return manifest_folder, manifest_interactive, manifest_verify + + +def get_conan_runner(): + print_commands_to_output = get_env("CONAN_PRINT_RUN_COMMANDS", False) + generate_run_log_file = get_env("CONAN_LOG_RUN_TO_FILE", False) + log_run_to_output = get_env("CONAN_LOG_RUN_TO_OUTPUT", True) + runner = ConanRunner(print_commands_to_output, generate_run_log_file, log_run_to_output) + return runner + + +def _get_reference(ref, cwd=None): + try: + reference = ConanFileReference.loads(ref) + except: + if "@" in ref: + raise + if not os.path.isabs(ref): + reference = os.path.normpath(os.path.join(cwd, ref)) + else: + reference = ref + return cwd, reference + + +def migrate_and_get_client_cache(base_folder, out, storage_folder=None): + # Init paths + client_cache = ClientCache(base_folder, storage_folder, out) + + # Migration system + migrator = ClientMigrator(client_cache, Version(CLIENT_VERSION), out) + migrator.migrate() + + return client_cache + + +# Profile helpers + + +def profile_from_args(profile, settings, options, env, scope, cwd, default_folder): + """ Return a Profile object, as the result of merging a potentially existing Profile + file and the args command-line arguments + """ + file_profile, _ = read_profile(profile, cwd, default_folder) + args_profile = _profile_parse_args(settings, options, env, scope) + + if file_profile: + file_profile.update(args_profile) + return file_profile + else: + return args_profile + + +def _profile_parse_args(settings, options, envs, scopes): + """ return a Profile object result of parsing raw data + """ + def _get_tuples_list_from_extender_arg(items): + if not items: + return [] + # Validate the pairs + for item in items: + chunks = item.split("=", 1) + if len(chunks) != 2: + raise ConanException("Invalid input '%s', use 'name=value'" % item) + return [(item[0], item[1]) for item in [item.split("=", 1) for item in items]] + + def _get_simple_and_package_tuples(items): + """Parse items like "thing:item=value or item2=value2 and returns a tuple list for + the simple items (name, value) and a dict for the package items + {package: [(item, value)...)], ...} + """ + simple_items = [] + package_items = defaultdict(list) + tuples = _get_tuples_list_from_extender_arg(items) + for name, value in tuples: + if ":" in name: # Scoped items + tmp = name.split(":", 1) + ref_name = tmp[0] + name = tmp[1] + package_items[ref_name].append((name, value)) + else: + simple_items.append((name, value)) + return simple_items, package_items + + def _get_env_values(env, package_env): + env_values = EnvValues() + for name, value in env: + env_values.add(name, EnvValues.load_value(value)) + for package, data in package_env.items(): + for name, value in data: + env_values.add(name, EnvValues.load_value(value), package) + return env_values + + result = Profile() + options = _get_tuples_list_from_extender_arg(options) + result.options = OptionsValues(options) + env, package_env = _get_simple_and_package_tuples(envs) + env_values = _get_env_values(env, package_env) + result.env_values = env_values + settings, package_settings = _get_simple_and_package_tuples(settings) + result.settings = OrderedDict(settings) + for pkg, values in package_settings.items(): + result.package_settings[pkg] = OrderedDict(values) + result.scopes = Scopes.from_list(scopes) if scopes else Scopes() + return result diff --git a/conans/client/conan_command_output.py b/conans/client/conan_command_output.py new file mode 100644 index 00000000000..c6a25c7a8e5 --- /dev/null +++ b/conans/client/conan_command_output.py @@ -0,0 +1,73 @@ +import os +import json + +from conans.model.ref import ConanFileReference + +from conans.client.printer import Printer + +from conans.client.remote_registry import RemoteRegistry + +from conans.client.grapher import ConanHTMLGrapher, ConanGrapher + +from conans.client.conan_api import prepare_cwd +from conans.util.files import save + + +class CommandOutputer(object): + + def __init__(self, user_io, client_cache): + self.user_io = user_io + self.client_cache = client_cache + + def build_order(self, info): + msg = ", ".join(str(s) for s in info) + self.user_io.out.info(msg) + + def json_build_order(self, info, json_output, cwd): + data = {"groups": [[str(ref) for ref in group] for group in info]} + json_str = json.dumps(data) + if json_output is True: # To the output + self.user_io.out.write(json_str) + else: # Path to a file + cwd = prepare_cwd(cwd) + if not os.path.isabs(json_output): + json_output = os.path.join(cwd, json_output) + save(json_output, json_str) + + def _read_dates(self, deps_graph): + ret = {} + for ref, _ in sorted(deps_graph.nodes): + if ref: + manifest = self.client_cache.load_manifest(ref) + ret[ref] = manifest.time_str + return ret + + def nodes_to_build(self, nodes_to_build): + self.user_io.out.info(", ".join(nodes_to_build)) + + def info(self, deps_graph, graph_updates_info, only, remote, package_filter, show_paths, project_reference): + registry = RemoteRegistry(self.client_cache.registry, self.user_io.out) + Printer(self.user_io.out).print_info(deps_graph, project_reference, + only, registry, graph_updates_info=graph_updates_info, + remote=remote, node_times=self._read_dates(deps_graph), + path_resolver=self.client_cache, package_filter=package_filter, + show_paths=show_paths) + + def info_graph(self, graph_filename, deps_graph, project_reference, cwd): + if graph_filename.endswith(".html"): + grapher = ConanHTMLGrapher(project_reference, deps_graph) + else: + grapher = ConanGrapher(project_reference, deps_graph) + + cwd = prepare_cwd(cwd) + if not os.path.isabs(graph_filename): + graph_filename = os.path.join(cwd, graph_filename) + grapher.graph_file(graph_filename) + + def print_search_references(self, references, pattern, raw): + printer = Printer(self.user_io.out) + printer.print_search_recipes(references, pattern, raw) + + def print_search_packages(self, ordered_packages, pattern, recipe_hash, packages_query): + printer = Printer(self.user_io.out) + printer.print_search_packages(ordered_packages, pattern, recipe_hash, packages_query) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 5267c4d729a..e4c58da999d 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -30,7 +30,7 @@ threads: [None, posix] libcxx: [libCstd, libstdcxx, libstlport, libstdc++] gcc: - version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9", "5.1", "5.2", "5.3", "5.4", "6.1", "6.2", "6.3"] + version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9", "5.1", "5.2", "5.3", "5.4", "6.1", "6.2", "6.3", "7.1"] libcxx: [libstdc++, libstdc++11] threads: [None, posix, win32] # Windows MinGW exception: [None, dwarf2, sjlj, seh] # Windows MinGW @@ -113,6 +113,7 @@ def env_vars(self): "CONAN_PRINT_RUN_COMMANDS": self._env_c("log.print_run_commands", "CONAN_PRINT_RUN_COMMANDS", "False"), "CONAN_COMPRESSION_LEVEL": self._env_c("general.compression_level", "CONAN_COMPRESSION_LEVEL", "9"), "CONAN_PYLINTRC": self._env_c("general.pylintrc", "CONAN_PYLINTRC", None), + "CONAN_PYLINT_WERR": self._env_c("general.pylint_werr", "CONAN_PYLINT_WERR", None), "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_RECIPE_LINTER": self._env_c("general.recipe_linter", "CONAN_RECIPE_LINTER", "True"), "CONAN_CPU_COUNT": self._env_c("general.cpu_count", "CONAN_CPU_COUNT", None), diff --git a/conans/client/configure_build_environment.py b/conans/client/configure_build_environment.py index 07c311b9acc..56fccb88d60 100644 --- a/conans/client/configure_build_environment.py +++ b/conans/client/configure_build_environment.py @@ -94,7 +94,8 @@ def __init__(self, conanfile): self.fpic = None def _get_host_build_target_flags(self, arch_detected, os_detected): - """Based on google search for build/host triplets, it could need a lot and complex verification""" + """Based on google search for build/host triplets, it could need a lot + and complex verification""" if not cross_building(self._conanfile.settings, os_detected, arch_detected): return False, False, False @@ -139,7 +140,10 @@ def configure(self, configure_dir=None, args=None, build=None, host=None, target https://gcc.gnu.org/onlinedocs/gccint/Configure-Terms.html """ - configure_dir = configure_dir or "./" + if configure_dir: + configure_dir = configure_dir.rstrip("/") + else: + configure_dir = "." auto_build, auto_host, auto_target = None, None, None if build is None or host is None or target is None: auto_build, auto_host, auto_target = self._get_host_build_target_flags(detected_architecture(), @@ -160,7 +164,8 @@ def configure(self, configure_dir=None, args=None, build=None, host=None, target triplet_args.append("--target %s" % (target or auto_target)) with environment_append(self.vars): - self._conanfile.run("%sconfigure %s %s" % (configure_dir, args_to_string(args), " ".join(triplet_args))) + self._conanfile.run("%s/configure %s %s" + % (configure_dir, args_to_string(args), " ".join(triplet_args))) def make(self, args=""): with environment_append(self.vars): diff --git a/conans/client/detect.py b/conans/client/detect.py index 362c430801d..54d5e4e88c1 100644 --- a/conans/client/detect.py +++ b/conans/client/detect.py @@ -25,7 +25,13 @@ def _gcc_compiler(output, compiler_exe="gcc"): try: _, out = _execute('%s -dumpversion' % compiler_exe) compiler = "gcc" - installed_version = re.search("([0-9]\.[0-9])", out).group() + installed_version = re.search("([0-9](\.[0-9])?)", out).group() + # Since GCC 7.1, -dumpversion return the major version number + # only ("7"). We must use -dumpfullversion to get the full version + # number ("7.1.1"). + if len(installed_version) == 1: + _, out = _execute('%s -dumpfullversion' % compiler_exe) + installed_version = re.search("([0-9]\.[0-9])", out).group() if installed_version: output.success("Found %s %s" % (compiler, installed_version)) return compiler, installed_version diff --git a/conans/client/export.py b/conans/client/export.py index 4ccfd171abb..4330a5a981b 100644 --- a/conans/client/export.py +++ b/conans/client/export.py @@ -97,7 +97,8 @@ def _init_export_folder(destination_folder): def execute_export(conanfile, origin_folder, destination_folder, output, filename=None): - def classify(patterns): + + def classify_patterns(patterns): patterns = patterns or [] included, excluded = [], [] for p in patterns: @@ -107,8 +108,8 @@ def classify(patterns): included.append(p) return included, excluded - included_exports, excluded_exports = classify(conanfile.exports) - included_sources, excluded_sources = classify(conanfile.exports_sources) + included_exports, excluded_exports = classify_patterns(conanfile.exports) + included_sources, excluded_sources = classify_patterns(conanfile.exports_sources) try: os.unlink(os.path.join(origin_folder, CONANFILE + 'c')) diff --git a/conans/client/file_copier.py b/conans/client/file_copier.py index c5ea306a62e..46a28458e48 100644 --- a/conans/client/file_copier.py +++ b/conans/client/file_copier.py @@ -3,6 +3,8 @@ import shutil from collections import defaultdict +from conans import tools + def report_copied_files(copied, output, warn=False): ext_files = defaultdict(list) @@ -44,7 +46,7 @@ def report(self, output, warn=False): report_copied_files(self._copied, output, warn) def __call__(self, pattern, dst="", src="", keep_path=True, links=False, symlinks=None, - excludes=None): + excludes=None, ignore_case=False): """ param pattern: an fnmatch file pattern of the files that should be copied. Eg. *.dll param dst: the destination local folder, wrt to current conanfile dir, to which @@ -67,13 +69,15 @@ def __call__(self, pattern, dst="", src="", keep_path=True, links=False, symlink src = os.path.join(self._base_src, src) dst = os.path.join(self._base_dst, dst) - files_to_copy, link_folders = self._filter_files(src, pattern, links, excludes) + + files_to_copy, link_folders = self._filter_files(src, pattern, links, excludes, + ignore_case) copied_files = self._copy_files(files_to_copy, src, dst, keep_path, links) self._link_folders(src, dst, link_folders) self._copied.extend(files_to_copy) return copied_files - def _filter_files(self, src, pattern, links, excludes=None): + def _filter_files(self, src, pattern, links, excludes, ignore_case): """ return a list of the files matching the patterns The list will be relative path names wrt to the root src folder @@ -86,7 +90,7 @@ def _filter_files(self, src, pattern, links, excludes=None): continue if links and os.path.islink(root): - linked_folders.append(root) + linked_folders.append(os.path.relpath(root, src)) subfolders[:] = [] continue basename = os.path.basename(root) @@ -105,27 +109,40 @@ def _filter_files(self, src, pattern, links, excludes=None): relative_name = os.path.normpath(os.path.join(relative_path, f)) filenames.append(relative_name) + if ignore_case: + filenames = {f.lower(): f for f in filenames} + pattern = pattern.lower() + files_to_copy = fnmatch.filter(filenames, pattern) if excludes: if not isinstance(excludes, (tuple, list)): excludes = (excludes, ) + if ignore_case: + excludes = [e.lower() for e in excludes] for exclude in excludes: files_to_copy = [f for f in files_to_copy if not fnmatch.fnmatch(f, exclude)] + if ignore_case: + files_to_copy = [filenames[f] for f in files_to_copy] + return files_to_copy, linked_folders - def _link_folders(self, src, dst, linked_folders): - for f in linked_folders: - relpath = os.path.relpath(f, src) - link = os.readlink(f) - abs_target = os.path.join(dst, relpath) - abs_link = os.path.join(dst, link) - try: - os.remove(abs_target) - except OSError: - pass - if os.path.exists(abs_link): - os.symlink(link, abs_target) + @staticmethod + def _link_folders(src, dst, linked_folders): + for linked_folder in linked_folders: + link = os.readlink(os.path.join(src, linked_folder)) + # The link is relative to the directory of the "linker_folder" + dest_dir = os.path.join(os.path.dirname(linked_folder), link) + if os.path.exists(os.path.join(dst, dest_dir)): + with tools.chdir(dst): + try: + # Remove the previous symlink + os.remove(linked_folder) + except OSError: + pass + # link is a string relative to linked_folder + # e.j: os.symlink("test/bar", "./foo/test_link") will create a link to foo/test/bar in ./foo/test_link + os.symlink(link, linked_folder) @staticmethod def _copy_files(files, src, dst, keep_path, symlinks): diff --git a/conans/client/generators/cmake_common.py b/conans/client/generators/cmake_common.py index bdeef3d3b69..fbaaad30a78 100644 --- a/conans/client/generators/cmake_common.py +++ b/conans/client/generators/cmake_common.py @@ -93,16 +93,24 @@ def cmake_global_vars(deps, build_type=""): # Property INTERFACE_LINK_FLAGS do not work, necessary to add to INTERFACE_LINK_LIBRARIES set_property(TARGET {name} PROPERTY INTERFACE_LINK_LIBRARIES ${{CONAN_FULLPATH_LIBS_{uname}}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_LIST}} $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_FULLPATH_LIBS_{uname}_RELEASE}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_RELEASE_LIST}}> $<$:${{CONAN_FULLPATH_LIBS_{uname}_DEBUG}} ${{CONAN_SHARED_LINKER_FLAGS_{uname}_DEBUG_LIST}} ${{CONAN_EXE_LINKER_FLAGS_{uname}_DEBUG_LIST}}> {deps}) set_property(TARGET {name} PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${{CONAN_INCLUDE_DIRS_{uname}}} $<$:${{CONAN_INCLUDE_DIRS_{uname}_RELEASE}}> + $<$:${{CONAN_INCLUDE_DIRS_{uname}_RELEASE}}> + $<$:${{CONAN_INCLUDE_DIRS_{uname}_RELEASE}}> $<$:${{CONAN_INCLUDE_DIRS_{uname}_DEBUG}}>) set_property(TARGET {name} PROPERTY INTERFACE_COMPILE_DEFINITIONS ${{CONAN_COMPILE_DEFINITIONS_{uname}}} $<$:${{CONAN_COMPILE_DEFINITIONS_{uname}_RELEASE}}> + $<$:${{CONAN_COMPILE_DEFINITIONS_{uname}_RELEASE}}> + $<$:${{CONAN_COMPILE_DEFINITIONS_{uname}_RELEASE}}> $<$:${{CONAN_COMPILE_DEFINITIONS_{uname}_DEBUG}}>) set_property(TARGET {name} PROPERTY INTERFACE_COMPILE_OPTIONS ${{CONAN_C_FLAGS_{uname}_LIST}} ${{CONAN_CXX_FLAGS_{uname}_LIST}} $<$:${{CONAN_C_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_CXX_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_C_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_CXX_FLAGS_{uname}_RELEASE_LIST}}> + $<$:${{CONAN_C_FLAGS_{uname}_RELEASE_LIST}} ${{CONAN_CXX_FLAGS_{uname}_RELEASE_LIST}}> $<$:${{CONAN_C_FLAGS_{uname}_DEBUG_LIST}} ${{CONAN_CXX_FLAGS_{uname}_DEBUG_LIST}}>) """ @@ -134,16 +142,17 @@ def generate_targets_section(dependencies): function(conan_find_libraries_abs_path libraries package_libdir libraries_abs_path) foreach(_LIBRARY_NAME ${libraries}) - unset(FOUND_LIBRARY CACHE) - find_library(FOUND_LIBRARY NAME ${_LIBRARY_NAME} PATHS ${package_libdir} NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) - if(FOUND_LIBRARY) - message(STATUS "Library ${_LIBRARY_NAME} found ${FOUND_LIBRARY}") - set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${FOUND_LIBRARY}) + unset(CONAN_FOUND_LIBRARY CACHE) + find_library(CONAN_FOUND_LIBRARY NAME ${_LIBRARY_NAME} PATHS ${package_libdir} NO_DEFAULT_PATH NO_CMAKE_FIND_ROOT_PATH) + if(CONAN_FOUND_LIBRARY) + message(STATUS "Library ${_LIBRARY_NAME} found ${CONAN_FOUND_LIBRARY}") + set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${CONAN_FOUND_LIBRARY}) else() message(STATUS "Library ${_LIBRARY_NAME} not found in package, might be system one") set(CONAN_FULLPATH_LIBS ${CONAN_FULLPATH_LIBS} ${_LIBRARY_NAME}) endif() endforeach() + unset(CONAN_FOUND_LIBRARY CACHE) set(${libraries_abs_path} ${CONAN_FULLPATH_LIBS} PARENT_SCOPE) endfunction() @@ -227,13 +236,13 @@ def generate_targets_section(dependencies): string(REGEX MATCH "compiler=([-A-Za-z0-9_ ]+)" _MATCHED ${CONANINFO}) if(DEFINED CMAKE_MATCH_1) - string(STRIP ${CMAKE_MATCH_1} _CONAN_INFO_COMPILER) + string(STRIP "${CMAKE_MATCH_1}" _CONAN_INFO_COMPILER) set(${CONAN_INFO_COMPILER} ${_CONAN_INFO_COMPILER} PARENT_SCOPE) endif() string(REGEX MATCH "compiler.version=([-A-Za-z0-9_.]+)" _MATCHED ${CONANINFO}) if(DEFINED CMAKE_MATCH_1) - string(STRIP ${CMAKE_MATCH_1} _CONAN_INFO_COMPILER_VERSION) + string(STRIP "${CMAKE_MATCH_1}" _CONAN_INFO_COMPILER_VERSION) set(${CONAN_INFO_COMPILER_VERSION} ${_CONAN_INFO_COMPILER_VERSION} PARENT_SCOPE) endif() endfunction() @@ -290,6 +299,7 @@ def generate_targets_section(dependencies): endif() # Avoid checks when cross compiling, apple-clang crashes because its APPLE but not apple-clang + # Actually CMake is detecting "clang" when you are using apple-clang, only if CMP0025 is set to NEW will detect apple-clang if( (CONAN_COMPILER STREQUAL "Visual Studio" AND NOT CMAKE_CXX_COMPILER_ID MATCHES MSVC) OR (CONAN_COMPILER STREQUAL "gcc" AND NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU") OR (CONAN_COMPILER STREQUAL "apple-clang" AND NOT CROSS_BUILDING AND (NOT APPLE OR NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")) OR @@ -318,10 +328,14 @@ def generate_targets_section(dependencies): if(CONAN_SYSTEM_INCLUDES) include_directories(SYSTEM ${CONAN_INCLUDE_DIRS} "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" + "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" + "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" "$<$:${CONAN_INCLUDE_DIRS_DEBUG}>") else() include_directories(${CONAN_INCLUDE_DIRS} "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" + "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" + "$<$:${CONAN_INCLUDE_DIRS_RELEASE}>" "$<$:${CONAN_INCLUDE_DIRS_DEBUG}>") endif() diff --git a/conans/client/generators/scons.py b/conans/client/generators/scons.py index dce1b1a600b..91417680aee 100644 --- a/conans/client/generators/scons.py +++ b/conans/client/generators/scons.py @@ -25,11 +25,19 @@ def content(self): all_flags = template.format(dep="conan", info=self.deps_build_info) sections.append(all_flags) + for config, cpp_info in self.deps_build_info.configs.items(): + all_flags = template.format(dep="conan:" + config, info=cpp_info) + sections.append(all_flags) + for dep_name, info in self.deps_build_info.dependencies: dep_name = dep_name.replace("-", "_") dep_flags = template.format(dep=dep_name, info=info) sections.append(dep_flags) + for config, cpp_info in info.configs.items(): + all_flags = template.format(dep=dep_name + ":" + config, info=cpp_info) + sections.append(all_flags) + sections.append("}\n") sections.append("Return('conan')\n") diff --git a/conans/client/importer.py b/conans/client/importer.py index c979d25e55b..bf6767a44f4 100644 --- a/conans/client/importer.py +++ b/conans/client/importer.py @@ -49,11 +49,12 @@ def undo_imports(current_path, output): def run_imports(conanfile, dest_folder, output): - file_importer = FileImporter(conanfile, dest_folder) + file_importer = _FileImporter(conanfile, dest_folder) conanfile.copy = file_importer + conanfile.imports_folder = dest_folder with environment_append(conanfile.env): conanfile.imports() - copied_files = file_importer.execute() + copied_files = file_importer.copied_files import_output = ScopedOutput("%s imports()" % output.scope, output) report_copied_files(copied_files, import_output) if copied_files: @@ -67,7 +68,7 @@ def run_imports(conanfile, dest_folder, output): return copied_files -class FileImporter(object): +class _FileImporter(object): """ manages the copy of files, resources, libs from the local store to the user space. E.g.: shared libs, dlls, they will be in the package folder of your configuration in the store. But you dont want to add every package to the @@ -80,10 +81,11 @@ class FileImporter(object): def __init__(self, conanfile, dst_folder): self._conanfile = conanfile self._dst_folder = dst_folder - self._copies = [] + self.copied_files = set() - def __call__(self, pattern, dst="", src="", root_package=None): - """ FileImporter is lazy, it just store requested copies, and execute them later + def __call__(self, pattern, dst="", src="", root_package=None, folder=False, + ignore_case=False, excludes=None): + """ param pattern: an fnmatch file pattern of the files that should be copied. Eg. *.dll param dst: the destination local folder, wrt to current conanfile dir, to which the files will be copied. Eg: "bin" @@ -92,36 +94,24 @@ def __call__(self, pattern, dst="", src="", root_package=None): param root_package: fnmatch pattern of the package name ("OpenCV", "Boost") from which files will be copied. Default: all packages in deps """ - self._copies.append((pattern, dst, src, root_package)) + if os.path.isabs(dst): + real_dst_folder = dst + else: + real_dst_folder = os.path.normpath(os.path.join(self._dst_folder, dst)) + + matching_paths = self._get_folders(root_package) + for name, matching_path in matching_paths.items(): + final_dst_path = os.path.join(real_dst_folder, name) if folder else real_dst_folder + file_copier = FileCopier(matching_path, final_dst_path) + files = file_copier(pattern, src=src, links=True, ignore_case=ignore_case, + excludes=excludes) + self.copied_files.update(files) def _get_folders(self, pattern): """ given the current deps graph, compute a dict {name: store-path} of each dependency """ - package_folders = [] if not pattern: - for name, deps_info in self._conanfile.deps_cpp_info.dependencies: - package_folders.append(deps_info.rootpath) - else: - for name, deps_info in self._conanfile.deps_cpp_info.dependencies: - if fnmatch.fnmatch(name, pattern): - package_folders.append(deps_info.rootpath) - return package_folders - - def execute(self): - """ Execute the stored requested copies, using a FileCopier as helper - return: set of copied files - """ - copied_files = set() - for pattern, dst_folder, src_folder, conan_name_pattern in self._copies: - if os.path.isabs(dst_folder): - real_dst_folder = dst_folder - else: - real_dst_folder = os.path.normpath(os.path.join(self._dst_folder, dst_folder)) - - matching_paths = self._get_folders(conan_name_pattern) - for matching_path in matching_paths: - file_copier = FileCopier(matching_path, real_dst_folder) - files = file_copier(pattern, src=src_folder, links=True) - copied_files.update(files) - return copied_files + return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies} + return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies + if fnmatch.fnmatch(pkg, pattern)} diff --git a/conans/client/installer.py b/conans/client/installer.py index 74aa6494f5f..58e6c2fc355 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -8,7 +8,7 @@ from conans.util.files import save, rmdir, mkdir from conans.model.ref import PackageReference from conans.util.log import logger -from conans.errors import ConanException, conanfile_exception_formatter +from conans.errors import ConanException, conanfile_exception_formatter, ConanExceptionInUserConanfileMethod from conans.client.packager import create_package from conans.client.generators import write_generators, TXTGenerator from conans.model.build_info import CppInfo @@ -72,7 +72,7 @@ def __init__(self, params, output): elif param == "never": never = True else: - self.patterns.append("%s*" % param) + self.patterns.append("%s" % param) if never and (self.outdated or self.missing or self.patterns): raise ConanException("--build=never not compatible with other options") @@ -82,12 +82,12 @@ def forced(self, reference, conanfile): if self.all: return True - ref = str(reference) if conanfile.build_policy_always: - out = ScopedOutput(ref, self._out) + out = ScopedOutput(str(reference), self._out) out.info("Building package from source as defined by build_policy='always'") return True + ref = reference.name # Patterns to match, if package matches pattern, build is forced force_build = any([fnmatch.fnmatch(ref, pattern) for pattern in self.patterns]) return force_build @@ -306,7 +306,7 @@ def _get_nodes(self, nodes_by_level, skip_nodes, build_mode): # A check to be sure that if introduced a pattern, something is going to be built if build_mode.patterns: - to_build = [str(n[0]) for n in nodes_to_build if n[3]] + to_build = [str(n[0].name) for n in nodes_to_build if n[3]] build_mode.check_matches(to_build) return nodes_to_build @@ -492,7 +492,10 @@ def check_max_path_len(src, files): self._out.writeln("") output.error("Package '%s' build failed" % conan_file.info.package_id()) output.warn("Build folder %s" % build_folder) - raise exc + if isinstance(exc, ConanExceptionInUserConanfileMethod): + raise exc + raise ConanException(exc) + finally: conan_file._conanfile_directory = export_folder # Now remove all files that were imported with imports() diff --git a/conans/client/linter.py b/conans/client/linter.py index af3bc5fe425..50f94a8a35d 100644 --- a/conans/client/linter.py +++ b/conans/client/linter.py @@ -27,6 +27,9 @@ def conan_linter(conanfile_path, out): if msgs: out.writeln("Linter warnings\n WARN: %s" % "\n WARN: ".join(msgs), front=Color.MAGENTA) + pylint_werr = os.environ.get("CONAN_PYLINT_WERR", None) + if pylint_werr and (py3_msgs or msgs): + raise ConanException("Package recipe has linter errors. Please fix them.") finally: sys.path.pop() @@ -82,9 +85,23 @@ def _normal_linter(conanfile_path): output_json = _runner(args) + dynamic_fields = "source_folder", "build_folder", "info_build" + + def _accept_message(msg): + symbol = msg.get("symbol") + text = msg.get("message") + if symbol == "no-member": + for field in dynamic_fields: + if field in text: + return False + if symbol == "not-callable" and "self.copy is not callable" == text: + return False + return True + result = [] for msg in output_json: if msg.get("type") in ("warning", "error"): - if msg.get("message") != "self.copy is not callable": + if _accept_message(msg): result.append("Linter. Line %s: %s" % (msg.get("line"), msg.get("message"))) + return result diff --git a/conans/client/loader.py b/conans/client/loader.py index f08b09410f0..19f7b7fc6ed 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -1,16 +1,16 @@ import os +from conans.client.loader_parse import ConanFileTextLoader, load_conanfile_class +from conans.client.profile_loader import read_conaninfo_profile from conans.errors import ConanException, NotFoundException +from conans.model.build_info import DepsCppInfo from conans.model.conan_file import ConanFile from conans.model.options import OptionsValues from conans.model.ref import ConanFileReference from conans.model.settings import Settings from conans.model.values import Values -from conans.util.files import load from conans.paths import BUILD_INFO -from conans.model.build_info import DepsCppInfo -from conans.model.profile import Profile -from conans.client.loader_parse import ConanFileTextLoader, load_conanfile_class +from conans.util.files import load def _load_info_file(current_path, conanfile, output, error): @@ -34,7 +34,7 @@ def _load_info_file(current_path, conanfile, output, error): def load_consumer_conanfile(conanfile_path, current_path, settings, runner, output, reference=None, error=False): - profile = Profile.read_conaninfo(current_path) + profile = read_conaninfo_profile(current_path) loader = ConanFileLoader(runner, settings, profile) if conanfile_path.endswith(".py"): consumer = not reference @@ -91,6 +91,7 @@ def load_conan(self, conanfile_path, output, consumer=False, reference=None): if consumer: self._user_options.descope_options(result.name) result.options.initialize_upstream(self._user_options) + self._user_options.clear_unscoped_options() # If this is the consumer project, it has no name result.scope = self._scopes.package_scope() else: @@ -132,8 +133,7 @@ def _parse_conan_txt(self, contents, path, output): conanfile.options.initialize_upstream(self._user_options) # imports method - conanfile.imports = ConanFileTextLoader.imports_method(conanfile, - parser.import_parameters) + conanfile.imports = parser.imports_method(conanfile) conanfile.scope = self._scopes.package_scope() conanfile._env_values.update(self._env_values) return conanfile diff --git a/conans/client/loader_parse.py b/conans/client/loader_parse.py index 276eca60bdb..fc7f1cb8cef 100644 --- a/conans/client/loader_parse.py +++ b/conans/client/loader_parse.py @@ -91,7 +91,7 @@ def _parse_file(conan_file_path): class ConanFileTextLoader(object): - """Parse a plain requirements file""" + """Parse a conanfile.txt file""" def __init__(self, input_text): # Prefer composition over inheritance, the __getattr__ was breaking things @@ -110,32 +110,67 @@ def options(self): return self._config_parser.options @property - def import_parameters(self): + def _import_parameters(self): + def _parse_args(param_string): + root_package, ignore_case, folder, excludes = None, False, False, None + params = param_string.split(",") + params = [p.split("=") for p in params if p] + for (var, value) in params: + var = var.strip() + value = value.strip() + if var == "root_package": + root_package = value + elif var == "ignore_case": + ignore_case = (value.lower() == "true") + elif var == "folder": + folder = (value.lower() == "true") + elif var == "excludes": + excludes = value.split() + else: + raise Exception("Invalid imports. Unknown argument %s" % var) + return root_package, ignore_case, folder, excludes + + def _parse_import(line): + pair = line.split("->") + source = pair[0].strip().split(',', 1) + dest = pair[1].strip() + src, pattern = source[0].strip(), source[1].strip() + return pattern, dest, src + ret = [] local_install_text = self._config_parser.imports - for local_install_line in local_install_text.splitlines(): - invalid_line_msg = "Invalid imports line: %s" \ - "\nEX: OpenCV/lib, * -> ./lib" % local_install_line + for line in local_install_text.splitlines(): + # discard blanks, comments, and discard trailing comments + line = line.strip() + if not line or line.startswith("#"): + continue + line = line.split("#", 1)[0] + + invalid_line_msg = "Invalid imports line: %s\nEX: OpenCV/lib, * -> ./lib" % line + if line.startswith("/") or line.startswith(".."): + raise ConanException("%s\n%s" % (invalid_line_msg, + "Import's paths can't begin with '/' or '..'")) try: - if local_install_line.startswith("/") or local_install_line.startswith(".."): - raise ConanException("Import's paths can't begin with '/' or '..'") - pair = local_install_line.split("->") - source = pair[0].strip().split(',', 1) - dest = pair[1].strip() - src, pattern = source[0].strip(), source[1].strip() - ret.append((pattern, dest, src)) - except ConanException as excp: - raise ConanException("%s\n%s" % (invalid_line_msg, excp.message)) - except: - raise ConanException(invalid_line_msg) + tokens = line.split("@", 1) + if len(tokens) > 1: + line = tokens[0] + params = tokens[1] + else: + params = "" + root_package, ignore_case, folder, excludes = _parse_args(params) + pattern, dest, src = _parse_import(line) + ret.append((pattern, dest, src, root_package, folder, ignore_case, excludes)) + except Exception as e: + raise ConanException("%s\n%s" % (invalid_line_msg, str(e))) return ret @property def generators(self): return self._config_parser.generators.splitlines() - @staticmethod - def imports_method(conan_file, parameters): + def imports_method(self, conan_file): + parameters = self._import_parameters + def imports(): for import_params in parameters: conan_file.copy(*import_params) diff --git a/conans/client/manager.py b/conans/client/manager.py index 430b5c963cb..33bfa55f9b5 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -1,3 +1,4 @@ +import json import os import time import shutil @@ -65,7 +66,11 @@ def export(self, user, conan_file_path, keep_source=False, filename=None): user_name, channel = user, "testing" src_folder = conan_file_path - conan_file_path = os.path.join(conan_file_path, filename or CONANFILE) + conanfile_name = filename or CONANFILE + conan_file_path = os.path.join(conan_file_path, conanfile_name) + if ((os.path.exists(conan_file_path) and conanfile_name not in os.listdir(src_folder)) or + (conanfile_name != "conanfile.py" and conanfile_name.lower() == "conanfile.py")): + raise ConanException("Wrong '%s' case" % conanfile_name) conan_linter(conan_file_path, self._user_io.out) conanfile = load_export_conanfile(conan_file_path, self._user_io.out) conan_ref = ConanFileReference(conanfile.name, conanfile.version, user_name, channel) @@ -112,6 +117,8 @@ def package_files(self, reference, package_folder, profile, force): raise ConanException("Package already exists. " "Please use --force, -f to overwrite it") shutil.copytree(package_folder, dest_package_folder) + recipe_hash = self._client_cache.load_manifest(reference).summary_hash + conanfile.info.recipe_hash = recipe_hash save(os.path.join(dest_package_folder, CONANINFO), conanfile.info.dumps()) # Create the digest for the package digest = FileTreeManifest.create(dest_package_folder) @@ -162,10 +169,40 @@ def _get_graph_builder(self, loader, update, remote_proxy): graph_builder = DepsGraphBuilder(remote_proxy, self._user_io.out, loader, resolver) return graph_builder - def info(self, reference, current_path, profile, remote=None, - info=None, filename=None, check_updates=False, - build_order=None, build_modes=None, graph_filename=None, package_filter=None, - show_paths=False): + def _get_deps_graph(self, reference, profile, filename, current_path, remote_proxy): + loader = ConanFileLoader(self._runner, self._client_cache.settings, profile) + conanfile = self._get_conanfile_object(loader, reference, filename, current_path) + graph_builder = self._get_graph_builder(loader, False, remote_proxy) + deps_graph = graph_builder.load(conanfile) + return deps_graph, graph_builder, conanfile + + def info_build_order(self, reference, profile, filename, build_order, remote, check_updates, cwd): + remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, + update=False, check_updates=check_updates) + deps_graph, _, _ = self._get_deps_graph(reference, profile, filename, cwd, remote_proxy) + result = deps_graph.build_order(build_order) + return result + + def info_nodes_to_build(self, reference, profile, filename, build_modes, remote, check_updates, cwd): + remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, + update=False, check_updates=check_updates) + deps_graph, _, conanfile = self._get_deps_graph(reference, profile, filename, cwd, remote_proxy) + installer = ConanInstaller(self._client_cache, self._user_io.out, remote_proxy, None) + build_mode = BuildMode(build_modes, self._user_io.out) + nodes = installer.nodes_to_build(deps_graph, build_mode) + counter = Counter(ref.conan.name for ref, _ in nodes) + ret = [ref if counter[ref.conan.name] > 1 else str(ref.conan) for ref, _ in nodes] + return ret, self._get_project_reference(reference, conanfile) + + def _get_project_reference(self, reference, conanfile): + if isinstance(reference, ConanFileReference): + project_reference = None + else: + project_reference = str(conanfile) + + return project_reference + + def info_get_graph(self, reference, current_path, profile, remote=None, filename=None, check_updates=False): """ Fetch and build all dependencies for the given reference @param reference: ConanFileReference or path to user space conanfile @param current_path: where the output files will be saved @@ -173,65 +210,20 @@ def info(self, reference, current_path, profile, remote=None, @param profile: Profile object with both the -s introduced options and profile readed values @param build_modes: List of build_modes specified @param filename: Optional filename of the conanfile - """ remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=False, check_updates=check_updates) - loader = ConanFileLoader(self._runner, self._client_cache.settings, profile) - conanfile = self._get_conanfile_object(loader, reference, filename, current_path) - graph_builder = self._get_graph_builder(loader, False, remote_proxy) - deps_graph = graph_builder.load(conanfile) - - if build_order: - result = deps_graph.build_order(build_order) - self._user_io.out.info(", ".join(str(s) for s in result)) - return - - if build_modes is not None: - installer = ConanInstaller(self._client_cache, self._user_io.out, remote_proxy, None) - build_mode = BuildMode(build_modes, self._user_io.out) - nodes = installer.nodes_to_build(deps_graph, build_mode) - counter = Counter(ref.conan.name for ref, _ in nodes) - self._user_io.out.info(", ".join((str(ref) - if counter[ref.conan.name] > 1 else str(ref.conan)) - for ref, _ in nodes)) - return + deps_graph, graph_builder, conanfile = self._get_deps_graph(reference, profile, filename, + current_path, remote_proxy) if check_updates: graph_updates_info = graph_builder.get_graph_updates_info(deps_graph) else: graph_updates_info = {} - def read_dates(deps_graph): - ret = {} - for ref, _ in sorted(deps_graph.nodes): - if ref: - manifest = self._client_cache.load_manifest(ref) - ret[ref] = manifest.time_str - return ret - - # Get project reference - project_reference = None - if isinstance(reference, ConanFileReference): - project_reference = None - else: - project_reference = str(conanfile) - - # Print results - if graph_filename: - if graph_filename.endswith(".html"): - grapher = ConanHTMLGrapher(project_reference, deps_graph) - else: - grapher = ConanGrapher(project_reference, deps_graph) - grapher.graph_file(graph_filename) - else: - registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) - Printer(self._user_io.out).print_info(deps_graph, project_reference, - info, registry, graph_updates_info, - remote, read_dates(deps_graph), - self._client_cache, package_filter, show_paths) + return deps_graph, graph_updates_info, self._get_project_reference(reference, conanfile) def install(self, reference, current_path, profile, remote=None, build_modes=None, filename=None, update=False, @@ -473,8 +465,21 @@ def upload(self, conan_reference_or_pattern, package_id=None, remote=None, all_p logger.debug("====> Time manager upload: %f" % (time.time() - t1)) - def search(self, pattern_or_reference=None, remote=None, ignorecase=True, packages_query=None): - """ Print the single information saved in conan.vars about all the packages + def _get_search_adapter(self, remote): + if remote: + remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) + adapter = remote_proxy + else: + adapter = self._search_manager + + return adapter + + def search_recipes(self, pattern, remote, ignorecase): + references = self._get_search_adapter(remote).search(pattern, ignorecase) + return references + + def search_packages(self, reference=None, remote=None, packages_query=None): + """ Return the single information saved in conan.vars about all the packages or the packages which match with a pattern Attributes: @@ -483,26 +488,20 @@ def search(self, pattern_or_reference=None, remote=None, ignorecase=True, packag packages_pattern = String query with binary packages properties: "arch=x86 AND os=Windows" """ - printer = Printer(self._user_io.out) - + packages_props = self._get_search_adapter(remote).search_packages(reference, packages_query) + ordered_packages = OrderedDict(sorted(packages_props.items())) if remote: - remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, - remote) - adapter = remote_proxy + remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) + remote = remote_proxy.registry.remote(remote) + manifest = self._remote_manager.get_conan_digest(reference, remote) + recipe_hash = manifest.summary_hash else: - adapter = self._search_manager - if isinstance(pattern_or_reference, ConanFileReference): - packages_props = adapter.search_packages(pattern_or_reference, packages_query) - ordered_packages = OrderedDict(sorted(packages_props.items())) try: - recipe_hash = self._client_cache.load_manifest(pattern_or_reference).summary_hash + recipe_hash = self._client_cache.load_manifest(reference).summary_hash except IOError: # It could not exist in local recipe_hash = None - printer.print_search_packages(ordered_packages, pattern_or_reference, - recipe_hash, packages_query) - else: - references = adapter.search(pattern_or_reference, ignorecase) - printer.print_search_recipes(references, pattern_or_reference) + + return ordered_packages, reference, recipe_hash, packages_query def copy(self, reference, package_ids, username, channel, force=False): """ Copy or move conanfile (exported) and packages to another user and or channel diff --git a/conans/client/new.py b/conans/client/new.py index 9363435c11e..0d4467cc437 100644 --- a/conans/client/new.py +++ b/conans/client/new.py @@ -1,5 +1,7 @@ -from conans.errors import ConanException import re +from conans.errors import ConanException +from conans.model.ref import ConanFileReference +from conans.client.new_ci import ci_get_files conanfile = """from conans import ConanFile, CMake, tools @@ -36,6 +38,7 @@ def package(self): self.copy("*hello.lib", dst="lib", keep_path=False) self.copy("*.dll", dst="bin", keep_path=False) self.copy("*.so", dst="lib", keep_path=False) + self.copy("*.dylib", dst="lib", keep_path=False) self.copy("*.a", dst="lib", keep_path=False) def package_info(self): @@ -204,8 +207,16 @@ def test(self): add_library(hello hello.cpp) """ +gitignore_template = """ +*.pyc +test_package/build + +""" -def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False, bare=False): + +def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False, bare=False, + visual_versions=None, linux_gcc_versions=None, linux_clang_versions=None, osx_clang_versions=None, + shared=None, upload_url=None, gitignore=None): try: tokens = ref.split("@") name, version = tokens[0].split("/") @@ -220,12 +231,15 @@ def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False raise ConanException("Bad parameter, please use full package name," "e.g: MyLib/1.2.3@user/testing") + # Validate it is a valid reference + ConanFileReference(name, version, user, channel) + if header and exports_sources: - raise ConanException("--header and --sources are incompatible options") + raise ConanException("'header' and 'sources' are incompatible options") if pure_c and (header or exports_sources): - raise ConanException("--pure_c is incompatible with --header and --sources") + raise ConanException("'pure_c' is incompatible with 'header' and 'sources'") if bare and (header or exports_sources): - raise ConanException("--bare is incompatible with --header and --sources") + raise ConanException("'bare' is incompatible with 'header' and 'sources'") if header: files = {"conanfile.py": conanfile_header.format(name=name, version=version, @@ -253,4 +267,9 @@ def get_files(ref, header=False, pure_c=False, test=False, exports_sources=False files["test_package/CMakeLists.txt"] = test_cmake files["test_package/example.cpp"] = test_main + if gitignore: + files[".gitignore"] = gitignore_template + + files.update(ci_get_files(name, version, user, channel, visual_versions, + linux_gcc_versions, linux_clang_versions, osx_clang_versions, shared, upload_url)) return files diff --git a/conans/client/new_ci.py b/conans/client/new_ci.py new file mode 100644 index 00000000000..bc11cff9fb9 --- /dev/null +++ b/conans/client/new_ci.py @@ -0,0 +1,206 @@ +from conans.errors import ConanException + +travis = """ +env: + global: + - CONAN_REFERENCE: "{name}/{version}" + - CONAN_USERNAME: "{user}" + - CONAN_LOGIN_USERNAME: "{user}" + - CONAN_CHANNEL: "{channel}" + {upload} +linux: &linux + os: linux + sudo: required + language: python + python: "3.6" + services: + - docker +osx: &osx + os: osx + language: generic +matrix: + include: +{configs} +install: + - chmod +x .travis/install.sh + - ./.travis/install.sh + +script: + - chmod +x .travis/run.sh + - ./.travis/run.sh +""" + +linux_config = """ + - <<: *linux""" + + +linux_config_gcc = linux_config + """ + env: CONAN_GCC_VERSIONS={version} CONAN_DOCKER_IMAGE=lasote/conangcc{name} +""" + +linux_config_clang = linux_config + """ + env: CONAN_CLANG_VERSIONS={version} CONAN_DOCKER_IMAGE=lasote/conanclang{name} +""" + +osx_config = """ + - <<: *osx + osx_image: xcode{xcode} + env: CONAN_APPLE_CLANG_VERSIONS={version} +""" + +build_py = """from conan.packager import ConanMultiPackager + + +if __name__ == "__main__": + builder = ConanMultiPackager(username="{user}", channel="{channel}") + builder.add_common_builds({shared}) + builder.run() +""" + +travis_install = """#!/bin/bash + +set -e +set -x + +if [[ "$(uname -s)" == 'Darwin' ]]; then + brew update || brew update + brew outdated pyenv || brew upgrade pyenv + brew install pyenv-virtualenv + brew install cmake || true + + if which pyenv > /dev/null; then + eval "$(pyenv init -)" + fi + + pyenv install 2.7.10 + pyenv virtualenv 2.7.10 conan + pyenv rehash + pyenv activate conan +fi + +pip install conan --upgrade +pip install conan_package_tools + +conan user +""" + + +travis_run = """#!/bin/bash + +set -e +set -x + +if [[ "$(uname -s)" == 'Darwin' ]]; then + if which pyenv > /dev/null; then + eval "$(pyenv init -)" + fi + pyenv activate conan +fi + +python build.py +""" + +appveyor = r"""build: false + +environment: + PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7.8" + PYTHON_ARCH: "32" + + CONAN_REFERENCE: "{name}/{version}" + CONAN_USERNAME: "{user}" + CONAN_LOGIN_USERNAME: "{user}" + CONAN_CHANNEL: "{channel}" + VS150COMNTOOLS: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\Common7\\Tools\\" + {upload} + matrix: +{configs} + +install: + - set PATH=%PATH%;%PYTHON%/Scripts/ + - pip.exe install conan --upgrade + - pip.exe install conan_package_tools + - conan user # It creates the conan data directory + +test_script: + - python build.py +""" + + +def get_build_py(name, user, channel, shared): + shared = 'shared_option_name="{}:shared"'.format(name) if shared else "" + return build_py.format(name=name, user=user, channel=channel, shared=shared) + + +def get_travis(name, version, user, channel, linux_gcc_versions, linux_clang_versions, osx_clang_versions, upload_url): + config = [] + + if linux_gcc_versions: + for gcc in linux_gcc_versions: + config.append(linux_config_gcc.format(version=gcc, name=gcc.replace(".", ""))) + + if linux_clang_versions: + for clang in linux_clang_versions: + config.append(linux_config_clang.format(version=clang, name=clang.replace(".", ""))) + + xcode_map = {"8.1": "8.3", + "8.0": "8.2", + "7.3": "7.3"} + for apple_clang in osx_clang_versions: + xcode = xcode_map[apple_clang] + config.append(osx_config.format(xcode=xcode, version=apple_clang)) + + configs = "".join(config) + upload = ('- CONAN_UPLOAD: "%s"\n' % upload_url) if upload_url else "" + files = {".travis.yml": travis.format(name=name, version=version, user=user, channel=channel, + configs=configs, upload=upload), + ".travis/install.sh": travis_install, + ".travis/run.sh": travis_run} + return files + + +def get_appveyor(name, version, user, channel, visual_versions, upload_url): + config = [] + visual_config = """ - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio {image} + CONAN_VISUAL_VERSIONS: {version} +""" + for visual_version in visual_versions: + image = "2017" if visual_version == "15" else "2015" + config.append(visual_config.format(image=image, version=visual_version)) + + configs = "".join(config) + upload = ('CONAN_UPLOAD: "%s"\n' % upload_url) if upload_url else "" + files = {"appveyor.yml": appveyor.format(name=name, version=version, user=user, + channel=channel, configs=configs, upload=upload)} + return files + + +def ci_get_files(name, version, user, channel, visual_versions, linux_gcc_versions, linux_clang_versions, + osx_clang_versions, shared, upload_url): + if shared and not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions): + raise ConanException("Trying to specify 'shared' in CI, but no CI system specified") + if not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions): + return {} + if visual_versions is True: + visual_versions = ["12", "14", "15"] + if linux_gcc_versions is True: + linux_gcc_versions = ["4.9", "5.4", "6.3"] + if linux_clang_versions is True: + linux_clang_versions = ["3.9", "4.0"] + if osx_clang_versions is True: + osx_clang_versions = ["7.3", "8.0", "8.1"] + if not visual_versions: + visual_versions = [] + if not linux_gcc_versions: + linux_gcc_versions = [] + if not osx_clang_versions: + osx_clang_versions = [] + build_py = get_build_py(name, user, channel, shared) + files = {"build.py": build_py} + if linux_gcc_versions or osx_clang_versions or linux_clang_versions: + files.update(get_travis(name, version, user, channel, linux_gcc_versions, linux_clang_versions, + osx_clang_versions, upload_url)) + if visual_versions: + files.update(get_appveyor(name, version, user, channel, visual_versions, upload_url)) + + return files diff --git a/conans/client/packager.py b/conans/client/packager.py index b7f3d0da6ca..d96b5aefa86 100644 --- a/conans/client/packager.py +++ b/conans/client/packager.py @@ -26,6 +26,7 @@ def new_method(pattern, src=""): conanfile.copy(pattern, dst_folder, src) return new_method + # FIXME: Deprecate these methods. Not documented. Confusing. Rely on LINTER conanfile.copy_headers = wrap(DEFAULT_INCLUDE) conanfile.copy_libs = wrap(DEFAULT_LIB) conanfile.copy_bins = wrap(DEFAULT_BIN) diff --git a/conans/client/printer.py b/conans/client/printer.py index be338439df6..0fc06eb76e2 100644 --- a/conans/client/printer.py +++ b/conans/client/printer.py @@ -157,8 +157,8 @@ def show(field): if isinstance(ref, ConanFileReference) and show("required"): # Excludes self._out.writeln(" Required by:", Color.BRIGHT_GREEN) for d in dependants: - ref = repr(d.conan_ref) if d.conan_ref else project_reference - self._out.writeln(" %s" % ref, Color.BRIGHT_YELLOW) + ref = d.conan_ref if d.conan_ref else project_reference + self._out.writeln(" %s" % str(ref), Color.BRIGHT_YELLOW) if show("requires"): depends = deps_graph.neighbors(node) @@ -167,19 +167,22 @@ def show(field): for d in depends: self._out.writeln(" %s" % repr(d.conan_ref), Color.BRIGHT_YELLOW) - def print_search_recipes(self, references, pattern): + def print_search_recipes(self, references, pattern, raw): """ Print all the exported conans information param pattern: wildcards, e.g., "opencv/*" """ - if not references: + if not references and not raw: warn_msg = "There are no packages" pattern_msg = " matching the %s pattern" % pattern self._out.info(warn_msg + pattern_msg if pattern else warn_msg) return - self._out.info("Existing package recipes:\n") - for conan_ref in sorted(references): - self._print_colored_line(str(conan_ref), indent=0) + if not raw: + self._out.info("Existing package recipes:\n") + for conan_ref in sorted(references): + self._print_colored_line(str(conan_ref), indent=0) + else: + self._out.writeln("\n".join([str(ref) for ref in references])) def print_search_packages(self, packages_props, reference, recipe_hash, packages_query): if not packages_props: diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py new file mode 100644 index 00000000000..ff693a0dd45 --- /dev/null +++ b/conans/client/profile_loader.py @@ -0,0 +1,209 @@ +import os +from collections import OrderedDict + +from conans.errors import ConanException +from conans.model.env_info import EnvValues, unquote +from conans.model.info import ConanInfo +from conans.model.options import OptionsValues +from conans.model.profile import Profile +from conans.model.ref import ConanFileReference +from conans.model.scope import Scopes +from conans.paths import CONANINFO +from conans.util.config_parser import ConfigParser +from conans.util.files import load, mkdir + + +class ProfileParser(object): + + def __init__(self, text): + self.vars = OrderedDict() # Order matters, if user declares F=1 and then FOO=12, and in profile MYVAR=$FOO, it will + self.includes = [] + self.profile_text = "" + + for counter, line in enumerate(text.splitlines()): + if not line or line.strip().startswith("#"): + continue + elif line.strip().startswith("["): + self.profile_text = "\n".join(text.splitlines()[counter:]) + break + elif line.strip().startswith("include("): + include = line.split("include(", 1)[1] + if not include.endswith(")"): + raise ConanException("Invalid include statement") + include = include[:-1] + self.includes.append(include) + else: + name, value = line.split("=", 1) + name = name.strip() + if " " in name: + raise ConanException("The names of the variables cannot contain spaces") + value = unquote(value) + self.vars[name] = value + + def apply_vars(self, repl_vars): + self.vars = self._apply_in_vars(repl_vars) + self.includes = self._apply_in_includes(repl_vars) + self.profile_text = self._apply_in_profile_text(repl_vars) + + def _apply_in_vars(self, repl_vars): + tmp_vars = OrderedDict() + for key, value in self.vars.items(): + for repl_key, repl_value in repl_vars.items(): + key = key.replace("$%s" % repl_key, repl_value) + value = value.replace("$%s" % repl_key, repl_value) + tmp_vars[key] = value + return tmp_vars + + def _apply_in_includes(self, repl_vars): + tmp_includes = [] + for include in self.includes: + for repl_key, repl_value in repl_vars.items(): + include = include.replace("$%s" % repl_key, repl_value) + tmp_includes.append(include) + return tmp_includes + + def _apply_in_profile_text(self, repl_vars): + tmp_text = self.profile_text + for repl_key, repl_value in repl_vars.items(): + tmp_text = tmp_text.replace("$%s" % repl_key, repl_value) + return tmp_text + +def read_conaninfo_profile(current_path): + profile = Profile() + conan_info_path = os.path.join(current_path, CONANINFO) + if os.path.exists(conan_info_path): + existing_info = ConanInfo.load_file(conan_info_path) + profile.settings = OrderedDict(existing_info.full_settings.as_list()) + profile.options = existing_info.full_options + profile.scopes = existing_info.scope + profile.env_values = existing_info.env_values + return profile + + +def read_profile(profile_name, cwd, default_folder): + """ Will look for "profile_name" in disk if profile_name is absolute path, + in current folder if path is relative or in the default folder otherwise. + return: a Profile object + """ + if not profile_name: + return None, None + + if os.path.isabs(profile_name): + profile_path = profile_name + folder = os.path.dirname(profile_name) + elif cwd and os.path.exists(os.path.join(cwd, profile_name)): # relative path name + profile_path = os.path.abspath(os.path.join(cwd, profile_name)) + folder = os.path.dirname(profile_path) + else: + if not os.path.exists(default_folder): + mkdir(default_folder) + profile_path = os.path.join(default_folder, profile_name) + folder = default_folder + + try: + text = load(profile_path) + except IOError: + if os.path.exists(folder): + profiles = [name for name in os.listdir(folder) if not os.path.isdir(name)] + else: + profiles = [] + current_profiles = ", ".join(profiles) or "[]" + raise ConanException("Specified profile '%s' doesn't exist.\nExisting profiles: " + "%s" % (profile_name, current_profiles)) + + try: + return _load_profile(text, profile_path, default_folder) + except ConanException as exc: + raise ConanException("Error reading '%s' profile: %s" % (profile_name, exc)) + + +def _load_profile(text, profile_path, default_folder): + """ Parse and return a Profile object from a text config like representation. + cwd is needed to be able to load the includes + """ + + try: + inherited_profile = Profile() + cwd = os.path.dirname(os.path.abspath(profile_path)) if profile_path else None + profile_parser = ProfileParser(text) + inherited_vars = profile_parser.vars + # Iterate the includes and call recursive to get the profile and variables from parent profiles + for include in profile_parser.includes: + # Recursion !! + profile, declared_vars = read_profile(include, cwd, default_folder) + inherited_profile.update(profile) + inherited_vars.update(declared_vars) + + # Apply the automatic PROFILE_DIR variable + if cwd: + inherited_vars["PROFILE_DIR"] = os.path.abspath(cwd) # Allows PYTHONPATH=$PROFILE_DIR/pythontools + + # Replace the variables from parents in the current profile + profile_parser.apply_vars(inherited_vars) + + # Current profile before update with parents (but parent variables already applied) + doc = ConfigParser(profile_parser.profile_text, + allowed_fields=["build_requires", "settings", "env", "scopes", "options"]) + + # Merge the inherited profile with the readed from current profile + _apply_inner_profile(doc, inherited_profile) + + # Return the intherited vars to apply them in the parent profile if exists + inherited_vars.update(profile_parser.vars) + return inherited_profile, inherited_vars + + except ConanException: + raise + except Exception as exc: + raise ConanException("Error parsing the profile text file: %s" % str(exc)) + + +def _apply_inner_profile(doc, base_profile): + """ + + :param doc: ConfigParser object from the current profile (excluding includes and vars, and with values already replaced) + :param base_profile: Profile inherited, it's used as a base profile to modify it. + :return: None + """ + + def get_package_name_value(item): + """Parse items like package:name=value or name=value""" + package_name = None + if ":" in item: + tmp = item.split(":", 1) + package_name, item = tmp + + name, value = item.split("=", 1) + name = name.strip() + value = unquote(value) + return package_name, name, value + + for setting in doc.settings.splitlines(): + setting = setting.strip() + if setting and not setting.startswith("#"): + if "=" not in setting: + raise ConanException("Invalid setting line '%s'" % setting) + package_name, name, value = get_package_name_value(setting) + if package_name: + base_profile.package_settings[package_name][name] = value + else: + base_profile.settings[name] = value + + if doc.build_requires: + # FIXME CHECKS OF DUPLICATED? + for req in doc.build_requires.splitlines(): + tokens = req.split(":", 1) + if len(tokens) == 1: + pattern, req_list = "*", req + else: + pattern, req_list = tokens + req_list = [ConanFileReference.loads(r.strip()) for r in req_list.split(",")] + base_profile.build_requires.setdefault(pattern, []).extend(req_list) + + if doc.scopes: + base_profile.update_scopes(Scopes.from_list(doc.scopes.splitlines())) + + if doc.options: + base_profile.options.update(OptionsValues.loads(doc.options)) + + base_profile.env_values.update(EnvValues.loads(doc.env)) diff --git a/conans/client/remote_registry.py b/conans/client/remote_registry.py index 71eacaf162c..5dca566e98a 100644 --- a/conans/client/remote_registry.py +++ b/conans/client/remote_registry.py @@ -151,12 +151,12 @@ def update_ref(self, conan_reference, remote): refs[conan_reference] = remote self._save(remotes, refs) - def add(self, remote_name, remote, verify_ssl=True): + def add(self, remote_name, remote, verify_ssl=True, insert=None): def exists_function(remotes): if remote_name in remotes: raise ConanException("Remote '%s' already exists in remotes (use update to modify)" % remote_name) - self._add_update(remote_name, remote, verify_ssl, exists_function) + self._add_update(remote_name, remote, verify_ssl, exists_function, insert) def remove(self, remote_name): with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): @@ -173,12 +173,21 @@ def exists_function(remotes): raise ConanException("Remote '%s' not found in remotes" % remote_name) self._add_update(remote_name, remote, verify_ssl, exists_function) - def _add_update(self, remote_name, remote, verify_ssl, exists_function): + def _add_update(self, remote_name, remote, verify_ssl, exists_function, insert=None): with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): remotes, refs = self._load() exists_function(remotes) urls = {r[0]: name for name, r in remotes.items() if name != remote_name} if remote in urls: raise ConanException("Remote '%s' already exists with same URL" % urls[remote]) - remotes[remote_name] = (remote, verify_ssl) + if insert is not None: + try: + insert_index = int(insert) + except: + raise ConanException("insert argument must be an integer") + remotes_list = list(remotes.items()) + remotes_list.insert(insert_index, (remote_name, (remote, verify_ssl))) + remotes = OrderedDict(remotes_list) + else: + remotes[remote_name] = (remote, verify_ssl) self._save(remotes, refs) diff --git a/conans/client/source.py b/conans/client/source.py index 2b590836059..70340e47a3d 100644 --- a/conans/client/source.py +++ b/conans/client/source.py @@ -9,7 +9,6 @@ from conans.paths import DIRTY_FILE, EXPORT_SOURCES_DIR, EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME,\ CONANFILE from conans.util.files import rmdir, save -from conans.client.export import execute_export def _merge_directories(src, dst): diff --git a/conans/model/options.py b/conans/model/options.py index f7a39c0712b..bfbc3adc37c 100644 --- a/conans/model/options.py +++ b/conans/model/options.py @@ -139,7 +139,6 @@ class OptionsValues(object): def __init__(self, values=None): self._package_values = PackageOptionValues() self._reqs_options = {} # {name("Boost": PackageOptionValues} - if not values: return @@ -177,6 +176,9 @@ def descope_options(self, name): if package_values: self._package_values.update(package_values) + def clear_unscoped_options(self): + self._package_values.clear() + def __getitem__(self, item): return self._reqs_options.setdefault(item, PackageOptionValues()) diff --git a/conans/model/profile.py b/conans/model/profile.py index 4a6c05cd1ad..66cdf5b23c6 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -1,17 +1,10 @@ from collections import OrderedDict - -from conans.model.ref import ConanFileReference -from conans.util.config_parser import ConfigParser -from conans.model.scope import Scopes, _root -from conans.errors import ConanException from collections import defaultdict -from conans.model.env_info import EnvValues, unquote + +from conans.model.env_info import EnvValues from conans.model.options import OptionsValues -import os -from conans.util.files import load, mkdir +from conans.model.scope import Scopes, _root from conans.model.values import Values -from conans.paths import CONANINFO -from conans.model.info import ConanInfo class Profile(object): @@ -38,112 +31,6 @@ def package_settings_values(self): result[pkg] = list(settings.items()) return result - @staticmethod - def read_conaninfo(current_path): - profile = Profile() - conan_info_path = os.path.join(current_path, CONANINFO) - if os.path.exists(conan_info_path): - existing_info = ConanInfo.load_file(conan_info_path) - profile.settings = OrderedDict(existing_info.full_settings.as_list()) - profile.options = existing_info.full_options - profile.scopes = existing_info.scope - profile.env_values = existing_info.env_values - return profile - - @staticmethod - def read_file(profile_name, cwd, default_folder): - """ Will look for "profile_name" in disk if profile_name is absolute path, - in current folder if path is relative or in the default folder otherwise. - return: a Profile object - """ - if not profile_name: - return None - - if os.path.isabs(profile_name): - profile_path = profile_name - folder = os.path.dirname(profile_name) - elif profile_name.startswith("."): # relative path name - profile_path = os.path.abspath(os.path.join(cwd, profile_name)) - folder = os.path.dirname(profile_path) - else: - folder = default_folder - if not os.path.exists(folder): - mkdir(folder) - profile_path = os.path.join(folder, profile_name) - - try: - text = load(profile_path) - except IOError: - if os.path.exists(folder): - profiles = [name for name in os.listdir(folder) if not os.path.isdir(name)] - else: - profiles = [] - current_profiles = ", ".join(profiles) or "[]" - raise ConanException("Specified profile '%s' doesn't exist.\nExisting profiles: " - "%s" % (profile_name, current_profiles)) - - try: - text = text.replace("$PROFILE_DIR", os.path.abspath(folder)) # Allows PYTHONPATH=$PROFILE_DIR/pythontools - return Profile.loads(text) - except ConanException as exc: - raise ConanException("Error reading '%s' profile: %s" % (profile_name, exc)) - - @staticmethod - def loads(text): - """ Parse and return a Profile object from a text config like representation - """ - def get_package_name_value(item): - """Parse items like package:name=value or name=value""" - package_name = None - if ":" in item: - tmp = item.split(":", 1) - package_name, item = tmp - - name, value = item.split("=", 1) - name = name.strip() - value = unquote(value) - return package_name, name, value - - try: - obj = Profile() - doc = ConfigParser(text, allowed_fields=["build_requires", "settings", "env", "scopes", "options"]) - - for setting in doc.settings.splitlines(): - setting = setting.strip() - if setting and not setting.startswith("#"): - if "=" not in setting: - raise ConanException("Invalid setting line '%s'" % setting) - package_name, name, value = get_package_name_value(setting) - if package_name: - obj.package_settings[package_name][name] = value - else: - obj.settings[name] = value - - if doc.build_requires: - # FIXME CHECKS OF DUPLICATED? - for req in doc.build_requires.splitlines(): - tokens = req.split(":", 1) - if len(tokens) == 1: - pattern, req_list = "*", req - else: - pattern, req_list = tokens - req_list = [ConanFileReference.loads(r.strip()) for r in req_list.split(",")] - obj.build_requires.setdefault(pattern, []).extend(req_list) - - if doc.scopes: - obj.scopes = Scopes.from_list(doc.scopes.splitlines()) - - if doc.options: - obj.options = OptionsValues.loads(doc.options) - - obj.env_values = EnvValues.loads(doc.env) - - return obj - except ConanException: - raise - except Exception as exc: - raise ConanException("Error parsing the profile text file: %s" % str(exc)) - def dumps(self): result = ["[build_requires]"] for pattern, req_list in self.build_requires.items(): @@ -178,6 +65,8 @@ def update(self, other): other.env_values.update(self.env_values) self.env_values = other.env_values self.options.update(other.options) + for pattern, req_list in other.build_requires.items(): + self.build_requires.setdefault(pattern, []).extend(req_list) def update_settings(self, new_settings): '''Mix the specified settings with the current profile. diff --git a/conans/requirements.txt b/conans/requirements.txt index fd2b4913691..719a44c4610 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -1,5 +1,5 @@ -PyJWT>=1.4.0, <1.6.0 -requests>=2.7.0, <2.14.0 +PyJWT>=1.4.0, <2.0.0 +requests>=2.7.0, <3.0.0 colorama>=0.3.3, <0.4.0 PyYAML>=3.11, <3.13.0 patch==1.16 diff --git a/conans/test/command/export_linter_test.py b/conans/test/command/export_linter_test.py index bcdbc44979b..fdc21cd2f0e 100644 --- a/conans/test/command/export_linter_test.py +++ b/conans/test/command/export_linter_test.py @@ -23,16 +23,15 @@ def test_basic(self): client = TestClient() client.save({CONANFILE: conanfile}) client.run("export lasote/stable") + self._check_linter(client.user_io.out) + + def _check_linter(self, output): if six.PY2: - self.assertIn("ERROR: Py3 incompatibility. Line 7: print statement used", - client.user_io.out) + self.assertIn("ERROR: Py3 incompatibility. Line 7: print statement used", output) self.assertIn("ERROR: Py3 incompatibility. Line 8: Calling a dict.iter*() method", - client.user_io.out) - - self.assertIn("WARN: Linter. Line 8: Unused variable 'k'", - client.user_io.out) - self.assertIn("WARN: Linter. Line 8: Unused variable 'v'", - client.user_io.out) + output) + self.assertIn("WARN: Linter. Line 8: Unused variable 'k'", output) + self.assertIn("WARN: Linter. Line 8: Unused variable 'v'", output) def test_disable_linter(self): client = TestClient() @@ -53,3 +52,36 @@ def test_custom_rc_linter(self): % os.path.join(client.current_folder, "pylintrc")) client.run("export lasote/stable") self.assertIn("Bad indentation. Found 4 spaces, expected 2", client.user_io.out) + + def test_dynamic_fields(self): + client = TestClient() + conanfile2 = """ +from conans import ConanFile +class TestConan(ConanFile): + name = "Hello" + version = "1.2" + def build(self): + self.output.info(self.source_folder) + self.output.info(self.package_folder) + self.output.info(self.build_folder) + self.output.info(self.conanfile_directory) + def package(self): + self.copy("*") + def build_id(self): + self.output.info(str(self.info_build)) +""" + client.save({CONANFILE: conanfile2}) + client.run("export lasote/stable") + self.assertNotIn("Linter", client.user_io.out) + # ensure nothing breaks + client.run("install Hello/1.2@lasote/stable --build") + + def test_warning_as_errors(self): + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("config set general.pylint_werr=True") + error = client.run("export lasote/stable", ignore_error=True) + self.assertTrue(error) + self._check_linter(client.user_io.out) + self.assertIn("ERROR: Package recipe has linter errors. Please fix them", + client.user_io.out) diff --git a/conans/test/command/export_test.py b/conans/test/command/export_test.py index ba5579f7675..ba8e096c79f 100644 --- a/conans/test/command/export_test.py +++ b/conans/test/command/export_test.py @@ -6,6 +6,7 @@ from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.model.manifest import FileTreeManifest from conans.test.utils.tools import TestClient +import platform class ExportSettingsTest(unittest.TestCase): @@ -27,6 +28,26 @@ class TestConan(ConanFile): self.assertIn("'Windows' is not a valid 'settings.os' value", client.user_io.out) self.assertIn("Possible values are ['Linux']", client.user_io.out) + def test_conanfile_case(self): + if platform.system() != "Windows": + return + client = TestClient() + conanfile = """ +from conans import ConanFile +class TestConan(ConanFile): + name = "Hello" + version = "1.2" + exports = "*" +""" + client.save({"Conanfile.py": conanfile}) + error = client.run("export lasote/stable", ignore_error=True) + self.assertTrue(error) + self.assertIn("Wrong 'conanfile.py' case", client.user_io.out) + + error = client.run("export lasote/stable -f Conanfile.py", ignore_error=True) + self.assertTrue(error) + self.assertIn("Wrong 'Conanfile.py' case", client.user_io.out) + def test_filename(self): client = TestClient() conanfile = """ @@ -37,6 +58,12 @@ class TestConan(ConanFile): """ client.save({"myconanfile.py": conanfile, "conanfile.py": ""}) + + if platform.system() == "Windows": + error = client.run("export lasote/stable -f MyConanfile.py", ignore_error=True) + self.assertTrue(error) + self.assertIn("Wrong 'MyConanfile.py' case", client.user_io.out) + client.run("export lasote/stable --file=myconanfile.py") self.assertIn("Hello/1.2@lasote/stable: A new conanfile.py version was exported", client.user_io.out) diff --git a/conans/test/command/info_test.py b/conans/test/command/info_test.py index b674f00fa35..30e50f69873 100644 --- a/conans/test/command/info_test.py +++ b/conans/test/command/info_test.py @@ -169,6 +169,27 @@ def only_names_test(self): self.assertIn("Invalid --only value", self.client.user_io.out) self.assertIn("with --path specified, allowed values:", self.client.user_io.out) + def test_cwd(self): + self.client = TestClient() + conanfile = """from conans import ConanFile +from conans.util.files import load, save + +class MyTest(ConanFile): + name = "Pkg" + version = "0.1" + settings = "build_type" + +""" + self.client.save({"subfolder/conanfile.py": conanfile}) + self.client.run("export --path ./subfolder lasote") + + self.client.run("info --cwd ./subfolder") + self.assertIn("Pkg/0.1@PROJECT", self.client.user_io.out) + + self.client.run("info --cwd ./subfolder --build_order Pkg/0.1@lasote/testing --json=jsonfile.txt") + path = os.path.join(self.client.current_folder, "subfolder", "jsonfile.txt") + self.assertTrue(os.path.exists(path)) + def reuse_test(self): self.client = TestClient() self._create("Hello0", "0.1") @@ -263,6 +284,13 @@ def build_order_test(self): self.assertEqual("[Hello0/0.1@lasote/stable], [Hello1/0.1@lasote/stable]\n", self.client.user_io.out) + self.client.run("info Hello1/0.1@lasote/stable -bo=Hello0/0.1@lasote/stable --json=file.json") + self.assertEqual('{"groups": [["Hello0/0.1@lasote/stable"], ["Hello1/0.1@lasote/stable"]]}', + load(os.path.join(self.client.current_folder, "file.json"))) + + self.client.run("info Hello1/0.1@lasote/stable -bo=Hello0/0.1@lasote/stable --json") + self.assertIn('{"groups": [["Hello0/0.1@lasote/stable"], ["Hello1/0.1@lasote/stable"]]}', self.client.user_io.out) + def diamond_build_order_test(self): self.client = TestClient() self._create("LibA", "0.1") diff --git a/conans/test/command/install_test.py b/conans/test/command/install_test.py index f28ce6b6fc7..3e570417ed8 100644 --- a/conans/test/command/install_test.py +++ b/conans/test/command/install_test.py @@ -72,11 +72,10 @@ def partials_test(self): self.client.run("install %s --build=missing" % (self.settings)) self.client.run("install %s --build=Bye" % (self.settings)) - self.assertIn("No package matching 'Bye*' pattern", self.client.user_io.out) + self.assertIn("No package matching 'Bye' pattern", self.client.user_io.out) for package in ["Hello0", "Hello1"]: - error = self.client.run("install %s --build=%s" % (self.settings, package)) - self.assertFalse(error) + self.client.run("install %s --build=%s" % (self.settings, package)) self.assertNotIn("No package matching", self.client.user_io.out) def reuse_test(self): diff --git a/conans/test/command/new_test.py b/conans/test/command/new_test.py index 25545a3aedf..c34b4c8804d 100644 --- a/conans/test/command/new_test.py +++ b/conans/test/command/new_test.py @@ -22,6 +22,19 @@ def new_test(self): client.run("info test_package") self.assertIn("MyPackage/1.3@myuser/testing", client.user_io.out) + def new_error_test(self): + """ packages with short name + """ + client = TestClient() + error = client.run('new A/1.3@myuser/testing', ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: 'A' is too short. Valid names must contain at least 2 characters.", + client.user_io.out) + error = client.run('new A2/1.3@myuser/u', ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: 'u' is too short. Valid names must contain at least 2 characters.", + client.user_io.out) + def new_dash_test(self): """ packages with dash """ @@ -93,3 +106,68 @@ def new_without_test(self): self.assertFalse(os.path.exists(os.path.join(root, "test_package/conanfile.py"))) self.assertFalse(os.path.exists(os.path.join(root, "test_package/CMakeLists.txt"))) self.assertFalse(os.path.exists(os.path.join(root, "test_package/example.cpp"))) + + def new_ci_test(self): + client = TestClient() + client.run('new MyPackage/1.3@myuser/testing -cis -ciw -cilg -cilc -cio -ciu=myurl') + root = client.current_folder + build_py = load(os.path.join(root, "build.py")) + self.assertIn('builder.add_common_builds(shared_option_name="MyPackage:shared")', + build_py) + self.assertNotIn('visual_versions=', build_py) + self.assertNotIn('gcc_versions=', build_py) + self.assertNotIn('clang_versions=', build_py) + self.assertNotIn('apple_clang_versions=', build_py) + appveyor = load(os.path.join(root, "appveyor.yml")) + self.assertIn("CONAN_UPLOAD: \"myurl\"", appveyor) + self.assertIn('CONAN_REFERENCE: "MyPackage/1.3"', appveyor) + self.assertIn('CONAN_USERNAME: "myuser"', appveyor) + self.assertIn('CONAN_CHANNEL: "testing"', appveyor) + self.assertIn('CONAN_VISUAL_VERSIONS: 12', appveyor) + self.assertIn('CONAN_VISUAL_VERSIONS: 14', appveyor) + self.assertIn('CONAN_VISUAL_VERSIONS: 15', appveyor) + + travis = load(os.path.join(root, ".travis.yml")) + self.assertIn("- CONAN_UPLOAD: \"myurl\"", travis) + self.assertIn('- CONAN_REFERENCE: "MyPackage/1.3"', travis) + self.assertIn('- CONAN_USERNAME: "myuser"', travis) + self.assertIn('- CONAN_CHANNEL: "testing"', travis) + self.assertIn('env: CONAN_GCC_VERSIONS=5.4 CONAN_DOCKER_IMAGE=lasote/conangcc54', + travis) + + def new_ci_test_partial(self): + client = TestClient() + root = client.current_folder + error = client.run('new MyPackage/1.3@myuser/testing -cis', ignore_error=True) + self.assertTrue(error) + + client.run('new MyPackage/1.3@myuser/testing -cilg') + self.assertTrue(os.path.exists(os.path.join(root, "build.py"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis.yml"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis/install.sh"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis/run.sh"))) + self.assertFalse(os.path.exists(os.path.join(root, "appveyor.yml"))) + + client = TestClient() + root = client.current_folder + client.run('new MyPackage/1.3@myuser/testing -ciw') + self.assertTrue(os.path.exists(os.path.join(root, "build.py"))) + self.assertFalse(os.path.exists(os.path.join(root, ".travis.yml"))) + self.assertFalse(os.path.exists(os.path.join(root, ".travis/install.sh"))) + self.assertFalse(os.path.exists(os.path.join(root, ".travis/run.sh"))) + self.assertTrue(os.path.exists(os.path.join(root, "appveyor.yml"))) + + client = TestClient() + root = client.current_folder + client.run('new MyPackage/1.3@myuser/testing -cio') + self.assertTrue(os.path.exists(os.path.join(root, "build.py"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis.yml"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis/install.sh"))) + self.assertTrue(os.path.exists(os.path.join(root, ".travis/run.sh"))) + self.assertFalse(os.path.exists(os.path.join(root, "appveyor.yml"))) + + client = TestClient() + root = client.current_folder + client.run('new MyPackage/1.3@myuser/testing -gi') + self.assertTrue(os.path.exists(os.path.join(root, ".gitignore"))) + diff --git a/conans/test/command/package_files_test.py b/conans/test/command/package_files_test.py index f9812e587c3..fa31e623a48 100644 --- a/conans/test/command/package_files_test.py +++ b/conans/test/command/package_files_test.py @@ -67,6 +67,10 @@ def test_new(self): self.assertIn("set(CONAN_LIBS_HELLO mycoollib)", cmakeinfo) self.assertIn("set(CONAN_LIBS mycoollib ${CONAN_LIBS})", cmakeinfo) + # ensure the recipe hash is computed and added + client.run("search Hello/0.1@lasote/stable") + self.assertIn("outdated from recipe: False", client.user_io.out) + def test_paths(self): client = TestClient() client.run("new Hello/0.1 --bare") diff --git a/conans/test/command/remote_test.py b/conans/test/command/remote_test.py index c35297d9b4f..42f48031d7a 100644 --- a/conans/test/command/remote_test.py +++ b/conans/test/command/remote_test.py @@ -24,7 +24,8 @@ def basic_test(self): self.client.run("remote add origin https://myurl") self.client.run("remote list") - self.assertIn("origin: https://myurl", self.client.user_io.out) + lines = str(self.client.user_io.out).splitlines() + self.assertIn("origin: https://myurl", lines[3]) self.client.run("remote update origin https://2myurl") self.client.run("remote list") @@ -40,6 +41,25 @@ def basic_test(self): output = str(self.client.user_io.out) self.assertIn("remote1: http://", output.splitlines()[0]) + def insert_test(self): + self.client.run("remote add origin https://myurl --insert") + self.client.run("remote list") + first_line = str(self.client.user_io.out).splitlines()[0] + self.assertIn("origin: https://myurl", first_line) + + self.client.run("remote add origin2 https://myurl2 --insert=0") + self.client.run("remote list") + lines = str(self.client.user_io.out).splitlines() + self.assertIn("origin2: https://myurl2", lines[0]) + self.assertIn("origin: https://myurl", lines[1]) + + self.client.run("remote add origin3 https://myurl3 --insert=1") + self.client.run("remote list") + lines = str(self.client.user_io.out).splitlines() + self.assertIn("origin2: https://myurl2", lines[0]) + self.assertIn("origin3: https://myurl3", lines[1]) + self.assertIn("origin: https://myurl", lines[2]) + def verify_ssl_test(self): client = TestClient() client.run("remote add my-remote http://someurl TRUE") diff --git a/conans/test/command/search_test.py b/conans/test/command/search_test.py index 7fb2d6858cc..f548a3d5fe6 100644 --- a/conans/test/command/search_test.py +++ b/conans/test/command/search_test.py @@ -163,6 +163,13 @@ def recipe_search_test(self): "NodeInfo/1.0.2@fenix/stable\n" "helloTest/1.4.10@fenix/stable\n", self.client.user_io.out) + def search_raw(self): + self.client.run("search Hello* --raw") + self.assertEquals("Hello/1.4.10@fenix/testing\n" + "Hello/1.4.11@fenix/testing\n" + "Hello/1.4.12@fenix/testing\n" + "helloTest/1.4.10@fenix/stable\n", self.client.user_io.out) + def recipe_pattern_search_test(self): self.client.run("search Hello*") self.assertEquals("Existing package recipes:\n\n" @@ -213,6 +220,7 @@ def package_search_nonescaped_characters_test(self): shutil.copytree(self.client.paths.store, self.servers["local"].paths.store) self.client.run("remove Hello* -f") self.client.run('search Hello/1.4.10@fenix/testing -q "compiler=gcc AND compiler.libcxx=libstdc++11" -r local') + self.assertIn("outdated from recipe: False", self.client.user_io.out) self.assertIn("LinuxPackageSHA", self.client.user_io.out) self.assertNotIn("PlatformIndependantSHA", self.client.user_io.out) self.assertNotIn("WindowsPackageSHA", self.client.user_io.out) diff --git a/conans/test/functional/conanfile_loader_test.py b/conans/test/functional/conanfile_loader_test.py index 5a451f692ab..9b251b874d2 100644 --- a/conans/test/functional/conanfile_loader_test.py +++ b/conans/test/functional/conanfile_loader_test.py @@ -8,9 +8,9 @@ from mock import Mock from conans.model.settings import Settings from conans.test.utils.test_files import temp_folder -from conans.model.scope import Scopes from conans.model.profile import Profile from collections import OrderedDict +from mock.mock import call class ConanLoaderTest(unittest.TestCase): @@ -43,6 +43,14 @@ def conanfile_txt_errors_test(self): with self.assertRaisesRegexp(ConanException, "Unexpected line"): ConanFileTextLoader(file_content) + file_content = '[imports]\nhello' + with self.assertRaisesRegexp(ConanException, "Invalid imports line: hello"): + ConanFileTextLoader(file_content).imports_method(None) + + file_content = '[imports]\nbin, * -> bin @ kk=3 ' + with self.assertRaisesRegexp(ConanException, "Unknown argument kk"): + ConanFileTextLoader(file_content).imports_method(None) + def plain_text_parser_test(self): # Valid content file_content = '''[requires] @@ -126,6 +134,28 @@ def load_conan_txt_test(self): with self.assertRaisesRegexp(ConanException, "is too long. Valid names must contain"): loader.load_conan_txt(file_path, None) + def load_imports_arguments_test(self): + file_content = ''' +[imports] +OpenCV/bin, * -> ./bin # I need this binaries +OpenCV/lib, * -> ./lib @ root_package=Pkg +OpenCV/data, * -> ./data @ root_package=Pkg, folder=True # Irrelevant +docs, * -> ./docs @ root_package=Pkg, folder=True, ignore_case=True, excludes="a b c" # Other +''' + tmp_dir = temp_folder() + file_path = os.path.join(tmp_dir, "file.txt") + save(file_path, file_content) + loader = ConanFileLoader(None, Settings(), Profile()) + ret = loader.load_conan_txt(file_path, None) + + ret.copy = Mock() + ret.imports() + expected = [call(u'*', u'./bin', u'OpenCV/bin', None, False, False, None), + call(u'*', u'./lib', u'OpenCV/lib', u'Pkg', False, False, None), + call(u'*', u'./data', u'OpenCV/data', u'Pkg', True, False, None), + call(u'*', u'./docs', u'docs', u'Pkg', True, True, [u'"a', u'b', u'c"'])] + self.assertEqual(ret.copy.call_args_list, expected) + def test_package_settings(self): # CREATE A CONANFILE TO LOAD tmp_dir = temp_folder() diff --git a/conans/test/functional/file_copier_test.py b/conans/test/functional/file_copier_test.py index 5188980ef3c..697b20c24f7 100644 --- a/conans/test/functional/file_copier_test.py +++ b/conans/test/functional/file_copier_test.py @@ -1,9 +1,10 @@ -import unittest import os -from conans.util.files import save, load -from conans.test.utils.test_files import temp_folder import platform +import unittest + from conans.client.file_copier import FileCopier +from conans.test.utils.test_files import temp_folder +from conans.util.files import save, load class FileCopierTest(unittest.TestCase): @@ -87,6 +88,24 @@ def linked_folder_missing_error_test(self): self.assertEqual("Hello1", load(os.path.join(folder2, "subdir1/file1.txt"))) self.assertEqual("Hello1", load(os.path.join(folder2, "subdir2/file1.txt"))) + def linked_relative_test(self): + if platform.system() != "Linux" and platform.system() != "Darwin": + return + + folder1 = temp_folder() + sub1 = os.path.join(folder1, "foo/other/file") + os.makedirs(sub1) + save(os.path.join(sub1, "file.txt"), "Hello") + sub2 = os.path.join(folder1, "foo/symlink") + os.symlink("other/file", sub2) + + folder2 = temp_folder() + copier = FileCopier(folder1, folder2) + copier("*", links=True) + symlink = os.path.join(folder2, "foo", "symlink") + self.assertTrue(os.path.islink(symlink)) + self.assertTrue(load(os.path.join(symlink, "file.txt")), "Hello") + def excludes_test(self): folder1 = temp_folder() sub1 = os.path.join(folder1, "subdir1") diff --git a/conans/test/functional/imports_tests.py b/conans/test/functional/imports_tests.py new file mode 100644 index 00000000000..5a93055e8bd --- /dev/null +++ b/conans/test/functional/imports_tests.py @@ -0,0 +1,69 @@ +import unittest +from conans.test.utils.tools import TestClient +from conans.util.files import load +import os + + +conanfile = """from conans import ConanFile + +class TestConan(ConanFile): + name = "%s" + version = "0.1" + exports = "*" + def package(self): + self.copy("*") +""" + + +class ImportTest(unittest.TestCase): + def _set_up(self): + client = TestClient() + client.save({"conanfile.py": conanfile % "LibA", + "LICENSE.txt": "LicenseA"}) + client.run("export lasote/testing") + + client.save({"conanfile.py": conanfile % "LibB" + " requires='LibA/0.1@lasote/testing'", + "LICENSE.md": "LicenseB"}, clean_first=True) + client.run("export lasote/testing") + + client.save({"conanfile.py": conanfile % "LibC" + " requires='LibB/0.1@lasote/testing'", + "license.txt": "LicenseC"}, clean_first=True) + client.run("export lasote/testing") + return client + + def imports_folders_test(self): + client = self._set_up() + + testconanfile = conanfile % "LibD" + " requires='LibC/0.1@lasote/testing'" + testconanfile += """ + def imports(self): + self.copy("license*", dst="licenses", folder=True, ignore_case=True) + import os + self.output.info("IMPORTED FOLDERS: %s " % sorted(os.listdir(self.imports_folder))) +""" + client.save({"conanfile.py": testconanfile}, clean_first=True) + client.run("install --build=missing") + self.assertIn("IMPORTED FOLDERS: [", client.user_io.out) + self.assertEqual(load(os.path.join(client.current_folder, "licenses/LibA/LICENSE.txt")), + "LicenseA") + self.assertEqual(load(os.path.join(client.current_folder, "licenses/LibB/LICENSE.md")), + "LicenseB") + self.assertEqual(load(os.path.join(client.current_folder, "licenses/LibC/license.txt")), + "LicenseC") + + def imports_folders_txt_test(self): + client = self._set_up() + + conanfile = """[requires] +LibC/0.1@lasote/testing +[imports] +., license* -> licenses @ folder=True, ignore_case=True, excludes=*.md # comment +""" + client.save({"conanfile.txt": conanfile}, clean_first=True) + client.run("install --build=missing") + self.assertEqual(load(os.path.join(client.current_folder, "licenses/LibA/LICENSE.txt")), + "LicenseA") + self.assertFalse(os.path.exists(os.path.join(client.current_folder, + "licenses/LibB/LICENSE.md"))) + self.assertEqual(load(os.path.join(client.current_folder, "licenses/LibC/license.txt")), + "LicenseC") diff --git a/conans/test/functional/profile_loader_test.py b/conans/test/functional/profile_loader_test.py new file mode 100644 index 00000000000..f626fb1a0df --- /dev/null +++ b/conans/test/functional/profile_loader_test.py @@ -0,0 +1,688 @@ +import os +import unittest +from collections import OrderedDict + +from conans.model.profile import Profile + +from conans.model.ref import ConanFileReference +from nose_parameterized import parameterized + +from conans.client.profile_loader import read_profile, ProfileParser +from conans.errors import ConanException +from conans.model.env_info import EnvValues +from conans.paths import CONANFILE +from conans.test.utils.cpp_test_files import cpp_hello_conan_files +from conans.test.utils.profiles import create_profile +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient +from conans.util.files import save, load + + +class ProfileParserTest(unittest.TestCase): + + def test_parser(self): + txt = """ +include(a/path/to\profile.txt) +VAR=2 +include(other/path/to/file.txt) +OTHERVAR=thing + +[settings] +os=2 +""" + a = ProfileParser(txt) + self.assertEquals(a.vars, {"VAR": "2", "OTHERVAR": "thing"}) + self.assertEquals(a.includes, ["a/path/to\profile.txt", "other/path/to/file.txt"]) + self.assertEquals(a.profile_text, """[settings] +os=2""") + + txt = "" + a = ProfileParser(txt) + self.assertEquals(a.vars, {}) + self.assertEquals(a.includes, []) + self.assertEquals(a.profile_text, "") + + txt = """ +include(a/path/to\profile.txt) +VAR=$REPLACE_VAR +include(other/path/to/$FILE) +OTHERVAR=thing + +[settings] +os=$OTHERVAR +""" + a = ProfileParser(txt) + a.apply_vars({"REPLACE_VAR": "22", "FILE": "MyFile", "OTHERVAR": "thing"}) + self.assertEquals(a.vars, {"VAR": "22", "OTHERVAR": "thing"}) + self.assertEquals(a.includes, ["a/path/to\profile.txt", "other/path/to/MyFile"]) + self.assertEquals(a.profile_text, """[settings] +os=thing""") + + + + +conanfile_scope_env = """ +import platform +from conans import ConanFile + +class AConan(ConanFile): + name = "Hello0" + version = "0.1" + settings = "os", "compiler", "arch" + + def build(self): + self.output.warn("Scope myscope: %s" % self.scope.myscope) + self.output.warn("Scope otherscope: %s" % self.scope.otherscope) + self.output.warn("Scope undefined: %s" % self.scope.undefined) + # Print environment vars + if self.settings.os == "Windows": + self.run("SET") + else: + self.run("env") + +""" + + +class ProfileTest(unittest.TestCase): + + def setUp(self): + self.client = TestClient() + + def profile_loads_test(self): + + tmp = temp_folder() + + prof = '''[env] + CXX_FLAGS="-DAAA=0" + [settings] + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) + + prof = '''[env] + CXX_FLAGS="-DAAA=0" + MyPackage:VAR=1 + MyPackage:OTHER=2 + OtherPackage:ONE=ONE + [settings] + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) + self.assertEquals(new_profile.env_values.env_dicts("MyPackage"), ({"OTHER": "2", + "VAR": "1", + 'CXX_FLAGS': '-DAAA=0'}, {})) + + self.assertEquals(new_profile.env_values.env_dicts("OtherPackage"), ({'CXX_FLAGS': '-DAAA=0', + 'ONE': 'ONE'}, {})) + + prof = '''[env] + CXX_FLAGS='-DAAA=0' + [settings] + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) + + prof = '''[env] + CXX_FLAGS=-DAAA=0 + [settings] + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) + + prof = '''[env] + CXX_FLAGS="-DAAA=0 + [settings] + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '"-DAAA=0'}, {})) + + prof = ''' + [settings] + zlib:compiler=gcc + compiler=Visual Studio + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEquals(new_profile.package_settings["zlib"], {"compiler": "gcc"}) + self.assertEquals(new_profile.settings["compiler"], "Visual Studio") + + def test_empty_env(self): + tmp = temp_folder() + profile, _ = self._get_profile(tmp, "") + self.assertTrue(isinstance(profile.env_values, EnvValues)) + + def profile_loads_win_test(self): + tmp = temp_folder() + prof = '''[env] + QTPATH=C:/QtCommercial/5.8/msvc2015_64/bin + QTPATH2="C:/QtCommercial2/5.8/msvc2015_64/bin" + ''' + new_profile, _ = self._get_profile(tmp, prof) + self.assertEqual(new_profile.env_values.data[None]["QTPATH"], + "C:/QtCommercial/5.8/msvc2015_64/bin") + self.assertEqual(new_profile.env_values.data[None]["QTPATH2"], + "C:/QtCommercial2/5.8/msvc2015_64/bin") + self.assertIn("QTPATH=C:/QtCommercial/5.8/msvc2015_64/bin", new_profile.dumps()) + self.assertIn("QTPATH2=C:/QtCommercial2/5.8/msvc2015_64/bin", new_profile.dumps()) + + def profile_load_dump_test(self): + + # Empty profile + tmp = temp_folder() + profile = Profile() + dump = profile.dumps() + new_profile, _ = self._get_profile(tmp, "") + self.assertEquals(new_profile.settings, profile.settings) + + # Settings + profile = Profile() + profile.settings["arch"] = "x86_64" + profile.settings["compiler"] = "Visual Studio" + profile.settings["compiler.version"] = "12" + + profile.env_values.add("CXX", "path/to/my/compiler/g++") + profile.env_values.add("CC", "path/to/my/compiler/gcc") + + profile.scopes["p1"]["conaning"] = "1" + profile.scopes["p2"]["testing"] = "2" + + profile.build_requires["*"] = ["android_toolchain/1.2.8@lasote/testing"] + profile.build_requires["zlib/*"] = ["cmake/1.0.2@lasote/stable", + "autotools/1.0.3@lasote/stable"] + + dump = profile.dumps() + new_profile, _ = self._get_profile(tmp, dump) + self.assertEquals(new_profile.settings, profile.settings) + self.assertEquals(new_profile.settings["arch"], "x86_64") + self.assertEquals(new_profile.settings["compiler.version"], "12") + self.assertEquals(new_profile.settings["compiler"], "Visual Studio") + + self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX': 'path/to/my/compiler/g++', + 'CC': 'path/to/my/compiler/gcc'}, {})) + + self.assertEquals(dict(new_profile.scopes)["p1"]["conaning"], '1') + self.assertEquals(dict(new_profile.scopes)["p2"]["testing"], '2') + + self.assertEquals(new_profile.build_requires["zlib/*"], + [ConanFileReference.loads("cmake/1.0.2@lasote/stable"), + ConanFileReference.loads("autotools/1.0.3@lasote/stable")]) + self.assertEquals(new_profile.build_requires["*"], + [ConanFileReference.loads("android_toolchain/1.2.8@lasote/testing")]) + + def bad_syntax_test(self): + self.client.save({CONANFILE: conanfile_scope_env}) + self.client.run("export lasote/stable") + + profile = ''' + [settings + ''' + clang_profile_path = os.path.join(self.client.client_cache.profiles_path, "clang") + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self.assertIn("Error reading 'clang' profile", self.client.user_io.out) + self.assertIn("Bad syntax", self.client.user_io.out) + + profile = ''' + [settings] + [invented] + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self.assertIn("Unrecognized field 'invented'", self.client.user_io.out) + self.assertIn("Error reading 'clang' profile", self.client.user_io.out) + + profile = ''' + [settings] + as + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self.assertIn("Error reading 'clang' profile: Invalid setting line 'as'", self.client.user_io.out) + + profile = ''' + [env] + as + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self.assertIn("Error reading 'clang' profile: Invalid env line 'as'", self.client.user_io.out) + + profile = ''' + [scopes] + as + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self.assertIn("Error reading 'clang' profile: Bad scope as", self.client.user_io.out) + + profile = ''' + [settings] + os = a value + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + # stripped "a value" + self.assertIn("'a value' is not a valid 'settings.os'", self.client.user_io.out) + + profile = ''' + [env] + ENV_VAR = a value + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr clang", ignore_error=True) + self._assert_env_variable_printed("ENV_VAR", "a value") + + profile = ''' + # Line with comments is not a problem + [env] + # Not even here + ENV_VAR = a value + ''' + save(clang_profile_path, profile) + self.client.run("install Hello0/0.1@lasote/stable --build -pr clang", ignore_error=True) + self._assert_env_variable_printed("ENV_VAR", "a value") + + @parameterized.expand([("", ), ("./local_profiles/", ), (temp_folder() + "/", )]) + def install_with_missing_profile_test(self, path): + self.client.save({CONANFILE: conanfile_scope_env}) + error = self.client.run('install -pr "%sscopes_env"' % path, ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: Specified profile '%sscopes_env' doesn't exist" % path, + self.client.user_io.out) + + def install_profile_env_test(self): + files = cpp_hello_conan_files("Hello0", "0.1", build=False) + files["conanfile.py"] = conanfile_scope_env + + create_profile(self.client.client_cache.profiles_path, "envs", settings={}, + env=[("A_VAR", "A_VALUE")], package_env={"Hello0": [("OTHER_VAR", "2")]}) + + self.client.save(files) + self.client.run("export lasote/stable") + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr envs") + self._assert_env_variable_printed("A_VAR", "A_VALUE") + self._assert_env_variable_printed("OTHER_VAR", "2") + + # Override with package var + self.client.run("install Hello0/0.1@lasote/stable --build -pr envs -e Hello0:A_VAR=OTHER_VALUE") + self._assert_env_variable_printed("A_VAR", "OTHER_VALUE") + self._assert_env_variable_printed("OTHER_VAR", "2") + + # Override package var with package var + self.client.run("install Hello0/0.1@lasote/stable --build -pr envs " + "-e Hello0:A_VAR=OTHER_VALUE -e Hello0:OTHER_VAR=3") + self._assert_env_variable_printed("A_VAR", "OTHER_VALUE") + self._assert_env_variable_printed("OTHER_VAR", "3") + + # Pass a variable with "=" symbol + self.client.run("install Hello0/0.1@lasote/stable --build -pr envs " + "-e Hello0:A_VAR=Valuewith=equal -e Hello0:OTHER_VAR=3") + self._assert_env_variable_printed("A_VAR", "Valuewith=equal") + self._assert_env_variable_printed("OTHER_VAR", "3") + + def install_profile_settings_test(self): + files = cpp_hello_conan_files("Hello0", "0.1", build=False) + + # Create a profile and use it + profile_settings = OrderedDict([("compiler", "Visual Studio"), + ("compiler.version", "12"), + ("compiler.runtime", "MD"), + ("arch", "x86")]) + + create_profile(self.client.client_cache.profiles_path, "vs_12_86", + settings=profile_settings, package_settings={}) + + self.client.save(files) + self.client.run("export lasote/stable") + self.client.run("install --build missing -pr vs_12_86") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + for setting, value in profile_settings.items(): + self.assertIn("%s=%s" % (setting, value), info) + + # Try to override some settings in install command + self.client.run("install --build missing -pr vs_12_86 -s compiler.version=14") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + for setting, value in profile_settings.items(): + if setting != "compiler.version": + self.assertIn("%s=%s" % (setting, value), info) + else: + self.assertIn("compiler.version=14", info) + + # Use package settings in profile + tmp_settings = OrderedDict() + tmp_settings["compiler"] = "gcc" + tmp_settings["compiler.libcxx"] = "libstdc++11" + tmp_settings["compiler.version"] = "4.8" + package_settings = {"Hello0": tmp_settings} + create_profile(self.client.client_cache.profiles_path, + "vs_12_86_Hello0_gcc", settings=profile_settings, + package_settings=package_settings) + # Try to override some settings in install command + self.client.run("install --build missing -pr vs_12_86_Hello0_gcc -s compiler.version=14") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + self.assertIn("compiler=gcc", info) + self.assertIn("compiler.libcxx=libstdc++11", info) + + # If other package is specified compiler is not modified + package_settings = {"NoExistsRecipe": tmp_settings} + create_profile(self.client.client_cache.profiles_path, + "vs_12_86_Hello0_gcc", settings=profile_settings, + package_settings=package_settings) + # Try to override some settings in install command + self.client.run("install --build missing -pr vs_12_86_Hello0_gcc -s compiler.version=14") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + self.assertIn("compiler=Visual Studio", info) + self.assertNotIn("compiler.libcxx", info) + + # Mix command line package settings with profile + package_settings = {"Hello0": tmp_settings} + create_profile(self.client.client_cache.profiles_path, "vs_12_86_Hello0_gcc", + settings=profile_settings, package_settings=package_settings) + + # Try to override some settings in install command + self.client.run("install --build missing -pr vs_12_86_Hello0_gcc" + " -s compiler.version=14 -s Hello0:compiler.libcxx=libstdc++") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + self.assertIn("compiler=gcc", info) + self.assertNotIn("compiler.libcxx=libstdc++11", info) + self.assertIn("compiler.libcxx=libstdc++", info) + + def install_profile_options_test(self): + files = cpp_hello_conan_files("Hello0", "0.1", build=False) + + create_profile(self.client.client_cache.profiles_path, "vs_12_86", + options=[("Hello0:language", 1), + ("Hello0:static", False)]) + + self.client.save(files) + self.client.run("install --build missing -pr vs_12_86") + info = load(os.path.join(self.client.current_folder, "conaninfo.txt")) + self.assertIn("language=1", info) + self.assertIn("static=False", info) + + def scopes_env_test(self): + # Create a profile and use it + create_profile(self.client.client_cache.profiles_path, "scopes_env", settings={}, + scopes={"Hello0:myscope": "1", + "ALL:otherscope": "2", + "undefined": "3"}, # undefined scope do not apply to my packages + env=[("CXX", "/path/tomy/g++"), ("CC", "/path/tomy/gcc")]) + self.client.save({CONANFILE: conanfile_scope_env}) + self.client.run("export lasote/stable") + self.client.run("install Hello0/0.1@lasote/stable --build missing -pr scopes_env") + + self.assertIn("Scope myscope: 1", self.client.user_io.out) + self.assertIn("Scope otherscope: 2", self.client.user_io.out) + self.assertIn("Scope undefined: None", self.client.user_io.out) + + self._assert_env_variable_printed("CC", "/path/tomy/gcc") + self._assert_env_variable_printed("CXX", "/path/tomy/g++") + + # The env variable shouldn't persist after install command + self.assertFalse(os.environ.get("CC", None) == "/path/tomy/gcc") + self.assertFalse(os.environ.get("CXX", None) == "/path/tomy/g++") + + def _get_profile(self, folder, txt): + abs_profile_path = os.path.join(folder, "Myprofile.txt") + save(abs_profile_path, txt) + return read_profile(abs_profile_path, None, None) + + def test_empty_env(self): + tmp = temp_folder() + profile, _ = self._get_profile(tmp, "[settings]") + self.assertTrue(isinstance(profile.env_values, EnvValues)) + + def test_package_test(self): + test_conanfile = '''from conans.model.conan_file import ConanFile +from conans import CMake +import os + +class DefaultNameConan(ConanFile): + name = "DefaultName" + version = "0.1" + settings = "os", "compiler", "arch", "build_type" + requires = "Hello0/0.1@lasote/stable" + + def build(self): + # Print environment vars + # self.run('cmake %s %s' % (self.conanfile_directory, cmake.command_line)) + if self.settings.os == "Windows": + self.run('echo "My var is %ONE_VAR%"') + else: + self.run('echo "My var is $ONE_VAR"') + + def test(self): + pass + +''' + files = {"conanfile.py": conanfile_scope_env, + "test_package/conanfile.py": test_conanfile} + # Create a profile and use it + create_profile(self.client.client_cache.profiles_path, "scopes_env", settings={}, + scopes={}, env=[("ONE_VAR", "ONE_VALUE")]) + + self.client.save(files) + self.client.run("test_package --profile scopes_env") + + self._assert_env_variable_printed("ONE_VAR", "ONE_VALUE") + self.assertIn("My var is ONE_VALUE", str(self.client.user_io.out)) + + # Try now with package environment vars + create_profile(self.client.client_cache.profiles_path, "scopes_env2", settings={}, + scopes={}, package_env={"DefaultName": [("ONE_VAR", "IN_TEST_PACKAGE")], + "Hello0": [("ONE_VAR", "PACKAGE VALUE")]}) + + self.client.run("test_package --profile scopes_env2") + + self._assert_env_variable_printed("ONE_VAR", "PACKAGE VALUE") + self.assertIn("My var is IN_TEST_PACKAGE", str(self.client.user_io.out)) + + # Try now overriding some variables with command line + self.client.run("test_package --profile scopes_env2 -e DefaultName:ONE_VAR=InTestPackageOverride " + "-e Hello0:ONE_VAR=PackageValueOverride ") + + self._assert_env_variable_printed("ONE_VAR", "PackageValueOverride") + self.assertIn("My var is InTestPackageOverride", str(self.client.user_io.out)) + + # A global setting in command line won't override a scoped package variable + self.client.run("test_package --profile scopes_env2 -e ONE_VAR=AnotherValue") + self._assert_env_variable_printed("ONE_VAR", "PACKAGE VALUE") + + def _assert_env_variable_printed(self, name, value): + self.assertIn("%s=%s" % (name, value), self.client.user_io.out) + + def info_with_profiles_test(self): + + self.client.run("remove '*' -f") + # Create a simple recipe to require + winreq_conanfile = ''' +from conans.model.conan_file import ConanFile + +class WinRequireDefaultNameConan(ConanFile): + name = "WinRequire" + version = "0.1" + settings = "os", "compiler", "arch", "build_type" + +''' + + files = {"conanfile.py": winreq_conanfile} + self.client.save(files) + self.client.run("export lasote/stable") + + # Now require the first recipe depending on OS=windows + conanfile = '''from conans.model.conan_file import ConanFile +import os + +class DefaultNameConan(ConanFile): + name = "Hello" + version = "0.1" + settings = "os", "compiler", "arch", "build_type" + + def config(self): + if self.settings.os == "Windows": + self.requires.add("WinRequire/0.1@lasote/stable") + +''' + files = {"conanfile.py": conanfile} + self.client.save(files) + self.client.run("export lasote/stable") + + # Create a profile that doesn't activate the require + create_profile(self.client.client_cache.profiles_path, "scopes_env", settings={"os": "Linux"}, + scopes={}) + + # Install with the previous profile + self.client.run("info Hello/0.1@lasote/stable --profile scopes_env") + self.assertNotIn('''Requires: + WinRequire/0.1@lasote/stable''', self.client.user_io.out) + + # Create a profile that activate the require + create_profile(self.client.client_cache.profiles_path, "scopes_env", settings={"os": "Windows"}, + scopes={}) + + # Install with the previous profile + self.client.run("info Hello/0.1@lasote/stable --profile scopes_env") + self.assertIn('''Requires: + WinRequire/0.1@lasote/stable''', self.client.user_io.out) + + def profile_vars_test(self): + tmp = temp_folder() + + txt = ''' + MY_MAGIC_VAR=The owls are not + + [env] + MYVAR=$MY_MAGIC_VAR what they seem. + ''' + profile, vars = self._get_profile(tmp, txt) + self.assertEquals("The owls are not what they seem.", profile.env_values.data[None]["MYVAR"]) + + # Order in replacement, variable names (simplification of preprocessor) + txt = ''' + P=Diane, the coffee at the Great Northern + P2=is delicious + + [env] + MYVAR=$P2 + ''' + profile, vars = self._get_profile(tmp, txt) + self.assertEquals("Diane, the coffee at the Great Northern2", profile.env_values.data[None]["MYVAR"]) + + # Variables without spaces + txt = ''' +VARIABLE WITH SPACES=12 +[env] +MYVAR=$VARIABLE WITH SPACES + ''' + with self.assertRaisesRegexp(ConanException, "The names of the variables cannot contain spaces"): + self._get_profile(tmp, txt) + + def test_profiles_includes(self): + + tmp = temp_folder() + def save_profile(txt, name): + abs_profile_path = os.path.join(tmp, name) + save(abs_profile_path, txt) + + os.mkdir(os.path.join(tmp, "subdir")) + + profile0 = """ +ROOTVAR=0 + + +[build_requires] + one/1.$ROOTVAR@lasote/stable +two/1.2@lasote/stable + +""" + save_profile(profile0, "subdir/profile0.txt") + + profile1 = """ + # Include in subdir, curdir +MYVAR=1 +include(profile0.txt) + + + + + +[settings] +os=Windows +[options] +zlib:aoption=1 +zlib:otheroption=1 +[env] +package1:ENVY=$MYVAR +[scopes] +my_scope=TRUE +""" + + save_profile(profile1, "subdir/profile1.txt") + + profile2 = """ +# Include in subdir +include(subdir/profile1.txt) +[settings] +os=$MYVAR +""" + + save_profile(profile2, "profile2.txt") + profile3 = """ +OTHERVAR=34 + +[scopes] +my_scope=AVALUE + +[build_requires] +one/1.5@lasote/stable + + +""" + save_profile(profile3, "profile3.txt") + + profile4 = """ + include(./profile2.txt) + include(./profile3.txt) + + [env] + MYVAR=FromProfile3And$OTHERVAR + + [options] + zlib:otheroption=12 + + """ + + save_profile(profile4, "profile4.txt") + + profile, vars = read_profile("./profile4.txt", tmp, None) + + self.assertEquals(vars, {"MYVAR": "1", "OTHERVAR": "34", "PROFILE_DIR": tmp , "ROOTVAR": "0"}) + self.assertEquals("FromProfile3And34", profile.env_values.data[None]["MYVAR"]) + self.assertEquals("1", profile.env_values.data["package1"]["ENVY"]) + self.assertEquals(profile.settings, {"os": "1"}) + self.assertEquals(profile.scopes.package_scope(), {"dev": True, "my_scope": "AVALUE"}) + self.assertEquals(profile.options.as_list(), [('zlib:aoption', '1'), ('zlib:otheroption', '12')]) + self.assertEquals(profile.build_requires, {"*": [ConanFileReference.loads("one/1.0@lasote/stable"), + ConanFileReference.loads("two/1.2@lasote/stable"), + ConanFileReference.loads("one/1.5@lasote/stable")]}) + + def profile_dir_test(self): + tmp = temp_folder() + txt = ''' +[env] +PYTHONPATH=$PROFILE_DIR/my_python_tools +''' + + def assert_path(profile): + pythonpath = profile.env_values.env_dicts("")[0]["PYTHONPATH"].replace("/", "\\") + self.assertEquals(pythonpath, os.path.join(tmp, "my_python_tools").replace("/", "\\")) + + abs_profile_path = os.path.join(tmp, "Myprofile.txt") + save(abs_profile_path, txt) + profile, _ = read_profile(abs_profile_path, None, None) + assert_path(profile) + + profile, _ = read_profile("./Myprofile.txt", tmp, None) + assert_path(profile) + + profile, _ = read_profile("Myprofile.txt", None, tmp) + assert_path(profile) diff --git a/conans/test/functional/registry_test.py b/conans/test/functional/registry_test.py index 758f2088694..6a586e22024 100644 --- a/conans/test/functional/registry_test.py +++ b/conans/test/functional/registry_test.py @@ -70,3 +70,20 @@ def refs_test(self): registry.set_ref(ref, remotes[0]) remote = registry.get_ref(ref) self.assertEqual(remote, remotes[0]) + + def insert_test(self): + f = os.path.join(temp_folder(), "aux_file") + save(f, "conan.io https://server.conan.io True") + registry = RemoteRegistry(f, TestBufferConanOutput()) + registry.add("repo1", "url1", True, insert=0) + self.assertEqual(registry.remotes, [("repo1", "url1", True), + ("conan.io", "https://server.conan.io", True)]) + registry.add("repo2", "url2", True, insert=1) + self.assertEqual(registry.remotes, [("repo1", "url1", True), + ("repo2", "url2", True), + ("conan.io", "https://server.conan.io", True)]) + registry.add("repo3", "url3", True, insert=5) + self.assertEqual(registry.remotes, [("repo1", "url1", True), + ("repo2", "url2", True), + ("conan.io", "https://server.conan.io", True), + ("repo3", "url3", True)]) diff --git a/conans/test/functional/symlink_package_test.py b/conans/test/functional/symlink_package_test.py new file mode 100644 index 00000000000..1d3ee348115 --- /dev/null +++ b/conans/test/functional/symlink_package_test.py @@ -0,0 +1,60 @@ +import platform +import unittest + +from conans.test.utils.tools import TestClient + + +class SymlinkPackageTest(unittest.TestCase): + + def test_symlink_created(self): + if platform.system() != "Linux" and platform.system() != "Darwin": + return + + conanfile = """from conans import ConanFile +import os + + +class TestlinksConan(ConanFile): + name = "test_links" + version = "1.0" + settings = "os", "compiler", "build_type", "arch" + generators = "cmake" + + def build(self): + os.makedirs("foo/test/bar") + with open("foo/test/bar/hello_world.txt", "w"): + os.utime("foo/test/bar/hello_world.txt", None) + os.symlink("test/bar", "foo/test_link") + + def package(self): + self.copy("*", src=".", dst=".", links=True) +""" + test_package = """from conans import ConanFile, CMake +from conans.errors import ConanException +import os + + +channel = os.getenv("CONAN_CHANNEL", "stable") +username = os.getenv("CONAN_USERNAME", "plex") + + +class TestlinksTestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + requires = "test_links/1.0@%s/%s" % (username, channel) + generators = "cmake" + + def test(self): + foopath = self.deps_cpp_info["test_links"].rootpath + "/foo/test_link" + assert os.path.exists(os.path.join(foopath, "hello_world.txt")) + if not os.path.islink(foopath): + raise ConanException("Not a link!") +""" + client = TestClient() + client.save({"conanfile.py": conanfile, + "test_package/conanfile.py": test_package}) + + client.run("test_package") + + +if __name__ == '__main__': + unittest.main() diff --git a/conans/test/integration/cmake_flags_test.py b/conans/test/integration/cmake_flags_test.py index b2afb1f65e2..1fce65d9c51 100644 --- a/conans/test/integration/cmake_flags_test.py +++ b/conans/test/integration/cmake_flags_test.py @@ -104,7 +104,7 @@ def targets_flags_test(self): self.assertNotIn("My", cmake_cxx_flags) self.assertIn("CONAN_CXX_FLAGS=MyFlag1 MyFlag2", client.user_io.out) self.assertIn("HELLO_CXX_FLAGS=MyFlag1;MyFlag2;" - "$<$:;>;$<$:;>", client.user_io.out) + "$<$:;>;$<$:;>;$<$:;>;$<$:;>", client.user_io.out) def targets_own_flags_test(self): client = TestClient() @@ -127,7 +127,7 @@ def targets_own_flags_test(self): self.assertIn("CmdCXXFlag", cmake_cxx_flags) self.assertIn("CONAN_CXX_FLAGS=MyFlag1 MyFlag2 CmdCXXFlag", client.user_io.out) self.assertIn("HELLO_CXX_FLAGS=MyFlag1;MyFlag2;" - "$<$:;>;$<$:;>", client.user_io.out) + "$<$:;>;$<$:;>;$<$:;>;$<$:;>", client.user_io.out) def transitive_targets_flags_test(self): client = TestClient() @@ -153,9 +153,9 @@ def transitive_targets_flags_test(self): self.assertIn("CONAN_CXX_FLAGS=MyFlag1 MyFlag2 MyChatFlag1 MyChatFlag2", client.user_io.out) self.assertIn("HELLO_CXX_FLAGS=MyFlag1;MyFlag2;" - "$<$:;>;$<$:;>", client.user_io.out) + "$<$:;>;$<$:;>;$<$:;>;$<$:;>", client.user_io.out) self.assertIn("CHAT_CXX_FLAGS=MyChatFlag1;MyChatFlag2;" - "$<$:;>;$<$:;>", client.user_io.out) + "$<$:;>;$<$:;>;$<$:;>;$<$:;>", client.user_io.out) def cmake_test_needed_settings(self): conanfile = """ diff --git a/conans/test/integration/conan_test_test.py b/conans/test/integration/conan_test_test.py index e095b40eda6..5393fa6aa7a 100644 --- a/conans/test/integration/conan_test_test.py +++ b/conans/test/integration/conan_test_test.py @@ -11,6 +11,35 @@ @attr("slow") class ConanTestTest(unittest.TestCase): + def transitive_same_name_test(self): + # https://github.com/conan-io/conan/issues/1366 + client = TestClient() + conanfile = ''' +from conans import ConanFile + +class HelloConan(ConanFile): + name = "HelloBar" + version = "0.1" +''' + test_package = ''' +from conans import ConanFile + +class HelloTestConan(ConanFile): + requires = "HelloBar/0.1@lasote/testing" + def test(self): + pass +''' + client.save({"conanfile.py": conanfile, "test_package/conanfile.py": test_package}) + client.run("test_package") + self.assertIn("HelloBar/0.1@lasote/testing: WARN: Forced build from source", + client.user_io.out) + client.save({"conanfile.py": conanfile.replace("HelloBar", "Hello") + + " requires='HelloBar/0.1@lasote/testing'", + "test_package/conanfile.py": test_package.replace("HelloBar", "Hello")}) + client.run("test_package") + self.assertNotIn("HelloBar/0.1@lasote/testing: WARN: Forced build from source", + client.user_io.out) + def test_package_env_test(self): client = TestClient() conanfile = ''' @@ -27,8 +56,6 @@ def package_info(self): from conans import ConanFile class HelloTestConan(ConanFile): - name = "HelloTest" - version = "0.1" requires = "Hello/0.1@lasote/testing" def build(self): diff --git a/conans/test/integration/only_source_test.py b/conans/test/integration/only_source_test.py index 819e713fa23..4aea3ecaf13 100644 --- a/conans/test/integration/only_source_test.py +++ b/conans/test/integration/only_source_test.py @@ -181,14 +181,14 @@ def reuse_test(self): # Use an invalid pattern and check that its not builded from source other_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) other_conan.run("install %s --build HelloInvalid" % str(conan_reference)) - self.assertIn("No package matching 'HelloInvalid*' pattern", other_conan.user_io.out) + self.assertIn("No package matching 'HelloInvalid' pattern", other_conan.user_io.out) self.assertFalse(os.path.exists(other_conan.paths.builds(conan_reference))) # self.assertFalse(os.path.exists(other_conan.paths.packages(conan_reference))) # Use another valid pattern and check that its not builded from source other_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) other_conan.run("install %s --build HelloInvalid -b Hello" % str(conan_reference)) - self.assertIn("No package matching 'HelloInvalid*' pattern", other_conan.user_io.out) + self.assertIn("No package matching 'HelloInvalid' pattern", other_conan.user_io.out) # self.assertFalse(os.path.exists(other_conan.paths.builds(conan_reference))) # self.assertFalse(os.path.exists(other_conan.paths.packages(conan_reference))) diff --git a/conans/test/integration/profile_build_requires_test.py b/conans/test/integration/profile_build_requires_test.py index ba0ce946324..e4de8d7347e 100644 --- a/conans/test/integration/profile_build_requires_test.py +++ b/conans/test/integration/profile_build_requires_test.py @@ -214,3 +214,40 @@ def build(self): self.assertEqual(1, str(client.user_io.out).splitlines().count("Hello World!")) self.assertIn("MyLib/0.1@lasote/stable: Hello world from python tool!", client.user_io.out) self.assertNotIn("Project: Hello world from python tool!", client.user_io.out) + + def build_requires_options_test(self): + client = TestClient() + lib_conanfile = """ +from conans import ConanFile + +class MyTool(ConanFile): + name = "MyTool" + version = "0.1" +""" + + client.save({CONANFILE: lib_conanfile}) + client.run("export lasote/stable") + conanfile = """ +from conans import ConanFile, tools + +class MyLib(ConanFile): + name = "MyLib" + version = "0.1" + build_requires = "MyTool/0.1@lasote/stable" + options = {"coverage": [True, False]} + def build(self): + self.output.info("Coverage %s" % self.options.coverage) +""" + client.save({CONANFILE: conanfile}, clean_first=True) + client.run("install -o MyLib:coverage=True --build missing") + self.assertIn("Installing build requires: [MyTool/0.1@lasote/stable]", + client.user_io.out) + client.run("build") + self.assertIn("Project: Coverage True", client.user_io.out) + + client.save({CONANFILE: conanfile}, clean_first=True) + client.run("install -o coverage=True") + self.assertIn("Installing build requires: [MyTool/0.1@lasote/stable]", + client.user_io.out) + client.run("build") + self.assertIn("Project: Coverage True", client.user_io.out) diff --git a/conans/test/integration/profile_test.py b/conans/test/integration/profile_test.py index d65df4af355..d5f4a771661 100644 --- a/conans/test/integration/profile_test.py +++ b/conans/test/integration/profile_test.py @@ -1,7 +1,8 @@ import unittest +from conans.client.profile_loader import _load_profile from conans.model.env_info import EnvValues -from conans.model.profile import Profile + from conans.test.utils.tools import TestClient from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.util.files import save, load @@ -253,10 +254,6 @@ def scopes_env_test(self): self.assertFalse(os.environ.get("CC", None) == "/path/tomy/gcc") self.assertFalse(os.environ.get("CXX", None) == "/path/tomy/g++") - def test_empty_env(self): - profile = Profile.loads("[settings]") - self.assertTrue(isinstance(profile.env_values, EnvValues)) - def test_package_test(self): test_conanfile = '''from conans.model.conan_file import ConanFile from conans import CMake diff --git a/conans/test/model/profile_test.py b/conans/test/model/profile_test.py index 0bf60502ef3..53f4e58cd4b 100644 --- a/conans/test/model/profile_test.py +++ b/conans/test/model/profile_test.py @@ -1,6 +1,7 @@ import os import unittest +from conans.client.profile_loader import _load_profile, read_profile from conans.model.profile import Profile from collections import OrderedDict @@ -11,54 +12,12 @@ class ProfileTest(unittest.TestCase): - def profile_test(self): - - # Empty profile - profile = Profile() - dump = profile.dumps() - new_profile = Profile.loads(dump) - self.assertEquals(new_profile.settings, profile.settings) - - # Settings - profile = Profile() - profile.settings["arch"] = "x86_64" - profile.settings["compiler"] = "Visual Studio" - profile.settings["compiler.version"] = "12" - - profile.env_values.add("CXX", "path/to/my/compiler/g++") - profile.env_values.add("CC", "path/to/my/compiler/gcc") - - profile.scopes["p1"]["conaning"] = "1" - profile.scopes["p2"]["testing"] = "2" - - profile.build_requires["*"] = ["android_toolchain/1.2.8@lasote/testing"] - profile.build_requires["zlib/*"] = ["cmake/1.0.2@lasote/stable", - "autotools/1.0.3@lasote/stable"] - - dump = profile.dumps() - new_profile = Profile.loads(dump) - self.assertEquals(new_profile.settings, profile.settings) - self.assertEquals(new_profile.settings["arch"], "x86_64") - self.assertEquals(new_profile.settings["compiler.version"], "12") - self.assertEquals(new_profile.settings["compiler"], "Visual Studio") - - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX': 'path/to/my/compiler/g++', - 'CC': 'path/to/my/compiler/gcc'}, {})) - - self.assertEquals(dict(new_profile.scopes)["p1"]["conaning"], '1') - self.assertEquals(dict(new_profile.scopes)["p2"]["testing"], '2') - - self.assertEquals(new_profile.build_requires["zlib/*"], - [ConanFileReference.loads("cmake/1.0.2@lasote/stable"), - ConanFileReference.loads("autotools/1.0.3@lasote/stable")]) - self.assertEquals(new_profile.build_requires["*"], - [ConanFileReference.loads("android_toolchain/1.2.8@lasote/testing")]) def profile_settings_update_test(self): prof = '''[settings] os=Windows ''' - new_profile = Profile.loads(prof) + new_profile, _ = _load_profile(prof, None, None) new_profile.update_settings([("OTHER", "2")]) self.assertEquals(new_profile.settings, OrderedDict([("os", "Windows"), ("OTHER", "2")])) @@ -72,7 +31,7 @@ def package_settings_update_test(self): prof = '''[settings] MyPackage:os=Windows ''' - np = Profile.loads(prof) + np , _ = _load_profile(prof, None, None) np.update_package_settings({"MyPackage": [("OTHER", "2")]}) self.assertEquals(np.package_settings_values, @@ -83,60 +42,6 @@ def package_settings_update_test(self): {"MyPackage": [("os", "Windows"), ("OTHER", "2"), ("compiler", "2"), ("compiler.version", "3")]}) - def profile_loads_test(self): - prof = '''[env] -CXX_FLAGS="-DAAA=0" -[settings] -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) - - prof = '''[env] -CXX_FLAGS="-DAAA=0" -MyPackage:VAR=1 -MyPackage:OTHER=2 -OtherPackage:ONE=ONE -[settings] -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) - self.assertEquals(new_profile.env_values.env_dicts("MyPackage"), ({"OTHER": "2", - "VAR": "1", - 'CXX_FLAGS': '-DAAA=0'}, {})) - - self.assertEquals(new_profile.env_values.env_dicts("OtherPackage"), ({'CXX_FLAGS': '-DAAA=0', - 'ONE': 'ONE'}, {})) - - prof = '''[env] -CXX_FLAGS='-DAAA=0' -[settings] -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) - - prof = '''[env] -CXX_FLAGS=-DAAA=0 -[settings] -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '-DAAA=0'}, {})) - - prof = '''[env] -CXX_FLAGS="-DAAA=0 -[settings] -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.env_values.env_dicts(""), ({'CXX_FLAGS': '"-DAAA=0'}, {})) - - prof = ''' -[settings] -zlib:compiler=gcc -compiler=Visual Studio -''' - new_profile = Profile.loads(prof) - self.assertEquals(new_profile.package_settings["zlib"], {"compiler": "gcc"}) - self.assertEquals(new_profile.settings["compiler"], "Visual Studio") - def profile_dump_order_test(self): # Settings profile = Profile() @@ -158,19 +63,6 @@ def profile_dump_order_test(self): [scopes] [env]""".splitlines(), profile.dumps().splitlines()) - def profile_loads_win_test(self): - prof = '''[env] -QTPATH=C:/QtCommercial/5.8/msvc2015_64/bin -QTPATH2="C:/QtCommercial2/5.8/msvc2015_64/bin" -''' - new_profile = Profile.loads(prof) - self.assertEqual(new_profile.env_values.data[None]["QTPATH"], - "C:/QtCommercial/5.8/msvc2015_64/bin") - self.assertEqual(new_profile.env_values.data[None]["QTPATH2"], - "C:/QtCommercial2/5.8/msvc2015_64/bin") - self.assertIn("QTPATH=C:/QtCommercial/5.8/msvc2015_64/bin", new_profile.dumps()) - self.assertIn("QTPATH2=C:/QtCommercial2/5.8/msvc2015_64/bin", new_profile.dumps()) - def apply_test(self): # Settings profile = Profile() @@ -196,25 +88,3 @@ def apply_test(self): '[options]\n[scopes]\np1:new_one=2\np2:testing=True\n' '[env]\nCC=path/to/my/compiler/gcc\nCXX=path/to/my/compiler/g++', profile.dumps()) - - def profile_dir_test(self): - tmp = temp_folder() - txt = ''' -[env] -PYTHONPATH=$PROFILE_DIR/my_python_tools -''' - - def assert_path(profile): - pythonpath = profile.env_values.env_dicts("")[0]["PYTHONPATH"].replace("/", "\\") - self.assertEquals(pythonpath, os.path.join(tmp, "my_python_tools").replace("/", "\\")) - - abs_profile_path = os.path.join(tmp, "Myprofile.txt") - save(abs_profile_path, txt) - profile = Profile.read_file(abs_profile_path, None, None) - assert_path(profile) - - profile = Profile.read_file("./Myprofile.txt", tmp, None) - assert_path(profile) - - profile = Profile.read_file("Myprofile.txt", None, tmp) - assert_path(profile) diff --git a/conans/test/util/output_test.py b/conans/test/util/output_test.py index 65a29d32d1f..20fb7b8081c 100644 --- a/conans/test/util/output_test.py +++ b/conans/test/util/output_test.py @@ -65,7 +65,7 @@ def unzip_output_test(self): new_out = StringIO() old_out = sys.stdout try: - sys.stdout = new_out + tools._global_output = ConanOutput(new_out) tools.unzip(zip_path, output_dir) finally: sys.stdout = old_out diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 94d3ec5d946..fff24f772de 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -1,12 +1,17 @@ import os import platform +import tempfile import unittest from collections import namedtuple + +from conans.client.client_cache import CONAN_CONF from nose.plugins.attrib import attr from conans import tools -from conans.client.conf import default_settings_yml +from conans.client.conan_api import ConanAPIV1 +from conans.client.conf import default_settings_yml, default_client_conf + from conans.errors import ConanException from conans.model.settings import Settings from conans.paths import CONANFILE @@ -15,10 +20,10 @@ from conans.test.utils.tools import TestClient, TestBufferConanOutput from conans.test.utils.visual_project_files import get_vs_project_files from conans.tools import OSInfo, SystemPackageTool, replace_in_file, AptTool +from conans.util.files import save class RunnerMock(object): - def __init__(self, return_ok=True): self.command_called = None self.return_ok = return_ok @@ -29,7 +34,6 @@ def __call__(self, command, output): # @UnusedVariable class ReplaceInFileTest(unittest.TestCase): - def setUp(self): text = u'J\xe2nis\xa7' self.tmp_folder = temp_folder() @@ -59,7 +63,6 @@ def test_replace_in_file(self): class ToolsTest(unittest.TestCase): - def cpu_count_test(self): cpus = tools.cpu_count() self.assertIsInstance(cpus, int) @@ -67,6 +70,41 @@ def cpu_count_test(self): with tools.environment_append({"CONAN_CPU_COUNT": "34"}): self.assertEquals(tools.cpu_count(), 34) + def test_global_tools_overrided(self): + client = TestClient() + + tools._global_requester = None + tools._global_output = None + + conanfile = """ +from conans import ConanFile, tools + +class HelloConan(ConanFile): + name = "Hello" + version = "0.1" + + def build(self): + assert(tools._global_requester != None) + assert(tools._global_output != None) + """ + client.save({"conanfile.py": conanfile}) + client.run("build") + + + # Not test the real commmand get_command if it's setting the module global vars + tools._global_requester = None + tools._global_output = None + tmp = tempfile.mkdtemp() + conf = default_client_conf.replace("\n[proxies]", "\n[proxies]\nhttp = http://myproxy.com") + os.mkdir(os.path.join(tmp, ".conan")) + save(os.path.join(tmp, ".conan", CONAN_CONF), conf) + with tools.environment_append({"CONAN_USER_HOME": tmp}): + conan_api = ConanAPIV1.factory() + conan_api.remote_list() + self.assertEquals(tools._global_requester.proxies, {"http": "http://myproxy.com"}) + + self.assertIsNotNone(tools._global_output.warn) + def test_environment_nested(self): with tools.environment_append({"A": "1", "Z": "40"}): with tools.environment_append({"A": "1", "B": "2"}): @@ -88,7 +126,7 @@ def system_package_tool_fail_when_not_0_returned_test(self): spt = SystemPackageTool(runner=runner) if platform.system() == "Linux" or platform.system() == "Darwin": msg = "Command 'sudo apt-get update' failed" if platform.system() == "Linux" \ - else "Command 'brew update' failed" + else "Command 'brew update' failed" with self.assertRaisesRegexp(ConanException, msg): spt.update() else: @@ -289,7 +327,6 @@ def run_in_bash_test(self): return class MockConanfile(object): - def __init__(self): self.command = "" self.output = namedtuple("output", "info")(lambda x: None) diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index 5f12f11c182..56ff6caa2c4 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -14,7 +14,8 @@ from conans import __version__ as CLIENT_VERSION, tools from conans.client.client_cache import ClientCache -from conans.client.command import Command, migrate_and_get_client_cache +from conans.client.command import Command +from conans.client.conan_api import migrate_and_get_client_cache, Conan from conans.client.conf import MIN_SERVER_COMPATIBLE_VERSION from conans.client.output import ConanOutput from conans.client.remote_manager import RemoteManager @@ -390,6 +391,10 @@ def _init_collaborators(self, user_io=None): # Handle remote connections self.remote_manager = RemoteManager(self.client_cache, auth_manager, self.user_io.out) + # Patch the globals in tools + tools._global_requester = requests + tools._global_output = self.user_io.out + def init_dynamic_vars(self, user_io=None): # Migration system self.client_cache = migrate_and_get_client_cache(self.base_folder, TestBufferConanOutput(), @@ -404,8 +409,8 @@ def run(self, command_line, user_io=None, ignore_error=False): tuple if required """ self.init_dynamic_vars(user_io) - - command = Command(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager) + conan = Conan(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager) + command = Command(conan, self.client_cache, self.user_io) args = shlex.split(command_line) current_dir = os.getcwd() os.chdir(self.current_folder) diff --git a/conans/tools.py b/conans/tools.py index 61f4c2cd743..5c0dde38d17 100644 --- a/conans/tools.py +++ b/conans/tools.py @@ -25,6 +25,10 @@ from conans.util.files import _generic_algorithm_sum, load, save, sha256sum, sha1sum, md5sum, md5 from conans.util.log import logger +# Default values +_global_requester = requests +_global_output = ConanOutput(sys.stdout) + def unix_path(path): """"Used to translate windows paths to MSYS unix paths like @@ -195,7 +199,7 @@ def cpu_count(): env_cpu_count = os.getenv("CONAN_CPU_COUNT", None) return int(env_cpu_count) if env_cpu_count else multiprocessing.cpu_count() except NotImplementedError: - print("WARN: multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") + _global_output.warn("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") return 1 # Safe guess @@ -243,15 +247,15 @@ def unzip(filename, destination=".", keep_permissions=False): if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): def print_progress(extracted_size, uncompress_size): - txt_msg = "Unzipping %.0f %%\r" % (extracted_size * 100.0 / uncompress_size) - print(txt_msg, end='') + txt_msg = "Unzipping %.0f %%" % (extracted_size * 100.0 / uncompress_size) + _global_output.rewrite_line(txt_msg) else: def print_progress(extracted_size, uncompress_size): pass with zipfile.ZipFile(filename, "r") as z: uncompress_size = sum((file_.file_size for file_ in z.infolist())) - print("Unzipping %s, this can take a while" % human_size(uncompress_size)) + _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) extracted_size = 0 if platform.system() == "Windows": for file_ in z.infolist(): @@ -263,7 +267,7 @@ def print_progress(extracted_size, uncompress_size): raise ValueError("Filename too long") z.extract(file_, full_path) except Exception as e: - print("Error extract %s\n%s" % (file_.filename, str(e))) + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) else: # duplicated for, to avoid a platform check for each zipped file for file_ in z.infolist(): extracted_size += file_.file_size @@ -276,7 +280,7 @@ def print_progress(extracted_size, uncompress_size): perm = file_.external_attr >> 16 & 0xFFF os.chmod(os.path.join(full_path, file_.filename), perm) except Exception as e: - print("Error extract %s\n%s" % (file_.filename, str(e))) + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) def untargz(filename, destination="."): @@ -294,13 +298,32 @@ def get(url): os.unlink(filename) +def ftp_download(ip, filename, login='', password=''): + import ftplib + try: + ftp = ftplib.FTP(ip, login, password) + ftp.login() + filepath, filename = os.path.split(filename) + if filepath: + ftp.cwd(filepath) + with open(filename, 'wb') as f: + ftp.retrbinary('RETR ' + filename, f.write) + except Exception as e: + raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) + finally: + try: + ftp.quit() + except: + pass + + def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): out = out or ConanOutput(sys.stdout, True) if verify: # We check the certificate using a list of known verifiers import conans.client.rest.cacert as cacert verify = cacert.file_path - downloader = Downloader(requests, out, verify=verify) + downloader = Downloader(_global_requester, out, verify=verify) downloader.download(url, filename, retry=retry, retry_wait=retry_wait) out.writeln("") @@ -403,16 +426,16 @@ def detected_architecture(): class OSInfo(object): """ Usage: - print(os_info.is_linux) # True/False - print(os_info.is_windows) # True/False - print(os_info.is_macos) # True/False - print(os_info.is_freebsd) # True/False - print(os_info.is_solaris) # True/False + (os_info.is_linux) # True/False + (os_info.is_windows) # True/False + (os_info.is_macos) # True/False + (os_info.is_freebsd) # True/False + (os_info.is_solaris) # True/False - print(os_info.linux_distro) # debian, ubuntu, fedora, centos... + (os_info.linux_distro) # debian, ubuntu, fedora, centos... - print(os_info.os_version) # 5.1 - print(os_info.os_version_name) # Windows 7, El Capitan + (os_info.os_version) # 5.1 + (os_info.os_version_name) # Windows 7, El Capitan if os_info.os_version > "10.1": pass @@ -576,7 +599,7 @@ def get_solaris_version_name(version): os_info = OSInfo() except Exception as exc: logger.error(exc) - print("Error detecting os_info") + _global_output.error("Error detecting os_info") class SystemPackageTool(object): @@ -621,7 +644,7 @@ def install(self, packages, update=True, force=False): def _installed(self, packages): for pkg in packages: if self._tool.installed(pkg): - print("Package already installed: %s" % pkg) + _global_output.info("Package already installed: %s" % pkg) return True return False @@ -641,7 +664,7 @@ def update(self): pass def install(self, package_name): - print("Warn: Only available for linux with apt-get or yum or OSx with brew") + _global_output.warn("Only available for linux with apt-get or yum or OSx with brew") def installed(self, package_name): return False @@ -684,6 +707,6 @@ def installed(self, package_name): def _run(runner, command): - print("Running: %s" % command) + _global_output.info("Running: %s" % command) if runner(command, True) != 0: raise ConanException("Command '%s' failed" % command)