From abd88b55c9d056747b3b199185c303861f0ccca5 Mon Sep 17 00:00:00 2001 From: Nicholas Ohs Date: Wed, 28 Oct 2020 00:10:36 +0100 Subject: [PATCH 1/2] IO module now behaves according to issue #121. --- clang_build/directories.py | 2 +- clang_build/io_tools.py | 364 +++++++++++++++++++----- clang_build/project.py | 3 +- docs/user_guide/inheritance.rst | 4 +- docs/user_guide/platform_dependence.rst | 4 +- 5 files changed, 296 insertions(+), 81 deletions(-) diff --git a/clang_build/directories.py b/clang_build/directories.py index 8ced663..306c58b 100644 --- a/clang_build/directories.py +++ b/clang_build/directories.py @@ -4,7 +4,7 @@ def __init__(self, files, dependencies): # Include directories self.include_private = files["include_directories"] - self.include_public = files["include_directories_public"] + self.include_public = files["public_include_directories"] # Default include path # if self.root_directory.joinpath('include').exists(): diff --git a/clang_build/io_tools.py b/clang_build/io_tools.py index 19a6e98..7ad920d 100644 --- a/clang_build/io_tools.py +++ b/clang_build/io_tools.py @@ -1,101 +1,317 @@ -from glob import iglob as _iglob -from pathlib import Path as _Path +"""Module for source file discovery and include directories.""" -def _get_header_files_in_folders(folders, exclude_patterns=[], recursive=True): - delimiter = '/**/' if recursive else '/*' - patterns = [str(folder) + delimiter + ext for ext in ('*.hpp', '*.hxx', '*.h') for folder in folders] - return _get_files_in_patterns(patterns) +from itertools import chain as _chain -def _get_source_files_in_folders(folders, exclude_patterns=[], recursive=True): - delimiter = '/**/' if recursive else '/*' - patterns = [str(folder) + delimiter + ext for ext in ('*.cpp', '*.cxx', '*.c') for folder in folders] - return _get_files_in_patterns(patterns) -def _get_files_in_patterns(patterns, exclude_patterns=[], recursive=True): - included = [_Path(f) for pattern in patterns for f in _iglob(str(pattern), recursive=recursive) if _Path(f).is_file()] - excluded = [_Path(f) for pattern in exclude_patterns for f in _iglob(str(pattern), recursive=recursive) if _Path(f).is_file()] - return list(f.resolve() for f in (set(included) - set(excluded))) +def _get_files_from_folders( + directories, extensions, target_directory, exclude_patterns, recursive +): + """Get files from a given list of directories. -def get_sources_and_headers(target_name, platform, target_options, target_root_directory, target_build_directory): - output = {'headers': [], 'include_directories': [], 'include_directories_public': [], 'sourcefiles': []} + Parameters + ---------- + directories : iterable + set of directories to browse for files + extensions : iterable + list of file extensions to search for + target_directory : pathlib.Path + the directory of the target for which to search files for + exclude_patterns : iterable + list of patterns specifying which files not to include + (relative to the target directory) + recursive : bool + whether to search for files recursively - # TODO: maybe the output should also include the root dir, build dir and potentially download dir? - # TODO: should warn when a specified directory does not exist! + Returns + ------- + set of pathlib.Path + the files discovered - # Options for include directories - include_options = [] - include_options += target_options.get('include_directories', []) + """ + delimiter = "**/" if recursive else "" + patterns = [delimiter + ext for ext in extensions] - include_options += target_options.get(platform, {}).get('include_directories', []) + return _get_files_from_patterns(patterns, directories) - _get_files_from_patterns( + exclude_patterns, [target_directory] + ) - exclude_options = [] - exclude_options += target_options.get('headers_exclude', []) - exclude_options += target_options.get(platform, {}).get('headers_exclude', []) +def _get_header_files_in_folders( + directories, target_directory, exclude_patterns, recursive +): + """Get header files from a given list of directories. - include_patterns = list(set(target_root_directory.joinpath(path) for path in include_options)) - exclude_patterns = list(set(target_root_directory.joinpath(path) for path in exclude_options)) + The extensions used to find these files are: - # Find header files - if include_patterns: - output['include_directories'] = include_patterns - output['headers'] += _get_header_files_in_folders(output['include_directories'], exclude_patterns=exclude_patterns, recursive=True) - else: - output['include_directories'] += [target_root_directory.joinpath(''), target_root_directory.joinpath('include'), - target_root_directory.joinpath(target_name)] - output['headers'] += _get_header_files_in_folders(output['include_directories'], exclude_patterns=exclude_patterns, recursive=False) + - *.hpp + - *.hxx + - *.h - # Options for public include directories, exclude patterns are the same - include_options_public = [] - include_options_public += target_options.get('include_directories_public', []) + Parameters + ---------- + directories : iterable + set of directories to browse for files + target_directory : pathlib.Path + the directory of the target for which to search files for + exclude_patterns : iterable + list of patterns specifying which files not to include + (relative to the target directory) + recursive : bool + whether to search for files recursively - include_options_public += target_options.get(platform, {}).get('include_directories_public', []) + Returns + ------- + set of pathlib.Path + the header files discovered - include_patterns = list(dict.fromkeys(target_root_directory.joinpath(path) for path in include_options_public)) - exclude_patterns = list(dict.fromkeys(target_root_directory.joinpath(path) for path in exclude_options)) + """ + return _get_files_from_folders( + directories, + ("*.hpp", "*.hxx", "*.h"), + target_directory, + exclude_patterns, + recursive, + ) - # Find header files - if include_patterns: - output['include_directories_public'] = include_patterns - output['headers'] += _get_header_files_in_folders(output['include_directories_public'], exclude_patterns=exclude_patterns, recursive=True) - else: - output['include_directories_public'] += [target_root_directory.joinpath(''), target_root_directory.joinpath('include'), - target_root_directory.joinpath(target_name, 'include')] - output['headers'] += _get_header_files_in_folders(output['include_directories_public'], exclude_patterns=exclude_patterns, recursive=False) - # Keep only include directories which exist - output['include_directories'] = [directory for directory in output['include_directories'] if directory.exists()] - output['include_directories_public'] = [directory for directory in output['include_directories_public'] if directory.exists()] +def _get_source_files_in_folders( + directories, target_directory, exclude_patterns, recursive +): + """Get source files from a given list of directories. - # Options for sources - sources_options = [] - sources_options += target_options.get('sources', []) + The extensions used to find these files are: + + - *.cpp + - *.cxx + - *.c + + Parameters + ---------- + directories : iterable + set of directories to browse for files + target_directory : pathlib.Path + the directory of the target for which to search files for + exclude_patterns : list + list of patterns specifying which files not to include + (relative to the target directory) + recursive : bool + whether to search for files recursively + + Returns + ------- + set of pathlib.Path + the source files discovered + + """ + return _get_files_from_folders( + directories, + ("*.cpp", "*.cxx", "*.c"), + target_directory, + exclude_patterns, + recursive, + ) + + +def _get_files_from_patterns(patterns, directories): + """Get files from given patterns for given directories. + + Each pattern is passed to to the :any:`pathlib.Path.glob` + function of each given directory and all files are accumulated + in a set. + + Parameters + ---------- + patterns: iterables + patterns to pass to `pathlib.Path.glob`. + directories: list of pathlib.Path + directories in which to glob for files + + Returns + ------- + set of pathlib.Path + the files discovered + + """ + return set( + _chain( + *tuple(directory.glob(pattern) for pattern in patterns for directory in directories) + ) + ) + + +def _get_global_and_platform_option(target_config, key, platform): + """Check a target dictionary for global and platform specific options. + + As include directory for example might be platform specific, + this function looks into a target configuration and returns both, + the global and the platform specific configuration for a given key. + + Parameters + ---------- + target_config : dict + configuration dict of a target + key : str + the key to look up in the dict + platform : str + the platform for which to get the configuration + + Returns + ------- + set + all found options + """ + option = set(target_config.get(key, [])) + option.update(target_config.get(platform, {}).get(key, [])) + + return option - sources_options += target_options.get(platform, {}).get('sources', []) - exclude_options = [] - exclude_options += target_options.get('sources_exclude', []) +def _existing_directories(relative_folders, target_root_directory): + """Return absolute paths of all given folders which exist. - exclude_options += target_options.get(platform, {}).get('sources_exclude', []) + Parameters + ---------- + relative_folders : iterable + folders to check + target_root_directory : pathlib.Path + target root path the relative folders are referring to - sources_patterns = list(dict.fromkeys(target_root_directory.joinpath(path) for path in sources_options)) - exclude_patterns = list(dict.fromkeys(target_root_directory.joinpath(path) for path in exclude_options)) + Returns + ------- + set + all existing folders as absolute paths - # Find source files from patterns (recursively) + """ + directories = { + path + for folder in relative_folders + if (path := (target_root_directory / folder).resolve()).is_dir() + } + + return directories + + +def get_sources_and_headers( + target_name, + platform, + target_config, + project_root_directory, +): + """Get sources, headers, and include folders (public and private) of a target. + + Parameters + ---------- + target_name : str + name of the target for which to obtain files + platform : str + the platform for which to build + target_config : dict + the configuration dict of the target + project_root_directory : pathlib.Path + the directory of the project that owns this target + + Returns + ------- + dict + With keys "headers", "include_directories", "public_include_directories", and + "sourcefiles" and lists as values + + """ + output = { + "headers": set(), + "include_directories": set(), + "public_include_directories": set(), + "sourcefiles": set(), + } + + # TODO: maybe the output should also include the root dir, build dir and potentially download dir? + # TODO: should warn when a specified directory does not exist! + + # Behaviour of the search lateron depends on whether there is a dedicated target directory + if ( + in_project_root := not ( + target_root_directory := (project_root_directory / target_name).resolve() + ).is_dir() + ) : + target_root_directory = project_root_directory + + # Options for include directories + custom_include_directories = _existing_directories( + _get_global_and_platform_option(target_config, "include_directories", platform), + target_root_directory, + ) + + exclude_patterns_headers = _get_global_and_platform_option( + target_config, "headers_exclude", platform + ) + + # Find header files + if custom_include_directories: + include_directories = custom_include_directories + recursive = True + else: + if (include_folder := (target_root_directory / "include").resolve()).exists(): + include_directories = {include_folder} + recursive = True + else: + include_directories = {target_root_directory} + recursive = not in_project_root + + output["include_directories"] |= include_directories + output["headers"] |= _get_header_files_in_folders( + include_directories, + target_root_directory, + exclude_patterns_headers, + recursive=recursive, + ) + + # Options for public include directories, exclude patterns are the same + custom_include_directories_public = _existing_directories( + _get_global_and_platform_option( + target_config, "public_include_directories", platform + ), + target_root_directory, + ) + + # Find header files for public directories, no default paths for public dirs + if custom_include_directories_public: + output["public_include_directories"] |= custom_include_directories_public + output["headers"] |= _get_header_files_in_folders( + custom_include_directories_public, + target_root_directory, + exclude_patterns=exclude_patterns_headers, + recursive=True, + ) + + # Options for sources + sources_patterns = _get_global_and_platform_option( + target_config, "sources", platform + ) + exclude_patterns = _get_global_and_platform_option( + target_config, "sources_exclude", platform + ) + + # Find source files from patterns (recursivity specified by patterns) if sources_patterns: - output['sourcefiles'] += _get_files_in_patterns(sources_patterns, exclude_patterns=exclude_patterns, recursive=True) - # Else search source files in folder with same name as target and src folder (recursively) + output["sourcefiles"] |= _get_files_from_patterns( + sources_patterns, [target_root_directory] + ) - _get_files_from_patterns(exclude_patterns, [target_root_directory]) + + # Else search source files in folder with same name as target and src folder else: - output['sourcefiles'] += _get_source_files_in_folders([target_root_directory.joinpath(target_name), target_root_directory.joinpath('src')], exclude_patterns=exclude_patterns, recursive=True) + if (src_folder := (target_root_directory / "src").resolve()).exists(): + source = {src_folder} + recursive = True - # Search the root folder as last resort (non-recursively) - if not output['sourcefiles']: - output['sourcefiles'] += _get_source_files_in_folders([target_root_directory], exclude_patterns=exclude_patterns, recursive=False) + else: + source = {target_root_directory} + recursive = not in_project_root - # Fill return dict - output['include_directories'] = list(dict.fromkeys(output['include_directories'] )) - output['include_directories_public'] = list(dict.fromkeys(output['include_directories_public'] )) - output['headers'] = list(dict.fromkeys(output['headers'] )) - output['sourcefiles'] = list(dict.fromkeys(output['sourcefiles'] )) + output["sourcefiles"] |= _get_source_files_in_folders( + source, + target_root_directory, + exclude_patterns, + recursive=recursive, + ) - return output \ No newline at end of file + return {key: list(val) for key, val in output.items()} diff --git a/clang_build/project.py b/clang_build/project.py index d7305fe..2cd1d6a 100644 --- a/clang_build/project.py +++ b/clang_build/project.py @@ -681,8 +681,7 @@ def _target_from_description(self, target_description): target_description.name, self._environment.toolchain.platform, target_description.config, - target_description.root_directory, - target_description.build_directory, + self.directory, ) # Create specific target if the target type was specified diff --git a/docs/user_guide/inheritance.rst b/docs/user_guide/inheritance.rst index 58a637b..70f38e2 100644 --- a/docs/user_guide/inheritance.rst +++ b/docs/user_guide/inheritance.rst @@ -11,7 +11,7 @@ Include directories Defaults are the target directory and "include". -**`include_directories_public`** +**`public_include_directories`** A list of public include directories, which are accessible to any dependent target. Note that these are forwarded up the dependency graph, i.e. a target adds all of @@ -25,7 +25,7 @@ Include directories [mylib] include_directories = ["src/mylib/include", "src/mylib/detail/include"] - include_directories_public = ["src/mylib/include"] + public_include_directories = ["src/mylib/include"] Flags diff --git a/docs/user_guide/platform_dependence.rst b/docs/user_guide/platform_dependence.rst index 894c798..5376392 100644 --- a/docs/user_guide/platform_dependence.rst +++ b/docs/user_guide/platform_dependence.rst @@ -7,7 +7,7 @@ Per-Platform Target Configuration The following lists and sections of the target configuration can be specified individually per platform: -- `include_directories` and `include_directories_public` +- `include_directories` and `public_include_directories` - `sources` - `flags` @@ -23,7 +23,7 @@ Example [mylib] target_type = "static library" - include_directories_public = ["include"] + public_include_directories = ["include"] sources = ["src/common.c"] [mylib.public_flags] From 41e287f9ad00a069d87f1fd009a74b1191cb37a6 Mon Sep 17 00:00:00 2001 From: Nicholas Ohs Date: Wed, 28 Oct 2020 00:15:48 +0100 Subject: [PATCH 2/2] Update to visual studio 2019 appveyor image (py38) --- .appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c2e3538..b844c6a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,4 +1,4 @@ -image: Visual Studio 2017 +image: Visual Studio 2019 platform: - x64 @@ -8,7 +8,7 @@ configuration: - Debug environment: - PYTHON: "C:\\Python36-x64" + PYTHON: "C:\\Python38-x64" build: off