From eb14aa5fe2b347d4be5225a516337ebc4691d38f Mon Sep 17 00:00:00 2001 From: Christophe Bedard Date: Thu, 11 Jun 2020 17:04:02 -0400 Subject: [PATCH] Add 'extends' feature to overlay repos files when importing --- test/import_extends.txt | 89 ++++++++++++++++++++++++++ test/list_extends.repos | 14 ++++ test/list_extends2.repos | 10 +++ test/list_extends_loop.repos | 6 ++ test/list_extends_loop2.repos | 6 ++ test/test_commands.py | 117 ++++++++++++++++++++++++++++++++++ vcstool/commands/import_.py | 53 ++++++++++++++- 7 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 test/import_extends.txt create mode 100644 test/list_extends.repos create mode 100644 test/list_extends2.repos create mode 100644 test/list_extends_loop.repos create mode 100644 test/list_extends_loop2.repos diff --git a/test/import_extends.txt b/test/import_extends.txt new file mode 100644 index 00000000..e22e97fc --- /dev/null +++ b/test/import_extends.txt @@ -0,0 +1,89 @@ +........ +=== ./immutable/hash (git) === +Cloning into '.'... +Note: switching to '377d5b3d03c212f015cc832fdb368f4534d0d583'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 377d5b3... update changelog +=== ./immutable/hash_tar (tar) === +Downloaded tarball from 'https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz' and unpacked it +=== ./immutable/hash_zip (zip) === +Downloaded zipfile from 'https://github.com/dirk-thomas/vcstool/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it +=== ./immutable/tag (git) === +Cloning into '.'... +Note: switching to '0.2.7'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 80aadd6... 0.2.7 +=== ./vcstool (git) === +Cloning into '.'... +=== ./vcstool-custom (git) === +Cloning into '.'... +Note: switching to '0.2.10'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 865990f... 0.2.10 +=== ./vcstool-old (git) === +Cloning into '.'... +Note: switching to '0.1.2'. + +You are in 'detached HEAD' state. You can look around, make experimental +changes and commit them, and you can discard any commits you make in this +state without impacting any branches by switching back to a branch. + +If you want to create a new branch to retain commits you create, you may +do so (now or later) by using -c with the switch command. Example: + + git switch -c + +Or undo this operation with: + + git switch - + +Turn off this advice by setting config variable advice.detachedHead to false + +HEAD is now at 0ac0d6f... 0.1.2 +=== ./without_version (git) === +Cloning into '.'... diff --git a/test/list_extends.repos b/test/list_extends.repos new file mode 100644 index 00000000..2363a679 --- /dev/null +++ b/test/list_extends.repos @@ -0,0 +1,14 @@ +extends: list_extends2.repos +repositories: + immutable/hash_tar: + type: tar + url: https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz + version: vcstool-afb4946c6a96aef37ad7770382b321beff0e0f26 + vcstool-custom: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: 0.2.10 + vcstool-old: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: 0.1.2 diff --git a/test/list_extends2.repos b/test/list_extends2.repos new file mode 100644 index 00000000..eec25143 --- /dev/null +++ b/test/list_extends2.repos @@ -0,0 +1,10 @@ +extends: list.repos +repositories: + immutable/tag: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: 0.2.7 + vcstool-old: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: 0.1.1 diff --git a/test/list_extends_loop.repos b/test/list_extends_loop.repos new file mode 100644 index 00000000..56005762 --- /dev/null +++ b/test/list_extends_loop.repos @@ -0,0 +1,6 @@ +extends: list_extends_loop2.repos +repositories: + vcstool: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: master diff --git a/test/list_extends_loop2.repos b/test/list_extends_loop2.repos new file mode 100644 index 00000000..3c9c994f --- /dev/null +++ b/test/list_extends_loop2.repos @@ -0,0 +1,6 @@ +extends: list_extends_loop.repos +repositories: + vcstool: + type: git + url: https://github.com/dirk-thomas/vcstool.git + version: 0.1.27 diff --git a/test/test_commands.py b/test/test_commands.py index 6231c09f..673f0b09 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -11,6 +11,10 @@ REPOS_FILE_URL = \ 'https://raw.githubusercontent.com/dirk-thomas/vcstool/master/test/list.repos' # noqa: E501 REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos') +REPOS_EXTENDS_FILE = os.path.join( + os.path.dirname(__file__), 'list_extends.repos') +REPOS_EXTENDS_LOOP_FILE = os.path.join( + os.path.dirname(__file__), 'list_extends_loop.repos') TEST_WORKSPACE = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'test_workspace') @@ -321,6 +325,119 @@ def test_status(self): expected = get_expected_output('status') self.assertEqual(output, expected) + def test_import_extends(self): + workdir = os.path.join(TEST_WORKSPACE, 'import-extends') + os.makedirs(workdir) + try: + output = run_command( + 'import', ['--input', REPOS_EXTENDS_FILE, '.'], + subfolder='import-extends') + # the actual output contains absolute paths + output = output.replace( + b'repository in ' + workdir.encode() + b'/', + b'repository in ./') + expected = get_expected_output('import_extends') + # newer git versions don't append three dots after the commit hash + assert output == expected or \ + output == expected.replace(b'... ', b' ') + finally: + rmtree(workdir) + + def test_import_extends_loop(self): + with self.assertRaises(subprocess.CalledProcessError) as e: + run_command( + 'import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.']) + self.assertIn( + b'Infinite loop in repos extensions', e.exception.output) + + def test_import_extends_merge(self): + from vcstool.commands.import_ import merge_repositories + # only one set of repos + repos = [ + { + 'some/path': { + 'type': 'git', + 'url': 'https://github.com/user/repo', + 'version': 'master', + }, + }, + ] + merged_repos = merge_repositories(repos) + self.assertDictEqual(repos[0], merged_repos) + # multiple sets repos + repos = [ + { + 'a/b/c': { + 'type': 'git', + 'url': 'https://github.com/a/bc', + 'version': '1.0.0', + }, + 'd/e': { + 'type': 'hg', + 'url': 'very.old.com', + 'version': '-1', + }, + 'f': { + 'type': 'svn', + 'url': 'https://gitlab.com/f/f', + 'version': 'master', + }, + 'g/h': { + 'type': 'git', + 'url': 'https://gitlab.com/g/h', + 'version': '2.7', + }, + }, + { + 'a/b/c': { + 'type': 'git', + 'url': 'https://github.com/a/bc', + 'version': 'master', + }, + 'i/j': { + 'type': 'git', + 'url': 'https://some.website', + 'version': '42', + }, + }, + { + 'g/h': { + 'type': 'git', + 'url': 'https://gitlab.com/custom/h', + 'version': '2.8', + }, + }, + ] + expected_merged_repos = { + 'a/b/c': { + 'type': 'git', + 'url': 'https://github.com/a/bc', + 'version': 'master', + }, + 'd/e': { + 'type': 'hg', + 'url': 'very.old.com', + 'version': '-1', + }, + 'f': { + 'type': 'svn', + 'url': 'https://gitlab.com/f/f', + 'version': 'master', + }, + 'g/h': { + 'type': 'git', + 'url': 'https://gitlab.com/custom/h', + 'version': '2.8', + }, + 'i/j': { + 'type': 'git', + 'url': 'https://some.website', + 'version': '42', + }, + } + merged_repos = merge_repositories(repos) + self.assertDictEqual(expected_merged_repos, merged_repos) + def run_command(command, args=None, subfolder=None): repo_root = os.path.dirname(os.path.dirname(__file__)) diff --git a/vcstool/commands/import_.py b/vcstool/commands/import_.py index 188b7217..13244fe8 100644 --- a/vcstool/commands/import_.py +++ b/vcstool/commands/import_.py @@ -79,12 +79,61 @@ def file_or_url_type(value): value, headers={'User-Agent': 'vcstool/' + vcstool_version}) -def get_repositories(yaml_file): +def load_yaml_file(yaml_file): try: - root = yaml.safe_load(yaml_file) + return yaml.safe_load(yaml_file) except yaml.YAMLError as e: raise RuntimeError('Input data is not valid yaml format: %s' % e) + +def get_repositories(yaml_file): + root = load_yaml_file(yaml_file) + repos = get_repositories_from_root(root) + if 'extends' not in root or not repos: + return repos + repos_list = [repos] + # If the initial file is passed through --input, consider the extended + # file path as being relative to it. Otherwise, if the initial file is + # passed through stdin, use curdir as the base path. + file_path = os.path.abspath(yaml_file.name) \ + if yaml_file.name != '' else None + base_rel_path = os.path.dirname(file_path) \ + if file_path else os.path.abspath(os.path.curdir) + file_paths = [file_path] if file_path else [] + while 'extends' in root: + extended_file_path = os.path.join(base_rel_path, root['extends']) + if any( + os.path.samefile(extended_file_path, path) + for path in file_paths + ): + raise RuntimeError( + 'Infinite loop in repos extensions: %s' % file_paths) + base_rel_path = os.path.dirname(extended_file_path) + file_paths.append(extended_file_path) + try: + with open(extended_file_path, 'r') as extended_file: + root = load_yaml_file(extended_file) + except IOError: + raise RuntimeError( + 'Could not find extended file: %s' % extended_file_path) + repos_list.append(get_repositories_from_root(root)) + repos_list.reverse() + return merge_repositories(repos_list) + + +def merge_repositories(repositories): + base_repos = repositories[0] + # merge second set of repos into first, then third one into that, etc. + for extension_repos in repositories[1:]: + for path, attributes in extension_repos.items(): + if path in base_repos: + base_repos[path].update(attributes) + else: + base_repos[path] = attributes + return base_repos + + +def get_repositories_from_root(root): try: repositories = root['repositories'] return get_repos_in_vcstool_format(repositories)