Add 'extends' feature to overlay repos files when importing

test/import_extends.txt

@@ -0,0 +1,89 @@
+........
+=== ./immutable/hash (git) ===
+Cloning into '.'...
+Note: switching to '377d5b3d03c212f015cc832fdb368f4534d0d583'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at 377d5b3... update changelog
+=== ./immutable/hash_tar (tar) ===
+Downloaded tarball from 'https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz' and unpacked it
+=== ./immutable/hash_zip (zip) ===
+Downloaded zipfile from 'https://github.com/dirk-thomas/vcstool/archive/377d5b3d03c212f015cc832fdb368f4534d0d583.zip' and unpacked it
+=== ./immutable/tag (git) ===
+Cloning into '.'...
+Note: switching to '0.2.7'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at 80aadd6... 0.2.7
+=== ./vcstool (git) ===
+Cloning into '.'...
+=== ./vcstool-custom (git) ===
+Cloning into '.'...
+Note: switching to '0.2.10'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at 865990f... 0.2.10
+=== ./vcstool-old (git) ===
+Cloning into '.'...
+Note: switching to '0.1.2'.
+
+You are in 'detached HEAD' state. You can look around, make experimental
+changes and commit them, and you can discard any commits you make in this
+state without impacting any branches by switching back to a branch.
+
+If you want to create a new branch to retain commits you create, you may
+do so (now or later) by using -c with the switch command. Example:
+
+  git switch -c <new-branch-name>
+
+Or undo this operation with:
+
+  git switch -
+
+Turn off this advice by setting config variable advice.detachedHead to false
+
+HEAD is now at 0ac0d6f... 0.1.2
+=== ./without_version (git) ===
+Cloning into '.'...

test/list_extends.repos

@@ -0,0 +1,14 @@
+extends: list_extends2.repos
+repositories:
+  immutable/hash_tar:
+    type: tar
+    url: https://github.com/dirk-thomas/vcstool/archive/afb4946c6a96aef37ad7770382b321beff0e0f26.tar.gz
+    version: vcstool-afb4946c6a96aef37ad7770382b321beff0e0f26
+  vcstool-custom:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: 0.2.10
+  vcstool-old:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: 0.1.2

test/list_extends2.repos

@@ -0,0 +1,10 @@
+extends: list.repos
+repositories:
+  immutable/tag:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: 0.2.7
+  vcstool-old:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: 0.1.1

test/list_extends_loop.repos

@@ -0,0 +1,6 @@
+extends: list_extends_loop2.repos
+repositories:
+  vcstool:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: master

test/list_extends_loop2.repos

@@ -0,0 +1,6 @@
+extends: list_extends_loop.repos
+repositories:
+  vcstool:
+    type: git
+    url: https://github.com/dirk-thomas/vcstool.git
+    version: 0.1.27

test/test_commands.py

@@ -11,6 +11,10 @@
 REPOS_FILE_URL = \
     'https://raw.githubusercontent.com/dirk-thomas/vcstool/master/test/list.repos'  # noqa: E501
 REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos')
+REPOS_EXTENDS_FILE = os.path.join(
+    os.path.dirname(__file__), 'list_extends.repos')
+REPOS_EXTENDS_LOOP_FILE = os.path.join(
+    os.path.dirname(__file__), 'list_extends_loop.repos')
 TEST_WORKSPACE = os.path.join(
     os.path.dirname(os.path.dirname(__file__)), 'test_workspace')
 
@@ -321,6 +325,119 @@ def test_status(self):
         expected = get_expected_output('status')
         self.assertEqual(output, expected)
 
+    def test_import_extends(self):
+        workdir = os.path.join(TEST_WORKSPACE, 'import-extends')
+        os.makedirs(workdir)
+        try:
+            output = run_command(
+                'import', ['--input', REPOS_EXTENDS_FILE, '.'],
+                subfolder='import-extends')
+            # the actual output contains absolute paths
+            output = output.replace(
+                b'repository in ' + workdir.encode() + b'/',
+                b'repository in ./')
+            expected = get_expected_output('import_extends')
+            # newer git versions don't append three dots after the commit hash
+            assert output == expected or \
+                output == expected.replace(b'... ', b' ')
+        finally:
+            rmtree(workdir)
+
+    def test_import_extends_loop(self):
+        with self.assertRaises(subprocess.CalledProcessError) as e:
+            run_command(
+                'import', ['--input', REPOS_EXTENDS_LOOP_FILE, '.'])
+        self.assertIn(
+            b'Infinite loop in repos extensions', e.exception.output)
+
+    def test_import_extends_merge(self):
+        from vcstool.commands.import_ import merge_repositories
+        # only one set of repos
+        repos = [
+            {
+                'some/path': {
+                    'type': 'git',
+                    'url': 'https://github.com/user/repo',
+                    'version': 'master',
+                },
+            },
+        ]
+        merged_repos = merge_repositories(repos)
+        self.assertDictEqual(repos[0], merged_repos)
+        # multiple sets repos
+        repos = [
+            {
+                'a/b/c': {
+                    'type': 'git',
+                    'url': 'https://github.com/a/bc',
+                    'version': '1.0.0',
+                },
+                'd/e': {
+                    'type': 'hg',
+                    'url': 'very.old.com',
+                    'version': '-1',
+                },
+                'f': {
+                    'type': 'svn',
+                    'url': 'https://gitlab.com/f/f',
+                    'version': 'master',
+                },
+                'g/h': {
+                    'type': 'git',
+                    'url': 'https://gitlab.com/g/h',
+                    'version': '2.7',
+                },
+            },
+            {
+                'a/b/c': {
+                    'type': 'git',
+                    'url': 'https://github.com/a/bc',
+                    'version': 'master',
+                },
+                'i/j': {
+                    'type': 'git',
+                    'url': 'https://some.website',
+                    'version': '42',
+                },
+            },
+            {
+                'g/h': {
+                    'type': 'git',
+                    'url': 'https://gitlab.com/custom/h',
+                    'version': '2.8',
+                },
+            },
+        ]
+        expected_merged_repos = {
+            'a/b/c': {
+                'type': 'git',
+                'url': 'https://github.com/a/bc',
+                'version': 'master',
+            },
+            'd/e': {
+                'type': 'hg',
+                'url': 'very.old.com',
+                'version': '-1',
+            },
+            'f': {
+                'type': 'svn',
+                'url': 'https://gitlab.com/f/f',
+                'version': 'master',
+            },
+            'g/h': {
+                'type': 'git',
+                'url': 'https://gitlab.com/custom/h',
+                'version': '2.8',
+            },
+            'i/j': {
+                'type': 'git',
+                'url': 'https://some.website',
+                'version': '42',
+            },
+        }
+        merged_repos = merge_repositories(repos)
+        self.assertDictEqual(expected_merged_repos, merged_repos)
+
 
 def run_command(command, args=None, subfolder=None):
     repo_root = os.path.dirname(os.path.dirname(__file__))

vcstool/commands/import_.py

@@ -79,12 +79,61 @@ def file_or_url_type(value):
         value, headers={'User-Agent': 'vcstool/' + vcstool_version})
 
 
-def get_repositories(yaml_file):
+def load_yaml_file(yaml_file):
     try:
-        root = yaml.safe_load(yaml_file)
+        return yaml.safe_load(yaml_file)
     except yaml.YAMLError as e:
         raise RuntimeError('Input data is not valid yaml format: %s' % e)
 
+
+def get_repositories(yaml_file):
+    root = load_yaml_file(yaml_file)
+    repos = get_repositories_from_root(root)
+    if 'extends' not in root or not repos:
+        return repos
+    repos_list = [repos]
+    # If the initial file is passed through --input, consider the extended
+    # file path as being relative to it. Otherwise, if the initial file is
+    # passed through stdin, use curdir as the base path.
+    file_path = os.path.abspath(yaml_file.name) \
+        if yaml_file.name != '<stdin>' else None
+    base_rel_path = os.path.dirname(file_path) \
+        if file_path else os.path.abspath(os.path.curdir)
+    file_paths = [file_path] if file_path else []
+    while 'extends' in root:
+        extended_file_path = os.path.join(base_rel_path, root['extends'])
+        if any(
+            os.path.samefile(extended_file_path, path)
+            for path in file_paths
+        ):
+            raise RuntimeError(
+                'Infinite loop in repos extensions: %s' % file_paths)
+        base_rel_path = os.path.dirname(extended_file_path)
+        file_paths.append(extended_file_path)
+        try:
+            with open(extended_file_path, 'r') as extended_file:
+                root = load_yaml_file(extended_file)
+        except IOError:
+            raise RuntimeError(
+                'Could not find extended file: %s' % extended_file_path)
+        repos_list.append(get_repositories_from_root(root))
+    repos_list.reverse()
+    return merge_repositories(repos_list)
+
+
+def merge_repositories(repositories):
+    base_repos = repositories[0]
+    # merge second set of repos into first, then third one into that, etc.
+    for extension_repos in repositories[1:]:
+        for path, attributes in extension_repos.items():
+            if path in base_repos:
+                base_repos[path].update(attributes)
+            else:
+                base_repos[path] = attributes
+    return base_repos
+
+
+def get_repositories_from_root(root):
     try:
         repositories = root['repositories']
         return get_repos_in_vcstool_format(repositories)