Allow fetching repos file from URL (#155)

* Allow fetching repos from url

* Make URL loading happen after arg parsing

* Remove existing test file

* Add test for importing with URL

* Expand on why a non-default user agent is needed

* Roll back to only catch specific exceptions

* Reverse 'file_or_url_type' logic

* Use 'vcstool/__version__' as a user agent

* move new function, change argument name

* revert unrelated default value change

* mention import supporting URLs in the README

Co-authored-by: Dan Rose <dan@digilabs.io>
Co-authored-by: Dirk Thomas <dirk-thomas@users.noreply.github.com>

README.rst

@@ -89,6 +89,7 @@ Usually the data of a previously exported file is piped in::
   vcs import < my.repos
 
 The ``import`` command also supports input in the `rosinstall file format <http://www.ros.org/doc/independent/api/rosinstall/html/rosinstall_file_format.html>`_.
+Beside passing a file path the command also supports passing a URL.
 
 Only for this command vcstool supports the pseudo clients ``tar`` and ``zip`` which fetch a tarball / zipfile from a URL and unpack its content.
 For those two types the ``version`` key is optional.

test/test_commands.py

@@ -8,6 +8,8 @@
 from vcstool.util import rmtree  # noqa: E402
 
 REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list.repos')
+REPOS_FILE_URL = \
+    'https://raw.githubusercontent.com/dirk-thomas/vcstool/master/test/list.repos'  # noqa: E501
 REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos')
 TEST_WORKSPACE = os.path.join(
     os.path.dirname(os.path.dirname(__file__)), 'test_workspace')
@@ -267,6 +269,25 @@ def test_import_shallow(self):
         finally:
             rmtree(workdir)
 
+    def test_import_url(self):
+        workdir = os.path.join(TEST_WORKSPACE, 'import-url')
+        os.makedirs(workdir)
+        try:
+            output = run_command(
+                'import', ['--input', REPOS_FILE_URL, '.'],
+                subfolder='import-url')
+            # the actual output contains absolute paths
+            output = output.replace(
+                b'repository in ' + workdir.encode() + b'/',
+                b'repository in ./')
+            expected = get_expected_output('import')
+            # newer git versions don't append ... after the commit hash
+            assert (
+                output == expected or
+                output == expected.replace(b'... ', b' '))
+        finally:
+            rmtree(workdir)
+
     def test_validate(self):
         output = run_command(
             'validate', ['--input', REPOS_FILE])

vcstool/commands/import_.py

@@ -4,6 +4,7 @@
 import os
 import sys
 
+from vcstool import __version__ as vcstool_version
 from vcstool.clients import vcstool_clients
 from vcstool.clients.vcs_base import run_command
 from vcstool.clients.vcs_base import which
@@ -14,6 +15,11 @@
 from vcstool.streams import set_streams
 import yaml
 
+try:
+    import urllib.request as request
+except ImportError:
+    import urllib2 as request
+
 from .command import add_common_arguments
 from .command import Command
 
@@ -41,7 +47,8 @@ def get_parser():
         description='Import the list of repositories', prog='vcs import')
     group = parser.add_argument_group('"import" command parameters')
     group.add_argument(
-        '--input', type=argparse.FileType('r'), default=sys.stdin)
+        '--input', type=file_or_url_type, default=sys.stdin,
+        help='Where to read YAML from', metavar='FILE_OR_URL')
     group.add_argument(
         '--force', action='store_true', default=False,
         help="Delete existing directories if they don't contain the "
@@ -63,6 +70,15 @@ def get_parser():
     return parser
 
 
+def file_or_url_type(value):
+    if os.path.exists(value) or '://' not in value:
+        return argparse.FileType('r')(value)
+    # use another user agent to avoid getting a 403 (forbidden) error,
+    # since some websites blacklist or block unrecognized user agents
+    return request.Request(
+        value, headers={'User-Agent': 'vcstool/' + vcstool_version})
+
+
 def get_repositories(yaml_file):
     try:
         root = yaml.safe_load(yaml_file)
@@ -190,8 +206,11 @@ def main(args=None, stdout=None, stderr=None):
         path_help='Base path to clone repositories to')
     args = parser.parse_args(args)
     try:
-        repos = get_repositories(args.input)
-    except RuntimeError as e:
+        input_ = args.input
+        if isinstance(input_, request.Request):
+            input_ = request.urlopen(input_)
+        repos = get_repositories(input_)
+    except (RuntimeError, request.URLError) as e:
         print(ansi('redf') + str(e) + ansi('reset'), file=sys.stderr)
         return 1
     jobs = generate_jobs(repos, args)