[issue_415] Add support for python 3

.appveyor.yml

@@ -1,20 +1,22 @@
 build: false
 
 environment:
-    PYTHON: "C:\\Python27"
-    TOXENV: "py27"
+  matrix:
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7"
+      TOXENV: "py27"
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6"
+      TOXENV: "py36"
 
 init:
-    - ECHO %PYTHON%
-    - ECHO %TOXENV%
-    - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
+  - ECHO %PYTHON%
+  - ECHO %TOXENV%
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%
 
 install:
-    - python -m pip install --upgrade pip
-    # Latest tox 2.3.x is currently busted:
-    # https://bitbucket.org/hpk42/tox/issues/314/tox-command-busted-on-windows
-    - pip install tox==2.2.0
-
+  - python -m pip install --upgrade pip
+  - pip install tox==3.7.0
 
 test_script:
-    - tox
+  - tox

.gitignore

@@ -1,6 +1,8 @@
+.idea/
 .cache/
 .tox/
 *.egg-info
 *.pyc
+.coverage
 build
 dist

.travis.yml

@@ -8,32 +8,38 @@ language: python
 
 matrix:
     include:
-        - python: 2.7
+        - name: "Run test suite on Linux with Python 2.7"
           os: linux
+          python: 2.7
           env: TOXENV=py27
-        - python: 2.7
+        - name: "Run test suite on Linux with Python 3.6"
           os: linux
+          python: 3.6
+          env: TOXENV=py36
+        - name: "Check code style with pylama on Linux - Python 3.6"
+          os: linux
+          python: 3.6
           env: TOXENV=pylama
-        - language: generic
-          os: osx
-          # 7.2 is OS X 10.11.x
+          # 10.1 is OS X 10.13.x
           # https://docs.travis-ci.com/user/languages/objective-c/#Supported-OS-X-iOS-SDK-versions
-          osx_image: xcode7.2
+        - name: "Run test suite on OS X with Python 2.7"
+          os: osx
+          osx_image: xcode10.1
+          language: generic
+          python: 2.7
           env: TOXENV=py27
+        - name: "Run test suite on OS X with Python 3.6"
+          language: generic
+          os: osx
+          env:
+            - TOXENV=py36
+            - HOMEBREW_NO_AUTO_UPDATE=1
 
 install:
-    - if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-        curl -O -s https://bootstrap.pypa.io/get-pip.py;
-        python get-pip.py --user;
-        export PATH=$PATH:~/Library/Python/2.7/bin;
-        pip install --user tox virtualenv;
-      else
-        pip install tox virtualenv;
-      fi
-
+    - ./.travis/install.sh
 
 script:
-    - tox
+    - ./.travis/run.sh
 
 notifications:
     email:

.travis/install.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+set -x
+
+ci_requirements="pip setuptools tox virtualenv"
+
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+    if [[ ${TOXENV} == *"py27"* ]]; then
+        # install pip on the system python
+        curl -O https://bootstrap.pypa.io/get-pip.py
+        python get-pip.py --user
+        python -m pip install --user virtualenv
+        python -m virtualenv .venv/
+    elif [[ ${TOXENV} == *"py3"* ]]; then
+        # install current python3 with homebrew
+        # NOTE: the formula is now named just "python"
+        brew install python
+        command -v python3
+        python3 --version
+        python3 -m pip install virtualenv
+        python3 -m virtualenv .venv/
+    else
+        echo "unsupported $TOXENV: "${TOXENV}
+        exit 1
+    fi
+    source .venv/bin/activate
+fi
+
+python -m pip install $ci_requirements

.travis/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+    source .venv/bin/activate
+fi
+
+tox

mozdownload/parser.py

@@ -7,10 +7,10 @@
 from __future__ import absolute_import, unicode_literals
 
 import re
-import urllib
-from HTMLParser import HTMLParser
 
 import requests
+from six.moves.html_parser import HTMLParser
+from six.moves.urllib.parse import unquote
 
 
 class DirectoryParser(HTMLParser):
@@ -68,7 +68,7 @@ def handle_starttag(self, tag, attrs):
                 # Links look like: /pub/firefox/nightly/2015/
                 # We have to trim the fragment down to the last item. Also to ensure we
                 # always get it, we remove a possible final slash first
-                url = urllib.unquote(attr[1])
+                url = unquote(attr[1])
                 self.active_url = url.rstrip('/').split('/')[-1]
 
                 return

mozdownload/scraper.py

@@ -10,22 +10,21 @@
 import os
 import re
 import sys
-import urllib
 from datetime import datetime
-from urlparse import urlparse
 
 import mozinfo
 import progressbar as pb
 import redo
 import requests
+from six.moves.urllib.parse import quote
+from six.moves.urllib.parse import urlparse
 
 from mozdownload import errors
+from mozdownload import treeherder
 from mozdownload.parser import DirectoryParser
 from mozdownload.timezones import PacificTimezone
-from mozdownload import treeherder
 from mozdownload.utils import urljoin
 
-
 APPLICATIONS = ('devedition', 'firefox', 'fennec', 'thunderbird')
 
 # Some applications contain all locales in a single build
@@ -164,6 +163,7 @@ def _create_directory_parser(self, url):
     @property
     def binary(self):
         """Return the name of the build."""
+
         def _get_binary():
             # Retrieve all entries from the remote virtual folder
             parser = self._create_directory_parser(self.path)
@@ -195,8 +195,8 @@ def binary_regex(self):
     @property
     def url(self):
         """Return the URL of the build."""
-        return urllib.quote(urljoin(self.path, self.binary),
-                            safe='%/:=&?~#+!$,;\'@()*[]|')
+        return quote(urljoin(self.path, self.binary),
+                     safe='%/:=&?~#+!$,;\'@()*[]|')
 
     @property
     def path(self):
@@ -248,6 +248,7 @@ def detect_platform(self):
 
     def download(self):
         """Download the specified file."""
+
         def total_seconds(td):
             # Keep backward compatibility with Python 2.6 which doesn't have
             # this method
@@ -417,7 +418,7 @@ def get_latest_build_date(self):
         parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex)
         if not parser.entries:
             message = 'Status file for %s build cannot be found' % \
-                self.platform_regex
+                      self.platform_regex
             raise errors.NotFoundError(message, url)
 
         # Read status file for the platform, retrieve build id,
@@ -467,13 +468,13 @@ def get_build_info_for_date(self, date, build_index=None):
             # ensure to select the correct subfolder for localized builds
             'L10N': '(-l10n)?' if self.locale_build else '',
             'PLATFORM': '' if self.application not in (
-                        'fennec') else '-' + self.platform
+                'fennec') else '-' + self.platform
         }
 
         parser.entries = parser.filter(regex)
         parser.entries = parser.filter(self.is_build_dir)
 
-        if has_time:
+        if date.strftime('%H-%M-%S') != '00-00-00':
             # If a time is included in the date, use it to determine the
             # build's index
             regex = r'.*%s.*' % date.strftime('%H-%M-%S')
@@ -482,7 +483,7 @@ def get_build_info_for_date(self, date, build_index=None):
         if not parser.entries:
             date_format = '%Y-%m-%d-%H-%M-%S' if has_time else '%Y-%m-%d'
             message = 'Folder for builds on %s has not been found' % \
-                self.date.strftime(date_format)
+                      self.date.strftime(date_format)
             raise errors.NotFoundError(message, url)
 
         # If no index has been given, set it to the last build of the day.
@@ -663,8 +664,8 @@ def query_versions(self, version=None):
         parser = self._create_directory_parser(url)
         if version:
             versions = parser.filter(RELEASE_AND_CANDIDATE_LATEST_VERSIONS[version])
-            from distutils.version import LooseVersion
-            versions.sort(key=LooseVersion)
+            from packaging.version import LegacyVersion
+            versions.sort(key=LegacyVersion)
             return [versions[-1]]
         else:
             return parser.entries
@@ -690,7 +691,7 @@ def get_build_info(self):
         parser = self._create_directory_parser(url)
         if not parser.entries:
             message = 'Folder for specific candidate builds at %s has not' \
-                'been found' % url
+                      'been found' % url
             raise errors.NotFoundError(message, url)
 
         self.show_matching_builds(parser.entries)
@@ -909,11 +910,11 @@ def get_build_info_for_index(self, build_index=None):
             # If a timestamp is given, retrieve the folder with the timestamp
             # as name
             parser.entries = self.timestamp in parser.entries and \
-                [self.timestamp]
+                             [self.timestamp]
 
         elif self.date:
             # If date is given, retrieve the subset of builds on that date
-            parser.entries = filter(self.date_matches, parser.entries)
+            parser.entries = list(filter(self.date_matches, parser.entries))
 
         if not parser.entries:
             message = 'No builds have been found'

mozdownload/treeherder.py

@@ -7,12 +7,12 @@
 from __future__ import absolute_import, unicode_literals
 
 import logging
+import six
 
 from thclient import TreeherderClient
 
 from mozdownload.errors import NotSupportedError
 
-
 PLATFORM_MAP = {
     'android-api-9': {'build_platform': 'android-2-3-armv7-api9'},
     'android-api-11': {'build_platform': 'android-4-0-armv7-api11'},
@@ -69,11 +69,11 @@ def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=
 
         try:
             self.logger.info('Querying {url} for list of builds for revision: {revision}'.format(
-                             url=self.client.server_url, revision=revision))
+                url=self.client.server_url, revision=revision))
 
             # Retrieve the option hash to filter for type of build (opt, and debug for now)
             option_hash = None
-            for key, values in self.client.get_option_collection_hash().iteritems():
+            for key, values in six.iteritems(self.client.get_option_collection_hash()):
                 for value in values:
                     if value['name'] == ('debug' if debug_build else 'opt'):
                         option_hash = key

requirements.txt

@@ -1,5 +1,6 @@
-mozinfo >= 0.9
+mozinfo >= 1.0.0
 progressbar2 >= 3.34.3
 redo==2.0.2
-requests >= 2.9.1, <3.0.0
-treeherder-client==5.0.0
+requests >= 2.21.0, <3.0.0
+treeherder-client >= 5.0.0, <6.0.0
+packaging == 19.0

tests/cli/test_cli_arguments.py

@@ -13,4 +13,4 @@ def test_unrecognized_argument():
                                          stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
         output = e.output
-    assert re.search(r'mozdownload: error: unrecognized arguments: --abc', output) is not None
+    assert re.search(r'mozdownload: error: unrecognized arguments: --abc'.encode('utf-8'), output) is not None

tests/cli/test_cli_print_url.py

@@ -17,4 +17,4 @@ def test_print_url_argument():
         output = subprocess.check_output(args, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:
         output = e.output
-    assert re.search(url, output) is not None
+    assert re.search(url.encode('utf-8'), output) is not None

tests/cli/test_correct_scraper.py

@@ -37,6 +37,8 @@
         'fname': '8fcac92cfcad-firefox-38.0a1.en-US.win32.installer.exe',
     },
 }
+
+
 @pytest.mark.parametrize("data", tests.values())
 def test_correct_cli_scraper(httpd, tmpdir, data, mocker):
     query_builds_by_revision = mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')

tests/cli/test_output.py

@@ -1,3 +1,9 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import subprocess
 
 from mozdownload import __version__, cli
@@ -6,4 +12,4 @@
 def test_cli_executes():
     """Test that cli will start and print usage message"""
     output = subprocess.check_output(['mozdownload', '--help'])
-    assert cli.__doc__.format(__version__) in output
+    assert cli.__doc__.format(__version__) in output.decode("utf-8")