Skip to content

Commit

Permalink
Add support for tar.xz archive file formats
Browse files Browse the repository at this point in the history
  • Loading branch information
whimboo committed Dec 11, 2024
1 parent 2128076 commit 0d21467
Show file tree
Hide file tree
Showing 22 changed files with 150 additions and 108 deletions.
37 changes: 28 additions & 9 deletions mozdownload/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,16 @@
'android-armeabi-v7a': 'apk',
'android-x86': 'apk',
'android-x86_64': 'apk',
'linux': 'tar.bz2',
'linux64': 'tar.bz2',
'linux-arm64': 'tar.bz2',
'linux': 'tar.xz',
'linux64': 'tar.xz',
'linux-arm64': 'tar.xz',
'mac': 'dmg',
'mac64': 'dmg',
'win32': 'exe',
'win64': 'exe'}

FALLBACK_FILE_EXTENSIONS = {'tar.xz': r'tar.bz2'}

PLATFORM_FRAGMENTS = {'android-arm64-v8a': r'android-arm64-v8a',
'android-armeabi-v7a': r'android-armeabi-v7a',
'android-x86': r'android-x86',
Expand Down Expand Up @@ -238,6 +240,16 @@ def path(self):
"""Return the path to the build folder."""
return urljoin(self.base_url, self.path_regex)

@property
def extension_regex(self):
extension = self.extension

fallback = FALLBACK_FILE_EXTENSIONS.get(extension)
if fallback:
extension = f'({extension}|{fallback})'

return extension

@property
def path_regex(self):
"""Return the regex for the path to the build folder."""
Expand Down Expand Up @@ -364,6 +376,13 @@ def _download():

return self.filename

def get_file_extension(self, binary):
extension = self.extension
if not binary.endswith(extension):
extension = FALLBACK_FILE_EXTENSIONS.get(extension)

return extension

def show_matching_builds(self, builds):
"""Output the matching builds."""
self.logger.info('Found %s build%s: %s' % (
Expand Down Expand Up @@ -581,7 +600,7 @@ def binary_regex(self):
self.application),
'LOCALE': self.locale,
'PLATFORM': self.platform_regex,
'EXT': self.extension,
'EXT': self.extension_regex,
'STUB': '-stub' if self.is_stub_installer else '',
'STUB_NEW': 'Installer' if self.is_stub_installer else ''}

Expand Down Expand Up @@ -682,7 +701,7 @@ def binary_regex(self):
}
return regex[self.platform] % {
'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, self.application),
'EXT': self.extension,
'EXT': self.extension_regex,
'PLATFORM': self.platform,
'STUB': 'Stub ' if self.is_stub_installer else '',
'STUB_NEW': ' Installer' if self.is_stub_installer else '',
Expand Down Expand Up @@ -717,7 +736,7 @@ def build_filename(self, binary):
'LOCALE': self.locale,
'PLATFORM': self.platform,
'STUB': '-stub' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.get_file_extension(binary)}

def get_build_info(self):
"""Define additional build information."""
Expand Down Expand Up @@ -813,7 +832,7 @@ def build_filename(self, binary):
'LOCALE': self.locale,
'PLATFORM': self.platform,
'STUB': '-stub' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.get_file_extension(binary)}


class TinderboxScraper(Scraper):
Expand Down Expand Up @@ -904,7 +923,7 @@ def binary_regex(self):
'PLATFORM': PLATFORM_FRAGMENTS[self.platform],
'STUB': '-stub' if self.is_stub_installer else '',
'STUB_NEW': 'setup' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.extension_regex}

def build_filename(self, binary):
"""Return the proposed filename with extension for the binary."""
Expand Down Expand Up @@ -1080,7 +1099,7 @@ def binary_regex(self):
'PLATFORM': PLATFORM_FRAGMENTS[self.platform],
'STUB': '-stub' if self.is_stub_installer else '',
'STUB_NEW': 'setup' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.extension_regex}

def build_filename(self, binary):
"""Return the proposed filename with extension for the binary."""
Expand Down
24 changes: 12 additions & 12 deletions tests/daily_scraper/test_daily_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win64.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win64.installer.exe'),
({'platform': 'linux'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.tar.xz'),
({'platform': 'linux64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-x86_64.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-x86_64.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-x86_64.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-x86_64.tar.xz'),
({'platform': 'linux-arm64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-aarch64.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-aarch64.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-aarch64.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-aarch64.tar.xz'),
({'platform': 'mac'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.mac.dmg',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.mac.dmg'),
Expand Down Expand Up @@ -74,11 +74,11 @@

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-x86_64.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-x86_64.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.mac.dmg',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.mac.dmg'),
Expand All @@ -89,8 +89,8 @@
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.win64-x86_64.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.win64-x86_64.installer.exe'),
({'application': 'thunderbird', 'platform': 'linux', 'branch': 'comm-central'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux', 'extension': 'txt'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.txt',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.txt'),
Expand Down
2 changes: 1 addition & 1 deletion tests/daily_scraper/test_revision.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def test_valid_revision(httpd, tmpdir, mocker):
platform='linux',
revision='6b92cb377496')
assert len(scraper.builds) == 1
assert scraper.url == (build_path + 'firefox-27.0a1.en-US.linux-i686.tar.bz2')
assert scraper.url == (build_path + 'firefox-27.0a1.en-US.linux-i686.tar.xz')


def test_invalid_revision(httpd, tmpdir, mocker):
Expand Down
28 changes: 16 additions & 12 deletions tests/release_candidate_scraper/test_release_candidate_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@
from mozdownload import ReleaseCandidateScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
firefox_tests = [
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux64.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux64.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.mac.dmg',
'firefox/candidates/23.0.1-candidates/build3/mac/en-US/Firefox 23.0.1.dmg'),
Expand Down Expand Up @@ -47,12 +46,15 @@
({'application': 'firefox', 'is_stub_installer': True, 'platform': 'win64', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.win64-stub.exe',
'firefox/candidates/23.0.1-candidates/build3/win64/en-US/Firefox Installer.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.linux.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.linux64.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux64.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.mac.dmg',
'thunderbird/candidates/17.0-candidates/build3/mac/en-US/Thunderbird 17.0.dmg'),
Expand All @@ -62,7 +64,9 @@
({'application': 'thunderbird', 'locale': 'de', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0-build3.de.win32.exe',
'thunderbird/candidates/17.0-candidates/build3/win32/de/Thunderbird Setup 17.0.exe'),
])
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests)
def test_scraper(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for ReleaseCandidateScraper"""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@
from mozdownload import ReleaseCandidateScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
firefox_tests = [
({'application': 'firefox', 'platform': 'linux', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.linux.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.linux64.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux64.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.mac.dmg',
'firefox/candidates/23.0.1-candidates/build3/mac/en-US/Firefox 23.0.1.dmg'),
Expand Down Expand Up @@ -59,12 +58,15 @@
({'application': 'firefox', 'platform': 'win64', 'version': 'latest-esr'},
'firefox-24.0esr-build1.en-US.win64.exe',
'firefox/candidates/24.0esr-candidates/build1/win64/en-US/Firefox Setup 24.0esr.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.linux.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.linux64.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux64.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.mac.dmg',
'thunderbird/candidates/17.0-candidates/build3/mac/en-US/Thunderbird 17.0.dmg'),
Expand Down Expand Up @@ -95,7 +97,9 @@
({'application': 'thunderbird', 'platform': 'win32', 'version': 'latest-esr'},
'thunderbird-17.0.1esr-build1.en-US.win32.exe',
'thunderbird/candidates/17.0.1esr-candidates/build1/win32/en-US/Thunderbird Setup 17.0.1esr.exe'),
])
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests)
def test_latest_build(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for latest release candidate builds"""

Expand Down
59 changes: 33 additions & 26 deletions tests/release_scraper/test_release_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,7 @@
from mozdownload import ReleaseScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-armeabi-v7a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
({'application': 'fenix', 'platform': 'android-x86', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86_64.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0', 'locale': 'de'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
firefox_tests = [
({'platform': 'win32', 'version': '23.0.1'},
'firefox-23.0.1.en-US.win32.exe',
'firefox/releases/23.0.1/win32/en-US/Firefox Setup 23.0.1.exe'),
Expand All @@ -39,11 +23,11 @@
'firefox-23.0.1.en-US.win64.exe',
'firefox/releases/23.0.1/win64/en-US/Firefox Setup 23.0.1.exe'),
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1.en-US.linux.tar.bz2',
'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1.en-US.linux.tar.xz',
'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': '23.0.1'},
'firefox-23.0.1.en-US.linux64.tar.bz2',
'firefox/releases/23.0.1/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1.en-US.linux64.tar.xz',
'firefox/releases/23.0.1/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': '23.0.1'},
'firefox-23.0.1.en-US.mac.dmg',
'firefox/releases/23.0.1/mac/en-US/Firefox 23.0.1.dmg'),
Expand All @@ -62,22 +46,45 @@
({'application': 'firefox', 'platform': 'win64', 'is_stub_installer': True, 'version': '23.0.1'},
'firefox-23.0.1.en-US.win64-stub.exe',
'firefox/releases/23.0.1/win64/en-US/Firefox Installer.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0.en-US.win32.exe',
'thunderbird/releases/17.0/win32/en-US/Thunderbird Setup 17.0.exe'),
({'application': 'thunderbird', 'platform': 'linux', 'version': '17.0'},
'thunderbird-17.0.en-US.linux.tar.bz2',
'thunderbird/releases/17.0/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0.en-US.linux.tar.xz',
'thunderbird/releases/17.0/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': '17.0'},
'thunderbird-17.0.en-US.linux64.tar.bz2',
'thunderbird/releases/17.0/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0.en-US.linux64.tar.xz',
'thunderbird/releases/17.0/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': '17.0'},
'thunderbird-17.0.en-US.mac.dmg',
'thunderbird/releases/17.0/mac/en-US/Thunderbird 17.0.dmg'),
({'application': 'thunderbird', 'locale': 'de', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0.de.win32.exe',
'thunderbird/releases/17.0/win32/de/Thunderbird Setup 17.0.exe'),
])
]

fenix_tests = [
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-armeabi-v7a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
({'application': 'fenix', 'platform': 'android-x86', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86_64.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0', 'locale': 'de'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fenix_tests)
def test_release_scraper(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for ReleaseScraper"""

Expand Down
Loading

0 comments on commit 0d21467

Please sign in to comment.