Skip to content

Commit

Permalink
Add support for tar.xz archive file formats
Browse files Browse the repository at this point in the history
  • Loading branch information
whimboo committed Dec 11, 2024
1 parent 2128076 commit dd6fa71
Show file tree
Hide file tree
Showing 22 changed files with 140 additions and 104 deletions.
23 changes: 18 additions & 5 deletions mozdownload/scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import re
import sys
from datetime import datetime
from pathlib import Path

import mozinfo
import progressbar as pb
Expand Down Expand Up @@ -68,9 +69,9 @@
'android-armeabi-v7a': 'apk',
'android-x86': 'apk',
'android-x86_64': 'apk',
'linux': 'tar.bz2',
'linux64': 'tar.bz2',
'linux-arm64': 'tar.bz2',
'linux': 'tar.(bz2|xz)',
'linux64': 'tar.(bz2|xz)',
'linux-arm64': 'tar.(bz2|xz)',
'mac': 'dmg',
'mac64': 'dmg',
'win32': 'exe',
Expand Down Expand Up @@ -364,6 +365,18 @@ def _download():

return self.filename

def get_file_extension(self, binary):
suffixes = Path(binary).suffixes

if (
suffixes[-1] in [".bz2", ".gz", ".xz", ".zst"] and
suffixes[-2] == ".tar"
):
# Handle known multi-part file extensions for archives
return "".join(suffixes[-2:])[1:]

return suffixes[-1][1:]

def show_matching_builds(self, builds):
"""Output the matching builds."""
self.logger.info('Found %s build%s: %s' % (
Expand Down Expand Up @@ -717,7 +730,7 @@ def build_filename(self, binary):
'LOCALE': self.locale,
'PLATFORM': self.platform,
'STUB': '-stub' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.get_file_extension(binary)}

def get_build_info(self):
"""Define additional build information."""
Expand Down Expand Up @@ -813,7 +826,7 @@ def build_filename(self, binary):
'LOCALE': self.locale,
'PLATFORM': self.platform,
'STUB': '-stub' if self.is_stub_installer else '',
'EXT': self.extension}
'EXT': self.get_file_extension(binary)}


class TinderboxScraper(Scraper):
Expand Down
24 changes: 12 additions & 12 deletions tests/daily_scraper/test_daily_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win64.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win64.installer.exe'),
({'platform': 'linux'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.tar.xz'),
({'platform': 'linux64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-x86_64.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-x86_64.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-x86_64.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-x86_64.tar.xz'),
({'platform': 'linux-arm64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-aarch64.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-aarch64.tar.bz2'),
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-aarch64.tar.xz',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-aarch64.tar.xz'),
({'platform': 'mac'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.mac.dmg',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.mac.dmg'),
Expand Down Expand Up @@ -74,11 +74,11 @@

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-x86_64.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-x86_64.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.mac.dmg',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.mac.dmg'),
Expand All @@ -89,8 +89,8 @@
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.win64-x86_64.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.win64-x86_64.installer.exe'),
({'application': 'thunderbird', 'platform': 'linux', 'branch': 'comm-central'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.xz',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux', 'extension': 'txt'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.txt',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.txt'),
Expand Down
2 changes: 1 addition & 1 deletion tests/daily_scraper/test_revision.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def test_valid_revision(httpd, tmpdir, mocker):
platform='linux',
revision='6b92cb377496')
assert len(scraper.builds) == 1
assert scraper.url == (build_path + 'firefox-27.0a1.en-US.linux-i686.tar.bz2')
assert scraper.url == (build_path + 'firefox-27.0a1.en-US.linux-i686.tar.xz')


def test_invalid_revision(httpd, tmpdir, mocker):
Expand Down
28 changes: 16 additions & 12 deletions tests/release_candidate_scraper/test_release_candidate_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@
from mozdownload import ReleaseCandidateScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
firefox_tests = [
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.linux64.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux64.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.mac.dmg',
'firefox/candidates/23.0.1-candidates/build3/mac/en-US/Firefox 23.0.1.dmg'),
Expand Down Expand Up @@ -47,12 +46,15 @@
({'application': 'firefox', 'is_stub_installer': True, 'platform': 'win64', 'version': '23.0.1'},
'firefox-23.0.1-build3.en-US.win64-stub.exe',
'firefox/candidates/23.0.1-candidates/build3/win64/en-US/Firefox Installer.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.linux.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.linux64.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux64.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': '17.0'},
'thunderbird-17.0-build3.en-US.mac.dmg',
'thunderbird/candidates/17.0-candidates/build3/mac/en-US/Thunderbird 17.0.dmg'),
Expand All @@ -62,7 +64,9 @@
({'application': 'thunderbird', 'locale': 'de', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0-build3.de.win32.exe',
'thunderbird/candidates/17.0-candidates/build3/win32/de/Thunderbird Setup 17.0.exe'),
])
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests)
def test_scraper(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for ReleaseCandidateScraper"""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@
from mozdownload import ReleaseCandidateScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
firefox_tests = [
({'application': 'firefox', 'platform': 'linux', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.linux.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.linux64.tar.bz2',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1-build3.en-US.linux64.tar.xz',
'firefox/candidates/23.0.1-candidates/build3/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': 'latest'},
'firefox-23.0.1-build3.en-US.mac.dmg',
'firefox/candidates/23.0.1-candidates/build3/mac/en-US/Firefox 23.0.1.dmg'),
Expand Down Expand Up @@ -59,12 +58,15 @@
({'application': 'firefox', 'platform': 'win64', 'version': 'latest-esr'},
'firefox-24.0esr-build1.en-US.win64.exe',
'firefox/candidates/24.0esr-candidates/build1/win64/en-US/Firefox Setup 24.0esr.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.linux.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.linux64.tar.bz2',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0-build3.en-US.linux64.tar.xz',
'thunderbird/candidates/17.0-candidates/build3/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': 'latest'},
'thunderbird-17.0-build3.en-US.mac.dmg',
'thunderbird/candidates/17.0-candidates/build3/mac/en-US/Thunderbird 17.0.dmg'),
Expand Down Expand Up @@ -95,7 +97,9 @@
({'application': 'thunderbird', 'platform': 'win32', 'version': 'latest-esr'},
'thunderbird-17.0.1esr-build1.en-US.win32.exe',
'thunderbird/candidates/17.0.1esr-candidates/build1/win32/en-US/Thunderbird Setup 17.0.1esr.exe'),
])
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests)
def test_latest_build(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for latest release candidate builds"""

Expand Down
59 changes: 33 additions & 26 deletions tests/release_scraper/test_release_scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,7 @@
from mozdownload import ReleaseScraper
from mozdownload.utils import urljoin


@pytest.mark.parametrize("args,filename,url", [
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-armeabi-v7a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
({'application': 'fenix', 'platform': 'android-x86', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86_64.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0', 'locale': 'de'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
firefox_tests = [
({'platform': 'win32', 'version': '23.0.1'},
'firefox-23.0.1.en-US.win32.exe',
'firefox/releases/23.0.1/win32/en-US/Firefox Setup 23.0.1.exe'),
Expand All @@ -39,11 +23,11 @@
'firefox-23.0.1.en-US.win64.exe',
'firefox/releases/23.0.1/win64/en-US/Firefox Setup 23.0.1.exe'),
({'application': 'firefox', 'platform': 'linux', 'version': '23.0.1'},
'firefox-23.0.1.en-US.linux.tar.bz2',
'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1.en-US.linux.tar.xz',
'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'linux64', 'version': '23.0.1'},
'firefox-23.0.1.en-US.linux64.tar.bz2',
'firefox/releases/23.0.1/linux-x86_64/en-US/firefox-23.0.1.tar.bz2'),
'firefox-23.0.1.en-US.linux64.tar.xz',
'firefox/releases/23.0.1/linux-x86_64/en-US/firefox-23.0.1.tar.xz'),
({'application': 'firefox', 'platform': 'mac', 'version': '23.0.1'},
'firefox-23.0.1.en-US.mac.dmg',
'firefox/releases/23.0.1/mac/en-US/Firefox 23.0.1.dmg'),
Expand All @@ -62,22 +46,45 @@
({'application': 'firefox', 'platform': 'win64', 'is_stub_installer': True, 'version': '23.0.1'},
'firefox-23.0.1.en-US.win64-stub.exe',
'firefox/releases/23.0.1/win64/en-US/Firefox Installer.exe'),
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0.en-US.win32.exe',
'thunderbird/releases/17.0/win32/en-US/Thunderbird Setup 17.0.exe'),
({'application': 'thunderbird', 'platform': 'linux', 'version': '17.0'},
'thunderbird-17.0.en-US.linux.tar.bz2',
'thunderbird/releases/17.0/linux-i686/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0.en-US.linux.tar.xz',
'thunderbird/releases/17.0/linux-i686/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'linux64', 'version': '17.0'},
'thunderbird-17.0.en-US.linux64.tar.bz2',
'thunderbird/releases/17.0/linux-x86_64/en-US/thunderbird-17.0.tar.bz2'),
'thunderbird-17.0.en-US.linux64.tar.xz',
'thunderbird/releases/17.0/linux-x86_64/en-US/thunderbird-17.0.tar.xz'),
({'application': 'thunderbird', 'platform': 'mac', 'version': '17.0'},
'thunderbird-17.0.en-US.mac.dmg',
'thunderbird/releases/17.0/mac/en-US/Thunderbird 17.0.dmg'),
({'application': 'thunderbird', 'locale': 'de', 'platform': 'win32', 'version': '17.0'},
'thunderbird-17.0.de.win32.exe',
'thunderbird/releases/17.0/win32/de/Thunderbird Setup 17.0.exe'),
])
]

fenix_tests = [
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-armeabi-v7a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
({'application': 'fenix', 'platform': 'android-x86', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'},
'fenix-120.1.0.multi.android-x86_64.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0', 'locale': 'de'},
'fenix-120.1.0.multi.android-arm64-v8a.apk',
'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
]

@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fenix_tests)
def test_release_scraper(httpd, tmpdir, args, filename, url):
"""Testing various download scenarios for ReleaseScraper"""

Expand Down
Loading

0 comments on commit dd6fa71

Please sign in to comment.