Skip to content

Commit

Permalink
Add Thunderbird daily default branch handling (#247) (#676)
Browse files Browse the repository at this point in the history
  • Loading branch information
@piri-p
piri-p authored May 8, 2024
1 parent 6f22139 commit 5ccbc47
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 56 deletions.
3 changes: 1 addition & 2 deletions mozdownload/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,8 @@ def parse_arguments(argv):
group = parser.add_argument_group('Daily builds', 'Extra options for daily builds.')
group.add_argument('--branch',
dest='branch',
default='mozilla-central',
metavar='BRANCH',
help='Name of the branch, default: "%(default)s"')
help='Name of the branch')
group.add_argument('--build-id',
dest='build_id',
metavar='BUILD_ID',
Expand Down
16 changes: 14 additions & 2 deletions mozdownload/scraper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@

# Used if the application is named differently than the binary on the server
APPLICATIONS_TO_BINARY_NAME = {'devedition': 'firefox'}
# Used for application-specific default branch
APPLICATIONS_TO_BRANCH = {'firefox': 'mozilla-central',
'thunderbird': 'comm-central'}
# Used when sorting versions
APPLICATIONS_TO_VERSION_CLASS = {'devedition': 'DeveditionVersion',
'firefox': 'FirefoxVersion',
Expand Down Expand Up @@ -59,6 +62,8 @@
# Chunk size when downloading a file
CHUNK_SIZE = 16 * 1024

DEFAULT_BRANCH = 'mozilla-central'

DEFAULT_FILE_EXTENSIONS = {'android-arm64-v8a': 'apk',
'android-armeabi-v7a': 'apk',
'android-x86': 'apk',
Expand Down Expand Up @@ -370,7 +375,7 @@ def show_matching_builds(self, builds):
class DailyScraper(Scraper):
"""Class to download a daily build from the Mozilla server."""

def __init__(self, branch='mozilla-central', build_id=None, date=None,
def __init__(self, branch=None, build_id=None, date=None,
build_number=None, revision=None, *args, **kwargs):
"""Create an instance of the daily scraper."""
self.branch = branch
Expand All @@ -383,6 +388,9 @@ def __init__(self, branch='mozilla-central', build_id=None, date=None,

def get_build_info(self):
"""Define additional build information."""
# Retrieve branch once knowing self.application from Scraper.__init__
if self.branch is None:
self.branch = APPLICATIONS_TO_BRANCH.get(self.application, DEFAULT_BRANCH)
# Retrieve build by revision
if self.revision:
th = treeherder.Treeherder(self.application, self.branch, self.platform)
Expand Down Expand Up @@ -808,7 +816,7 @@ def build_filename(self, binary):
class TinderboxScraper(Scraper):
"""Class to download a tinderbox build of a Gecko based application."""

def __init__(self, branch='mozilla-central', build_number=None, date=None,
def __init__(self, branch=None, build_number=None, date=None,
debug_build=False, revision=None, *args, **kwargs):
"""Create instance of a tinderbox scraper."""
self.branch = branch
Expand All @@ -825,6 +833,10 @@ def __init__(self, branch='mozilla-central', build_number=None, date=None,

def get_build_info(self):
"""Define additional build information."""
# Retrieve branch once knowing self.application from Scraper.__init__

if self.branch is None:
self.branch = APPLICATIONS_TO_BRANCH.get(self.application, DEFAULT_BRANCH)
# Retrieve build by revision
if self.revision:
th = treeherder.Treeherder(self.application, self.branch, self.platform)
Expand Down
58 changes: 30 additions & 28 deletions tests/daily_scraper/test_daily_scraper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,49 +16,49 @@
({'platform': 'win32'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
({'platform': 'win32', 'branch': 'mozilla-central'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
({'platform': 'win64', 'branch': 'mozilla-central'},
({'platform': 'win64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win64.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win64.installer.exe'),
({'platform': 'linux', 'branch': 'mozilla-central'},
({'platform': 'linux'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.tar.bz2'),
({'platform': 'linux64', 'branch': 'mozilla-central'},
({'platform': 'linux64'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-x86_64.tar.bz2',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-x86_64.tar.bz2'),
({'platform': 'mac', 'branch': 'mozilla-central'},
({'platform': 'mac'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.mac.dmg',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.mac.dmg'),
({'platform': 'linux', 'branch': 'mozilla-central', 'extension': 'txt'},
({'platform': 'win32', 'branch': 'mozilla-central'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
({'platform': 'linux', 'extension': 'txt'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.en-US.linux-i686.txt',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/firefox-27.0a1.en-US.linux-i686.txt'),
({'platform': 'win32', 'branch': 'mozilla-central', 'locale': 'it'},
({'platform': 'win32', 'locale': 'it'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.it.win32.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central-l10n/firefox-27.0a1.it.win32.installer.exe'),
({'platform': 'win32', 'branch': 'mozilla-central', 'locale': 'sv-SE'},
({'platform': 'win32', 'locale': 'sv-SE'},
'2013-10-01-03-02-04-mozilla-central-firefox-27.0a1.sv-SE.win32.installer.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central-l10n/firefox-27.0a1.sv-SE.win32.installer.exe'),
({'platform': 'win32', 'branch': 'mozilla-central', 'build_id': '20130706031213'},
({'platform': 'win32', 'build_id': '20130706031213'},
'2013-07-06-03-12-13-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/07/2013-07-06-03-12-13-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
({'platform': 'win32', 'branch': 'mozilla-central', 'date': '2013-07-02'},
({'platform': 'win32', 'date': '2013-07-02'},
'2013-07-02-04-12-13-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/07/2013-07-02-04-12-13-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
({'platform': 'win32', 'branch': 'mozilla-central', 'date': '2013-07-02', 'build_number': 1},
({'platform': 'win32', 'date': '2013-07-02', 'build_number': 1},
'2013-07-02-03-12-13-mozilla-central-firefox-27.0a1.en-US.win32.installer.exe',
'firefox/nightly/2013/07/2013-07-02-03-12-13-mozilla-central/firefox-27.0a1.en-US.win32.installer.exe'),
# Old stub format
({'platform': 'win32', 'branch': 'mozilla-central', 'date': '2013-09-30', 'is_stub_installer': True},
({'platform': 'win32', 'date': '2013-09-30', 'is_stub_installer': True},
'2013-09-30-03-02-04-mozilla-central-firefox-27.0a1.en-US.win32.installer-stub.exe',
'firefox/nightly/2013/09/2013-09-30-03-02-04-mozilla-central/firefox-27.0a1.en-US.win32.installer-stub.exe'),
# Old file name format
({'platform': 'win64', 'branch': 'mozilla-central', 'date': '2013-09-30'},
({'platform': 'win64', 'date': '2013-09-30'},
'2013-09-30-03-02-04-mozilla-central-firefox-27.0a1.en-US.win64-x86_64.installer.exe',
'firefox/nightly/2013/09/2013-09-30-03-02-04-mozilla-central/firefox-27.0a1.en-US.win64-x86_64.installer.exe'),
# New stub format
({'platform': 'win32', 'branch': 'mozilla-central', 'is_stub_installer': True},
({'platform': 'win32', 'is_stub_installer': True},
'2013-10-01-03-02-04-mozilla-central-Firefox Installer.en-US.exe',
'firefox/nightly/2013/10/2013-10-01-03-02-04-mozilla-central/Firefox Installer.en-US.exe'),
({'platform': 'win32', 'branch': 'mozilla-aurora'},
Expand All @@ -70,38 +70,40 @@
]

thunderbird_tests = [
({'application': 'thunderbird', 'platform': 'linux', 'branch': 'comm-central'},
({'application': 'thunderbird', 'platform': 'linux'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
({'application': 'thunderbird', 'platform': 'linux64', 'branch': 'comm-central'},
({'application': 'thunderbird', 'platform': 'linux64'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-x86_64.tar.bz2'),
({'application': 'thunderbird', 'platform': 'mac', 'branch': 'comm-central'},
({'application': 'thunderbird', 'platform': 'mac'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.mac.dmg',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.mac.dmg'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central'},
({'application': 'thunderbird', 'platform': 'win32'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.win32.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win64', 'branch': 'comm-central'},
({'application': 'thunderbird', 'platform': 'win64'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.win64-x86_64.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.win64-x86_64.installer.exe'),
({'application': 'thunderbird', 'platform': 'linux', 'branch': 'comm-central', 'extension': 'txt'},
({'application': 'thunderbird', 'platform': 'linux', 'branch': 'comm-central'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.tar.bz2',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.tar.bz2'),
({'application': 'thunderbird', 'platform': 'linux', 'extension': 'txt'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.en-US.linux-i686.txt',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central/thunderbird-27.0a1.en-US.linux-i686.txt'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central', 'locale': 'it'},
({'application': 'thunderbird', 'platform': 'win32', 'locale': 'it'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.it.win32.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central-l10n/thunderbird-27.0a1.it.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central', 'locale': 'sv-SE'},
({'application': 'thunderbird', 'platform': 'win32', 'locale': 'sv-SE'},
'2013-10-01-03-02-04-comm-central-thunderbird-27.0a1.sv-SE.win32.installer.exe',
'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-central-l10n/thunderbird-27.0a1.sv-SE.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central', 'build_id': '20130710110153'},
({'application': 'thunderbird', 'platform': 'win32', 'build_id': '20130710110153'},
'2013-07-10-11-01-53-comm-central-thunderbird-27.0a1.en-US.win32.installer.exe',
'thunderbird/nightly/2013/07/2013-07-10-11-01-53-comm-central/thunderbird-27.0a1.en-US.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central', 'date': '2013-07-10'},
({'application': 'thunderbird', 'platform': 'win32', 'date': '2013-07-10'},
'2013-07-10-11-01-53-comm-central-thunderbird-27.0a1.en-US.win32.installer.exe',
'thunderbird/nightly/2013/07/2013-07-10-11-01-53-comm-central/thunderbird-27.0a1.en-US.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-central',
'date': '2013-07-10', 'build_number': 1},
({'application': 'thunderbird', 'platform': 'win32', 'date': '2013-07-10', 'build_number': 1},
'2013-07-10-10-01-53-comm-central-thunderbird-27.0a1.en-US.win32.installer.exe',
'thunderbird/nightly/2013/07/2013-07-10-10-01-53-comm-central/thunderbird-27.0a1.en-US.win32.installer.exe'),
({'application': 'thunderbird', 'platform': 'win32', 'branch': 'comm-aurora'},
Expand Down
19 changes: 19 additions & 0 deletions tests/tinderbox_scraper/test_invalid_branch_tinderbox.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import pytest

from mozdownload import TinderboxScraper
import mozdownload.errors as errors

@pytest.mark.parametrize('args', [
({'platform': 'win32', 'branch': 'invalid'}),
])
def test_invalid_branch_tinderbox(httpd, tmpdir, args):
"""Testing download scenarios with invalid branch parameters for TinderboxScraper"""

with pytest.raises(errors.NotFoundError):
TinderboxScraper(destination=str(tmpdir), base_url=httpd.get_url(), **args)
Loading

0 comments on commit 5ccbc47

Please sign in to comment.