This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import sys | |
config = { | |
"virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll', | |
"virtualenv_path": 'venv', | |
"exes": { | |
'python': 'c:/mozilla-build/python27/python', | |
'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'], | |
'hg': 'c:/mozilla-build/hg/hg', | |
'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(), | |
'%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()], | |
'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'], | |
}, | |
"find_links": [ | |
"http://pypi.pvt.build.mozilla.org/pub", | |
"http://pypi.pub.build.mozilla.org/pub", | |
], | |
"pip_index": False, | |
"buildbot_json_path": "buildprops.json", | |
"default_actions": [ | |
'clobber', | |
'read-buildbot-config', | |
'checkout', | |
'download-and-extract', | |
'create-virtualenv', | |
'install', | |
'run-media-tests', | |
], | |
"default_blob_upload_servers": [ | |
"https://blobupload.elasticbeanstalk.com", | |
], | |
"blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"), | |
"in_tree_config": "config/mozharness/marionette.py", | |
"download_minidump_stackwalk": True, | |
"download_symbols": "ondemand", | |
"firefox_media_repo": 'https://github.com/mjzffr/firefox-media-tests.git', | |
"firefox_media_branch": 'master', | |
"firefox_media_rev": 'ae05e246c2ebb1451b12275cbe69a2e3971bc811', | |
"firefox_ui_repo": 'https://github.com/mozilla/firefox-ui-tests.git', | |
"firefox_ui_branch": 'master', | |
"firefox_ui_rev": '7bfdb3e50a92261f177faf95bb6cb2e727229e51', | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# ***** BEGIN LICENSE BLOCK ***** | |
# This Source Code Form is subject to the terms of the Mozilla Public | |
# License, v. 2.0. If a copy of the MPL was not distributed with this | |
# file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
# ***** BEGIN LICENSE BLOCK ***** | |
"""firefox_media_tests.py | |
Author: Maja Frydrychowicz | |
""" | |
import copy | |
import os | |
import re | |
from mozharness.base.log import ERROR, WARNING | |
from mozharness.base.script import PreScriptAction | |
from mozharness.mozilla.testing.testbase import (TestingMixin, | |
testing_config_options) | |
from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper | |
from mozharness.mozilla.vcstools import VCSToolsScript | |
BUSTED = 'busted' | |
TESTFAILED = 'testfailed' | |
UNKNOWN = 'unknown' | |
EXCEPTION = 'exception' | |
SUCCESS = 'success' | |
class JobResultParser(TestSummaryOutputParserHelper): | |
""" Parses test output to determine overall result.""" | |
def __init__(self, **kwargs): | |
super(JobResultParser, self).__init__(**kwargs) | |
self.return_code = 0 | |
# External-resource errors that should not count as test failures | |
self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*' | |
r'TimeoutException: Error loading page,' | |
r' timed out') | |
self.exceptions = [] | |
def parse_single_line(self, line): | |
super(JobResultParser, self).parse_single_line(line) | |
if self.exception_re.match(line): | |
self.exceptions.append(line) | |
@property | |
def status(self): | |
status = UNKNOWN | |
if self.passed and self.failed == 0: | |
status = SUCCESS | |
elif self.exceptions: | |
status = EXCEPTION | |
elif self.failed: | |
status = TESTFAILED | |
elif self.return_code: | |
status = BUSTED | |
return status | |
class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript): | |
job_result_parser = None | |
error_list = [ | |
{'substr': 'FAILED (errors=', 'level': WARNING}, | |
{'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR}, | |
{'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR}, | |
{'substr': 'Timeout waiting for marionette on port', 'level': ERROR}, | |
{'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH|CRASH|ERROR|FAIL)'''), 'level': ERROR}, | |
{'regex': re.compile(r'''(\b\w*Exception)'''), 'level': ERROR}, | |
{'regex': re.compile(r'''(\b\w*Error)'''), 'level': ERROR}, | |
] | |
config_options = [ | |
[["--media-urls"], | |
{"action": "store", | |
"dest": "media_urls", | |
"help": "Path to ini file that lists media urls for tests.", | |
}], | |
[["--profile"], | |
{"action": "store", | |
"dest": "profile", | |
"default": None, | |
"help": "Path to FF profile that should be used by Marionette", | |
}], | |
[["--test-timeout"], | |
{"action": "store", | |
"dest": "test_timeout", | |
"default": 10000, | |
"help": ("Number of seconds without output before" | |
"firefox-media-tests is killed." | |
"Set this based on expected time for all media to play."), | |
}], | |
[["--tests"], | |
{"action": "store", | |
"dest": "tests", | |
"default": None, | |
"help": ("Test(s) to run. Path to test_*.py or " | |
"test manifest (*.ini)"), | |
}], | |
[["--e10s"], | |
{"dest": "e10s", | |
"action": "store_true", | |
"default": False, | |
"help": "Enable e10s when running marionette tests." | |
}], | |
[['--firefox-media-repo'], { | |
'dest': 'firefox_media_repo', | |
'default': 'https://github.com/mjzffr/firefox-media-tests.git', | |
'help': 'which firefox_media_tests repo to use', | |
}], | |
[['--firefox-media-branch'], { | |
'dest': 'firefox_media_branch', | |
'default': 'master', | |
'help': 'which branch to use for firefox_media_tests', | |
}], | |
[['--firefox-media-rev'], { | |
'dest': 'firefox_media_rev', | |
'default': 'ae05e246c2ebb1451b12275cbe69a2e3971bc811', | |
'help': 'which firefox_media_tests revision to use', | |
}], | |
[['--firefox-ui-repo'], { | |
'dest': 'firefox_ui_repo', | |
'default': 'https://github.com/mozilla/firefox-ui-tests.git', | |
'help': 'which firefox_ui_tests repo to use', | |
}], | |
[['--firefox-ui-branch'], { | |
'dest': 'firefox_ui_branch', | |
'default': 'master', | |
'help': 'which branch to use for firefox_ui_tests', | |
}], | |
[['--firefox-ui-rev'], { | |
'dest': 'firefox_ui_rev', | |
'default': '7bfdb3e50a92261f177faf95bb6cb2e727229e51', | |
'help': 'which firefox_ui_tests revision to use', | |
}], | |
] + (copy.deepcopy(testing_config_options)) | |
def __init__(self, config_options=[], all_actions=[], **kwargs): | |
self.config_options += config_options | |
if not all_actions: | |
all_actions = [ | |
'clobber', | |
'checkout', | |
'create-virtualenv', | |
'run-media-tests', | |
] | |
super(FirefoxMediaTestsBase, self).__init__( | |
config_options=self.config_options, | |
all_actions=all_actions, | |
**kwargs | |
) | |
c = self.config | |
self.media_urls = c.get('media_urls') | |
self.profile = c.get('profile') | |
self.test_timeout = int(c.get('test_timeout')) | |
self.tests = c.get('tests') | |
self.e10s = c.get('e10s') | |
@PreScriptAction('create-virtualenv') | |
def _pre_create_virtualenv(self, action): | |
dirs = self.query_abs_dirs() | |
requirements_file = os.path.join(dirs['firefox_media_dir'], | |
'requirements.txt') | |
if os.path.isfile(requirements_file): | |
self.register_virtualenv_module(requirements=[requirements_file]) | |
self.register_virtualenv_module(name='firefox-ui-tests', | |
url=dirs['firefox_ui_dir']) | |
self.register_virtualenv_module(name='firefox-media-tests', | |
url=dirs['firefox_media_dir']) | |
def query_abs_dirs(self): | |
if self.abs_dirs: | |
return self.abs_dirs | |
abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs() | |
dirs = { | |
'firefox_media_dir': os.path.join(abs_dirs['abs_work_dir'], | |
'firefox-media-tests') | |
} | |
dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'], | |
'firefox-ui-tests') | |
abs_dirs.update(dirs) | |
self.abs_dirs = abs_dirs | |
return self.abs_dirs | |
@PreScriptAction('checkout') | |
def _pre_checkout(self, action): | |
super(FirefoxMediaTestsBase, self)._pre_checkout(action) | |
c = self.config | |
dirs = self.query_abs_dirs() | |
self.firefox_media_vc = { | |
'branch': c['firefox_media_branch'], | |
'repo': c['firefox_media_repo'], | |
'revision': c['firefox_media_rev'], | |
'dest': dirs['firefox_media_dir'], | |
} | |
self.firefox_ui_vc = { | |
'branch': c['firefox_ui_branch'], | |
'repo': c['firefox_ui_repo'], | |
'revision': c['firefox_ui_rev'], | |
'dest': dirs['firefox_ui_dir'] | |
} | |
def checkout(self): | |
revision = self.vcs_checkout(vcs='gittool', **self.firefox_media_vc) | |
if revision: | |
self.vcs_checkout(vcs='gittool', **self.firefox_ui_vc) | |
def _query_cmd(self): | |
""" Determine how to call firefox-media-tests """ | |
if not self.binary_path: | |
self.fatal("Binary path could not be determined. " | |
"Should be set by default during 'install' action.") | |
cmd = ['firefox-media-tests'] | |
cmd += ['--binary', self.binary_path] | |
if self.symbols_path: | |
cmd += ['--symbols-path', self.symbols_path] | |
if self.media_urls: | |
cmd += ['--urls', self.media_urls] | |
if self.profile: | |
cmd += ['--profile', self.profile] | |
if self.tests: | |
cmd.append(self.tests) | |
if self.e10s: | |
cmd.append('--e10s') | |
return cmd | |
def run_media_tests(self): | |
cmd = self._query_cmd() | |
self.job_result_parser = JobResultParser( | |
config=self.config, | |
log_obj=self.log_obj, | |
error_list=self.error_list | |
) | |
return_code = self.run_command( | |
cmd, | |
output_timeout=self.test_timeout, | |
output_parser=self.job_result_parser | |
) | |
self.job_result_parser.return_code = return_code | |
return self.job_result_parser.status |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# ***** BEGIN LICENSE BLOCK ***** | |
# This Source Code Form is subject to the terms of the Mozilla Public | |
# License, v. 2.0. If a copy of the MPL was not distributed with this | |
# file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
# ***** BEGIN LICENSE BLOCK ***** | |
"""firefox_media_tests_buildbot.py | |
Author: Maja Frydrychowicz | |
""" | |
import copy | |
import glob | |
import os | |
import sys | |
sys.path.insert(1, os.path.dirname(sys.path[0])) | |
from mozharness.base.log import ERROR, DEBUG, INFO | |
from mozharness.base.script import PreScriptAction, PostScriptAction | |
from mozharness.mozilla.blob_upload import ( | |
BlobUploadMixin, | |
blobupload_config_options | |
) | |
from mozharness.mozilla.buildbot import ( | |
TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE | |
) | |
from mozharness.mozilla.testing.firefox_media_tests import ( | |
FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS | |
) | |
class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin): | |
def __init__(self): | |
config_options = copy.deepcopy(blobupload_config_options) | |
super(FirefoxMediaTestsBuildbot, self).__init__( | |
config_options=config_options, | |
all_actions=['clobber', | |
'read-buildbot-config', | |
'checkout', | |
'download-and-extract', | |
'create-virtualenv', | |
'install', | |
'run-media-tests', | |
], | |
) | |
c = self.config | |
self.installer_url = c.get('installer_url') | |
self.installer_path = c.get('installer_path') | |
self.binary_path = c.get('binary_path') | |
self.test_packages_url = c.get('test_packages_url') | |
@PreScriptAction('create-virtualenv') | |
def _pre_create_virtualenv(self, action): | |
dirs = self.query_abs_dirs() | |
requirements = os.path.join(dirs['abs_test_install_dir'], | |
'config', | |
'marionette_requirements.txt') | |
if os.access(requirements, os.F_OK): | |
self.register_virtualenv_module(requirements=[requirements], | |
two_pass=True) | |
super(FirefoxMediaTestsBuildbot, self)._pre_create_virtualenv(action) | |
def query_abs_dirs(self): | |
if self.abs_dirs: | |
return self.abs_dirs | |
dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs() | |
dirs['abs_blob_upload_dir'] = os.path.join(dirs['abs_work_dir'], | |
'blobber_upload_dir') | |
dirs['abs_test_install_dir'] = os.path.join(dirs['abs_work_dir'], | |
'tests') | |
self.abs_dirs = dirs | |
return self.abs_dirs | |
def _query_cmd(self): | |
""" Determine how to call firefox-media-tests """ | |
cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd() | |
# configure logging | |
dirs = self.query_abs_dirs() | |
blob_upload_dir = dirs.get('abs_blob_upload_dir') | |
cmd += ['--gecko-log', os.path.join(blob_upload_dir, | |
'gecko.log')] | |
cmd += ['--log-html', os.path.join(blob_upload_dir, | |
'media_tests.html')] | |
cmd += ['--log-mach', os.path.join(blob_upload_dir, | |
'media_tests_mach.log')] | |
return cmd | |
def run_media_tests(self): | |
status = super(FirefoxMediaTestsBuildbot, self).run_media_tests() | |
if status == SUCCESS: | |
tbpl_status = TBPL_SUCCESS | |
else: | |
tbpl_status = TBPL_FAILURE | |
if status == TESTFAILED: | |
tbpl_status = TBPL_WARNING | |
self.buildbot_status(tbpl_status) | |
@PostScriptAction('run-media-tests') | |
def _collect_uploads(self, action, success=None): | |
""" Copy extra (log) files to blob upload dir. """ | |
dirs = self.query_abs_dirs() | |
log_dir = dirs.get('abs_log_dir') | |
blob_upload_dir = dirs.get('abs_blob_upload_dir') | |
if not log_dir or not blob_upload_dir: | |
return | |
self.mkdir_p(blob_upload_dir) | |
# Move firefox-media-test screenshots into log_dir | |
screenshots_dir = os.path.join(dirs['base_work_dir'], | |
'screenshots') | |
log_screenshots_dir = os.path.join(log_dir, 'screenshots') | |
if os.access(log_screenshots_dir, os.F_OK): | |
self.rmtree(log_screenshots_dir) | |
if os.access(screenshots_dir, os.F_OK): | |
self.move(screenshots_dir, log_screenshots_dir) | |
# logs to upload: broadest level (info), error, screenshots | |
uploads = glob.glob(os.path.join(log_screenshots_dir, '*')) | |
log_files = self.log_obj.log_files | |
log_level = self.log_obj.log_level | |
def append_path(filename, dir=log_dir): | |
if filename: | |
uploads.append(os.path.join(dir, filename)) | |
append_path(log_files.get(ERROR)) | |
# never upload debug logs | |
if log_level == DEBUG: | |
append_path(log_files.get(INFO)) | |
else: | |
append_path(log_files.get(log_level)) | |
# in case of SimpleFileLogger | |
append_path(log_files.get('default')) | |
for f in uploads: | |
if os.access(f, os.F_OK): | |
dest = os.path.join(blob_upload_dir, os.path.basename(f)) | |
self.copyfile(f, dest) | |
if __name__ == '__main__': | |
media_test = FirefoxMediaTestsBuildbot() | |
media_test.run_and_exit() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# ***** BEGIN LICENSE BLOCK ***** | |
# This Source Code Form is subject to the terms of the Mozilla Public | |
# License, v. 2.0. If a copy of the MPL was not distributed with this | |
# file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
# ***** BEGIN LICENSE BLOCK ***** | |
"""firefox_media_tests_jenkins.py | |
Requirements for running this in Jenkins: | |
- working directory ("Jenkins workspace") contains clone of | |
firefox-media-tests repo, including submodule(s) (unless you use | |
checkout action) | |
- virtualenv, pip | |
- Environment variables: | |
- MOZHARNESSHOME: path to mozharness source dir | |
- MINIDUMP_STACKWALK: path to minidump_stackwalk binary | |
- TREEHERDER_CONFIG: path to credentials files | |
- On Windows, the mozilla-build system | |
- Treeherder-related actions require: | |
- treeherding.py | |
- s3.py | |
- parsers.py | |
Author: Maja Frydrychowicz | |
""" | |
import copy | |
import os | |
import sys | |
mozharnesspath = os.environ.get('MOZHARNESSHOME') | |
if mozharnesspath: | |
sys.path.insert(1, mozharnesspath) | |
else: | |
print 'MOZHARNESSHOME not set' | |
from mozharness.base.log import (SimpleFileLogger, MultiFileLogger, | |
DEBUG, ERROR, INFO) | |
from mozharness.base.script import PreScriptAction, PostScriptAction | |
from mozharness.mozilla.testing.firefox_media_tests import ( | |
FirefoxMediaTestsBase, BUSTED, TESTFAILED, UNKNOWN, EXCEPTION, SUCCESS, | |
) | |
from mozharness.mozilla.testing.treeherder.treeherder_mixin import ( | |
TreeherderMixin, | |
treeherder_config_options | |
) | |
class FirefoxMediaTestsJenkins(TreeherderMixin, FirefoxMediaTestsBase): | |
def __init__(self): | |
config_options = [ | |
[["--symbols-url"], | |
{"action": "store", | |
"dest": "symbols_url", | |
"default": None, | |
"help": "URL to the crashreporter-symbols.zip", | |
}], | |
[["--jenkins-build-tag"], | |
{"action": "store", | |
"dest": "jenkins_build_tag", | |
"default": os.environ.get('BUILD_TAG', ''), | |
"help": "$BUILD_TAG in shell Jenkins build step", | |
}], | |
[["--jenkins-build-url"], | |
{"action": "store", | |
"dest": "jenkins_build_url", | |
"default": os.environ.get('BUILD_URL', ''), | |
"help": "$BUILD_URL in shell Jenkins build step", | |
}], | |
[["--log-date-format"], | |
{"action": "store", | |
"dest": "log_date_format", | |
"default": None, | |
"help": r"Default: '%H:%M:%S'", | |
}], | |
] + copy.deepcopy(treeherder_config_options) | |
super(FirefoxMediaTestsJenkins, self).__init__( | |
config_options=config_options, | |
all_actions=['clobber', | |
'checkout', | |
'download-and-extract', | |
'create-virtualenv', | |
'install', | |
'submit-treeherder-running', | |
'run-media-tests', | |
'submit-treeherder-complete', | |
], | |
default_actions=['clobber', | |
'download-and-extract', | |
'create-virtualenv', | |
'install', | |
'submit-treeherder-running', | |
'run-media-tests', | |
'submit-treeherder-complete', | |
], | |
) | |
c = self.config | |
self.installer_url = c.get('installer_url') | |
self.symbols_url = c.get('symbols_url') | |
# logs to upload with TreeherderMixin | |
self.media_logs = {'gecko.log'} | |
# Allow config to set log_date_format; %H:%M:%S is useful for log parsing | |
def new_log_obj(self, default_log_level="info"): | |
c = self.config | |
log_dir = os.path.join(c['base_work_dir'], c.get('log_dir', 'logs')) | |
log_config = { | |
"logger_name": 'Simple', | |
"log_name": 'log', | |
"log_dir": log_dir, | |
"log_level": default_log_level, | |
"log_format": '%(asctime)s %(levelname)8s - %(message)s', | |
"log_to_console": True, | |
"append_to_log": False, | |
} | |
# This is the only difference with overridden method | |
if c.get('log_date_format'): | |
log_config['log_date_format'] = c['log_date_format'] | |
log_type = self.config.get("log_type", "multi") | |
for key in log_config.keys(): | |
value = self.config.get(key, None) | |
if value is not None: | |
log_config[key] = value | |
if log_type == "multi": | |
self.log_obj = MultiFileLogger(**log_config) | |
else: | |
self.log_obj = SimpleFileLogger(**log_config) | |
@PostScriptAction() | |
def log_action_completed(self, action, success=None): | |
""" Record end of each action to simplify parsing log into steps """ | |
msg = '##### Finished %s step. Success: %s' % (action, success) | |
if action == 'run_marionette_tests' and self.job_result_parser: | |
msg += ' - Result: %s' % (self.job_result_parser.status or UNKNOWN) | |
self.info(msg) | |
def query_abs_dirs(self): | |
if self.abs_dirs: | |
return self.abs_dirs | |
abs_dirs = super(FirefoxMediaTestsJenkins, self).query_abs_dirs() | |
if 'checkout' not in self.actions: | |
# assume that project was previously checked out by Jenkins | |
dirs = { | |
'firefox_media_dir': os.path.join(abs_dirs['base_work_dir']) | |
} | |
dirs['firefox_ui_dir'] = os.path.join(dirs['firefox_media_dir'], | |
'firefox-ui-tests') | |
abs_dirs.update(dirs) | |
self.abs_dirs = abs_dirs | |
return self.abs_dirs | |
def preflight_download_and_extract(self): | |
super(FirefoxMediaTestsJenkins, self).preflight_download_and_extract() | |
message = '' | |
if not self.symbols_url: | |
message += ("symbols-url isn't set!\n" | |
"You can set this by specifying --symbols-url URL.\n") | |
if message: | |
self.fatal(message + "Can't run download-and-extract... exiting") | |
def download_and_extract(self, *args, **kwargs): | |
""" | |
download and extract test zip / download installer | |
overriding TestingMixin's implementation to be able to skip | |
_download_test_zip and _read_tree_config | |
""" | |
# Swap plain http for https when we're downloading from ftp | |
# See bug 957502 and friends | |
from_ = "http://ftp.mozilla.org" | |
to_ = "https://ftp-ssl.mozilla.org" | |
for attr in 'test_url', 'symbols_url', 'installer_url': | |
url = getattr(self, attr) | |
if url and url.startswith(from_): | |
new_url = url.replace(from_, to_) | |
self.info("Replacing url %s -> %s" % (url, new_url)) | |
setattr(self, attr, new_url) | |
self._download_installer() | |
self._download_and_extract_symbols() | |
def _query_cmd(self): | |
""" Determine how to call firefox-media-tests """ | |
cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd() | |
# configure logging | |
cmd += ['--log-tbpl', '-'] | |
dirs = self.query_abs_dirs() | |
log_dir = dirs['abs_log_dir'] | |
cmd += ['--gecko-log', os.path.join(log_dir, 'gecko.log')] | |
self.media_logs.add('gecko.log') | |
cmd += ['--log-html', os.path.join(log_dir, 'media_tests.html')] | |
self.media_logs.add('media_tests.html') | |
cmd += ['--log-mach', os.path.join(log_dir, 'media_tests_mach.log')] | |
self.media_logs.add('media_tests_mach.log') | |
return cmd | |
def run_media_tests(self): | |
status = super(FirefoxMediaTestsJenkins, self).run_media_tests() | |
if status == SUCCESS: | |
self.info("Marionette: %s" % status) | |
else: | |
self.error("Marionette: %s" % status) | |
@PostScriptAction('run-media-tests') | |
def _collect_screenshots(self, action, success=None): | |
dirs = self.query_abs_dirs() | |
log_dir = dirs.get('abs_log_dir') | |
if not log_dir: | |
return | |
screenshots_dir = os.path.join(dirs['base_work_dir'], 'screenshots') | |
old_screenshots_dir = os.path.join(log_dir, 'screenshots') | |
if os.access(old_screenshots_dir, os.F_OK): | |
self.rmtree(old_screenshots_dir) | |
if os.access(screenshots_dir, os.F_OK): | |
self.move(screenshots_dir, old_screenshots_dir) | |
@PostScriptAction('create-virtualenv') | |
def setup_treeherding(self, action, success=None): | |
if not success: | |
return | |
if self.config['treeherding_off']: | |
self.info(self.off_message) | |
return | |
super(FirefoxMediaTestsJenkins, self).setup_treeherding(action, success) | |
from mozharness.mozilla.testing.treeherder.treeherding import TestJob | |
class JenkinsJob(TestJob): | |
def __init__(self, **kwargs): | |
super(JenkinsJob, self).__init__(**kwargs) | |
self.jenkins_build_tag = '' # computed | |
self.jenkins_build_url = '' # computed | |
@property | |
def unique_s3_prefix(self): | |
# e.g. mozilla-aurora/aurora/mac/x86_64/20150520030205/ | |
# jenkins-webrtc-aurora-mac-nightly-win64-529/somesuffix | |
if not self.jenkins_build_tag: | |
return super(JenkinsJob, self).unique_s3_prefix | |
prefix = ('{0}/{1}/{2}/' | |
'{3}/{4}/{5}/').format(self.build['repo'], | |
self.build['release'], | |
self.build['platform'], | |
self.build['architecture'], | |
self.build['build_id'], | |
self.jenkins_build_tag) | |
return prefix.replace(' ', '-') | |
self.job = JenkinsJob() | |
@PostScriptAction('install') | |
def initialize_job(self, action, success=None): | |
if not success: | |
return | |
if self.config['treeherding_off'] or not self.treeherder: | |
self.info(self.off_message) | |
return | |
super(FirefoxMediaTestsJenkins, self).initialize_job(action, success) | |
c = self.config | |
self.job.jenkins_build_tag = c['jenkins_build_tag'] | |
self.job.jenkins_build_url = c['jenkins_build_url'] | |
self.job.name = c['jenkins_build_tag'] | |
if c['jenkins_build_url']: | |
self.job.job_details.append({ | |
'url': self.job.jenkins_build_url, | |
'value': 'Jenkins Build URL (VPN required)', | |
'content_type': 'link', | |
'title': 'artifact uploaded'} | |
) | |
else: | |
self.warning('Job has no Jenkins build url') | |
if c['jenkins_build_tag']: | |
self.job.job_details.append({ | |
'value': self.job.jenkins_build_tag, | |
'content_type': 'text', | |
'title': 'artifact uploaded'} | |
) | |
else: | |
self.warning('Job has no Jenkins build tag') | |
@PreScriptAction('submit-treeherder-complete') | |
def update_job_complete(self, action): | |
if self.config['treeherding_off'] or not self.treeherder: | |
self.info(self.off_message) | |
return | |
super(FirefoxMediaTestsJenkins, self).update_job_complete(action) | |
dirs = self.query_abs_dirs() | |
log_dir = dirs.get('abs_log_dir') | |
# copy media_urls ini file with txt extension for convenient web view | |
if self.media_urls: | |
url_config = os.path.abspath(self.media_urls) | |
wrk_url_config = os.path.join(dirs['abs_work_dir'], | |
os.path.basename(url_config) + '.txt') | |
if os.access(wrk_url_config, os.F_OK): | |
self.rmtree(wrk_url_config) | |
if os.access(url_config, os.F_OK): | |
self.copyfile(url_config, wrk_url_config) | |
self.job.config_files.append(wrk_url_config) | |
# instead of uploading all logs, upload broadest, error and gecko | |
if log_dir: | |
def add_log(name, parse=False): | |
if not name: | |
return | |
log_path = os.path.join(log_dir, name) | |
if os.path.exists(log_path): | |
self.job.log_files.append(log_path) | |
if parse: | |
self.job.parsed_logs.append(log_path) | |
add_log(self.log_obj.log_files.get(ERROR)) | |
# Never upload debug logs | |
if self.log_obj.log_level == DEBUG: | |
main_log = self.log_obj.log_files.get(INFO) | |
else: | |
main_log = self.log_obj.log_files.get(self.log_obj.log_level) | |
add_log(main_log, parse=True) | |
# in case of SimpleFileLogger | |
add_log(self.log_obj.log_files.get('default')) | |
# extra log files saved by marionette | |
for f in self.media_logs: | |
add_log(f) | |
# Replace default upload dir (all logs) with screenshots dir | |
screenshots_dir = os.path.join(log_dir, 'screenshots') | |
if os.path.exists(screenshots_dir): | |
self.job.upload_dir = os.path.abspath(screenshots_dir) | |
else: | |
self.job.upload_dir = '' | |
if __name__ == '__main__': | |
media_test = FirefoxMediaTestsJenkins() | |
media_test.run_and_exit() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# ***** BEGIN LICENSE BLOCK ***** | |
# This Source Code Form is subject to the terms of the Mozilla Public | |
# License, v. 2.0. If a copy of the MPL was not distributed with this | |
# file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
# ***** BEGIN LICENSE BLOCK ***** | |
import json | |
import os | |
import traceback | |
from mozharness.base.script import PreScriptAction, PostScriptAction | |
treeherder_config_options = [ | |
[["--no-treeherding"], | |
{"action": "store_true", | |
"dest": "treeherding_off", | |
"default": False, # i.e. Treeherding is on by default | |
"help": "Disable submission to Treeherder", | |
}], | |
[["--job-name"], | |
{"action": "store", | |
"dest": "job_name", | |
"help": ("Job name to submit to Treeherder. " | |
"e.g. MSE Video Playback Tests"), | |
}], | |
[["--job-symbol"], | |
{"action": "store", | |
"dest": "job_symbol", | |
"help": "Job symbol to submit to Treeherder.", | |
}], | |
[["--treeherder-url"], | |
{"action": "store", | |
"dest": "treeherder_url", | |
"help": "e.g. https://treeherder.allizom.org", | |
}], | |
[["--treeherder-credentials"], | |
{"action": "store", | |
"dest": "treeherder_credentials_path", | |
"help": "Path to credentials json file.", | |
}], | |
[["--s3-credentials"], | |
{"action": "store", | |
"dest": "s3_credentials_path", | |
"help": "Path to credentials json file", | |
}], | |
] | |
class TreeherderMixin(object): | |
""" BaseScriptMixin for uploading job results to Treeherder. | |
Uploads logs to S3 bucket | |
Requires the VirtualenvMixin in order to install dependencies: | |
treeherder-client, requests, boto, mozinfo, mozversion | |
Local dependencies: treeherding.py, s3.py, parsers.py | |
Interacts with TestingMixin via an 'install' PostScriptAction, as well as | |
self.binary_path, self.installer_path | |
Config dependencies: | |
- treeherder_url | |
- treeherder_credentials_path (path to json file) | |
- s3_credentials_path (path to json file) | |
- group/job name/symbol, who, description, reason | |
""" | |
off_message = "Treeherding is off or not set up; nothing to do." | |
job = None | |
treeherder = None | |
@PreScriptAction('create-virtualenv') | |
def _configure_treeherding_virtualenv(self, action): | |
if self.config['treeherding_off']: | |
self.info(self.off_message) | |
return | |
self.register_virtualenv_module('boto', method='pip', | |
optional=True) | |
self.register_virtualenv_module('treeherder-client==1.6', | |
method='pip', optional=True) | |
# For populating self.job with build/machine data | |
self.register_virtualenv_module('mozinfo', | |
method='pip', optional=True) | |
self.register_virtualenv_module('mozversion', | |
method='pip', optional=True) | |
@PostScriptAction('create-virtualenv') | |
def setup_treeherding(self, action, success=None): | |
if not success: | |
return | |
if self.config['treeherding_off']: | |
self.info(self.off_message) | |
return | |
self.activate_virtualenv() | |
try: | |
# Imports are inside methods (rather than global) because we depend | |
# on virtual-environment setup that should happen earlier in the | |
# mozharness script. (We want to work in exactly one venv and that | |
# venv should be created by mozharness. We don't want a mozharness | |
# venv within a venv created externally.) | |
from treeherding import TreeherderSubmission, TestJob | |
options = self._get_treeherder_options() | |
s3_bucket = self._get_s3_bucket() | |
self.info("Initializing Treeherder client") | |
self.treeherder = TreeherderSubmission(self.log_obj.logger, | |
options, | |
s3_bucket) | |
self.job = TestJob() | |
except Exception: | |
self.warning("Unable to init Treeherder client: %s" % | |
traceback.format_exc()) | |
def _get_treeherder_options(self): | |
""" Returns TreeherderOptions instance populated based on config. | |
Prerequisite: A venv has been created and necessary packages have been | |
installed. | |
""" | |
self.info("Collecting Treeherder options.") | |
from treeherding import TreeherderOptions | |
c = self.config | |
options = TreeherderOptions() | |
options.treeherder_url = c['treeherder_url'] | |
dirs = self.query_abs_dirs() | |
credentials_path = os.path.join(dirs['base_work_dir'], | |
c['treeherder_credentials_path']) | |
options.treeherder_credentials_path = credentials_path | |
try: | |
with open(options.treeherder_credentials_path) as f: | |
credentials_string = f.read() | |
options.treeherder_credentials = json.loads(credentials_string) | |
except IOError: | |
msg = ('Treeherder credentials file not ' | |
'found at {0}.'.format(options.treeherder_credentials_path)) | |
self.warning(msg) | |
return options | |
def _get_s3_bucket(self): | |
""" Returns S3Bucket instance populated based on config. | |
Prerequisite: A venv has been created and necessary packages have been | |
installed. | |
""" | |
self.info("Setting up S3Bucket.") | |
from s3 import S3Bucket | |
c = self.config | |
dirs = self.query_abs_dirs() | |
credentials_path = os.path.join(dirs['base_work_dir'], | |
c['s3_credentials_path']) | |
try: | |
with open(credentials_path) as f: | |
config_string = f.read() | |
s3_config = json.loads(config_string) | |
return S3Bucket(s3_config['s3_bucket_name'], | |
s3_config['aws_access_key_id'], | |
s3_config['aws_access_key'], | |
self.log_obj.logger) | |
except IOError: | |
msg = ('S3 credentials file not ' | |
'found at {0}.'.format(credentials_path)) | |
self.warning(msg) | |
@PostScriptAction('install') | |
def initialize_job(self, action, success=None): | |
""" Populate basic job info (build, machine, group/job name/symbol). | |
Should override this to add job info that is specific to your script. | |
""" | |
if not success: | |
return | |
if self.config['treeherding_off'] or not self.treeherder: | |
self.info(self.off_message) | |
return | |
from treeherding import collect_job_info | |
try: | |
collect_job_info(self.job, self.binary_path, | |
os.path.basename(self.installer_path)) | |
c = self.config | |
self.job.group_name = c['group_name'] | |
self.job.group_symbol = c['group_symbol'] | |
self.job.job_name = c['job_name'] | |
self.job.job_symbol = c['job_symbol'] | |
self.job.description = c['job_description'] | |
self.job.reason = c['job_reason'] | |
self.job.who = c['job_who'] | |
except Exception: | |
self.warning("Unable to init job data (build, machine): %s" % | |
traceback.format_exc()) | |
def submit_treeherder_running(self): | |
""" Submit job to Treeherder with status "running". | |
Prerequisite: job should be populated with basic info like | |
job/group name/symbol, revision, project. | |
""" | |
if self.config['treeherding_off'] or not self.treeherder: | |
self.info(self.off_message) | |
return | |
self.treeherder.submit_running([self.job]) | |
@PreScriptAction('submit-treeherder-complete') | |
def update_job_complete(self, action): | |
""" Prepare results and artifacts (log files, config files) """ | |
if self.job_result_parser: | |
# anything with status, passed, failed and todo int/str attributes | |
# is a suitable test_result | |
self.job.test_result = self.job_result_parser | |
self.job.result = self.job.test_result.status | |
else: | |
self.job.result = 'unknown' | |
self.job.upload_dir = self.query_abs_dirs().get('abs_log_dir') | |
def submit_treeherder_complete(self): | |
""" Submit job to Treeherder with status "completed". | |
Prerequisite: job should be populated with results and artifacts | |
""" | |
if self.config['treeherding_off'] or not self.treeherder: | |
self.info(self.off_message) | |
return | |
self.treeherder.submit_complete([self.job]) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
So where are all of these files actually checked in?