Created
July 14, 2017 21:11
-
-
Save rail/e9829640d592587ecf139d90e29e24b8 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/buildfarm/release/release-runner.py b/buildfarm/release/release-runner.py | |
--- a/buildfarm/release/release-runner.py | |
+++ b/buildfarm/release/release-runner.py | |
@@ -503,17 +503,18 @@ def main(options): | |
"l10n_changesets": release['l10n_changesets'], | |
"extra_balrog_submitter_params": extra_balrog_submitter_params + " --product " + release["product"].capitalize(), | |
"publish_to_balrog_channels": publish_to_balrog_channels, | |
"snap_enabled": branchConfig.get("snap_enabled", {}).get(release["product"], False), | |
"update_verify_channel": branchConfig.get("update_verify_channel", {}).get(release["product"]), | |
"update_verify_requires_cdn_push": branchConfig.get("update_verify_requires_cdn_push", False), | |
} | |
- validate_graph_kwargs(queue, gpg_key_path, **kwargs) | |
+ # TODO: en-US validation for multiple tasks | |
+ # validate_graph_kwargs(queue, gpg_key_path, **kwargs) | |
graph = make_task_graph_strict_kwargs(**kwargs) | |
rr.update_status(release, "Submitting task graph") | |
log.info("Task graph generated!") | |
import pprint | |
log.debug(pprint.pformat(graph, indent=4, width=160)) | |
print(scheduler.createTaskGraph(graph_id, graph)) | |
rr.mark_as_completed(release) | |
diff --git a/lib/python/kickoff/__init__.py b/lib/python/kickoff/__init__.py | |
--- a/lib/python/kickoff/__init__.py | |
+++ b/lib/python/kickoff/__init__.py | |
@@ -149,38 +149,59 @@ def get_platform_locales(l10n_changesets | |
return [l for l in l10n_changesets.keys() if l != ignore] | |
def get_l10n_config(index, product, branch, revision, platforms, | |
l10n_platforms, l10n_changesets, tc_task_indexes): | |
l10n_platform_configs = {} | |
for platform in l10n_platforms: | |
- route = tc_task_indexes[platform]['signed'].format(rev=revision) | |
+ # mar_tools lives in the unsigned task | |
+ unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision) | |
+ unsigned_task = index.findTask(unsigned_route) | |
+ if tc_task_indexes[platform]['ci_system'] == 'bb': | |
+ mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format( | |
+ taskid=unsigned_task["taskId"] | |
+ ) | |
+ else: | |
+ mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format( | |
+ taskid=unsigned_task["taskId"] | |
+ ) | |
+ | |
+ # en-US binary lives all over the places! | |
+ if platform.startswith("linux"): | |
+ route = tc_task_indexes[platform]['signed'].format(rev=revision) | |
+ elif platform.startswith("mac"): | |
+ if tc_task_indexes[platform]['ci_system'] == 'bb': | |
+ route = tc_task_indexes[platform]['signed'].format(rev=revision) | |
+ else: | |
+ route = tc_task_indexes[platform]['repackage'].format(rev=revision) | |
+ elif platform.startswith("win"): | |
+ # BB has all binaries in one task | |
+ # TC generates target.zip in this task, the installer is set below | |
+ route = tc_task_indexes[platform]['signed'].format(rev=revision) | |
signed_task = index.findTask(route) | |
en_us_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format( | |
taskid=signed_task["taskId"] | |
) | |
- if tc_task_indexes[platform]['unsigned'] != tc_task_indexes[platform]['signed']: | |
- # TC based builds use different tasks for en-US (signed) and | |
- # martools (unsigned) | |
- unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision) | |
- unsigned_task = index.findTask(unsigned_route) | |
- mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format( | |
- taskid=unsigned_task["taskId"] | |
- ) | |
- else: | |
- mar_tools_url = en_us_binary_url | |
l10n_platform_configs[platform] = { | |
"locales": get_platform_locales(l10n_changesets, platform), | |
"en_us_binary_url": en_us_binary_url, | |
"mar_tools_url": mar_tools_url, | |
"chunks": platforms[platform].get("l10n_chunks", 10), | |
} | |
+ # Windows installer is a different beast | |
+ if platform.startswith("win") and tc_task_indexes[platform]['ci_system'] == 'tc': | |
+ route = tc_task_indexes[platform]['repackage'].format(rev=revision) | |
+ installer_task = index.findTask(route) | |
+ en_us_installer_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format( | |
+ taskid=installer_task["taskId"] | |
+ ) | |
+ l10n_platform_configs[platform]['en_us_installer_binary_url'] = en_us_installer_binary_url | |
return { | |
"platforms": l10n_platform_configs, | |
"changesets": l10n_changesets, | |
} | |
def get_en_US_config(index, product, branch, revision, platforms, | |
@@ -189,17 +210,23 @@ def get_en_US_config(index, product, bra | |
for platform in platforms: | |
signed_route = tc_task_indexes[platform]['signed'].format(rev=revision) | |
signed_task = index.findTask(signed_route) | |
unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision) | |
unsigned_task = index.findTask(unsigned_route) | |
platform_configs[platform] = { | |
"signed_task_id": signed_task["taskId"], | |
"unsigned_task_id": unsigned_task["taskId"], | |
+ "ci_system": tc_task_indexes[platform]["ci_system"], | |
} | |
+ for t in ("repackage", "repackage-signing"): | |
+ if t in tc_task_indexes[platform]: | |
+ route = tc_task_indexes[platform][t].format(rev=revision) | |
+ task = index.findTask(route) | |
+ platform_configs[platform]['{}_task_id'.format(t)] = task['taskId'] | |
return { | |
"platforms": platform_configs, | |
} | |
# FIXME: the following function should be removed and we should use | |
# next_version provided by ship-it | |
diff --git a/lib/python/kickoff/build_status.py b/lib/python/kickoff/build_status.py | |
--- a/lib/python/kickoff/build_status.py | |
+++ b/lib/python/kickoff/build_status.py | |
@@ -41,71 +41,71 @@ class LoggedError(Exception): | |
class EnUsBuildsWatcher: | |
# TODO: Bug 1300147 as well | |
def __init__(self, index, release_name, submitted_at, revision, | |
platforms, queue, tc_task_indexes): | |
self.taskcluster_index = index | |
self.release_name = release_name | |
self.revision = revision | |
- self.task_per_platform = {p: None for p in platforms} | |
+ self.platforms_to_check = list(platforms) | |
self.queue = queue | |
self.tc_task_indexes = tc_task_indexes | |
self._timeout_watcher = TimeoutWatcher(start_timestamp=submitted_at) | |
def are_builds_completed(self): | |
if self._timeout_watcher.timed_out: | |
raise TimeoutWatcher.TimeoutError( | |
self.release_name, self._timeout_watcher.start_timestamp) | |
self._fetch_completed_tasks() | |
+ return len(self.platforms_to_check) == 0 | |
- return len(self._platforms_with_no_task) == 0 | |
+ def _all_tasks_ready(self, platform): | |
+ for t in ('signed', 'unsigned', 'repackage', 'repackage-signing'): | |
+ | |
+ if t not in self.tc_task_indexes[platform]: | |
+ log.debug("Ignoring `%s` index type for %s", t, platform) | |
+ continue | |
+ | |
+ # Tasks are always completed if they are referenced in the index | |
+ # https://docs.taskcluster.net/reference/core/index Assuming that | |
+ # the signed tasks are completed after their unsigned counterparts | |
+ route = self.tc_task_indexes[platform][t].format(rev=self.revision) | |
+ task_id = self.taskcluster_index.findTask(route)['taskId'] | |
+ # Bug 1307326 - consider only tasks indexed with rank > 0. | |
+ # If `rank` is unknown use tier-1 tasks. | |
+ task = self.queue.task(task_id) | |
+ rank = task["extra"].get("index", {}).get("rank") | |
+ tier = task["extra"].get("treeherder", {}).get("tier") | |
+ if rank is None: | |
+ eligible = tier == 1 | |
+ else: | |
+ eligible = rank != 0 | |
+ if not eligible: | |
+ log.debug("Ignoring task %s because rank (%s) or tier (%s)", | |
+ task_id, rank, tier) | |
+ # Do not bother all tasks if some are not finished yet | |
+ return False | |
+ | |
+ return True | |
def _fetch_completed_tasks(self): | |
- platforms_with_no_task = self._platforms_with_no_task | |
log.debug('Release "%s" still has to find tasks for %s', | |
- self.release_name, platforms_with_no_task) | |
+ self.release_name, self.platforms_to_check) | |
- for platform in platforms_with_no_task: | |
+ for platform in list(self.platforms_to_check): | |
try: | |
- # Tasks are always completed if they are referenced in the | |
- # index https://docs.taskcluster.net/reference/core/index | |
- # Assuming that the signed tasks are completed after their | |
- # unsigned counterparts | |
- route = self.tc_task_indexes[platform]['signed'].format( | |
- rev=self.revision) | |
- task_id = self.taskcluster_index.findTask(route)['taskId'] | |
- # Bug 1307326 - consider only tasks indexed with rank > 0. | |
- # If `rank` is unknown use tier-1 tasks. | |
- task = self.queue.task(task_id) | |
- rank = task["extra"].get("index", {}).get("rank") | |
- tier = task["extra"].get("treeherder", {}).get("tier") | |
- if rank is None: | |
- eligible = tier == 1 | |
- else: | |
- eligible = rank != 0 | |
- if not eligible: | |
- log.debug("Ignoring task %s because rank (%s) or tier (%s)", | |
- task_id, rank, tier) | |
- continue | |
+ if self._all_tasks_ready(platform): | |
+ log.info("All tasks for %s are ready", platform) | |
+ self.platforms_to_check.remove(platform) | |
except TaskclusterRestFailure: | |
log.debug('Task for platform "%s" is not yet created for ' | |
'release "%s"', platform, self.release_name) | |
- continue | |
- | |
- log.debug('Task "%s" was found for release "%s" and platform "%s"', | |
- task_id, self.release_name, platform) | |
- self.task_per_platform[platform] = task_id | |
- | |
- @property | |
- def _platforms_with_no_task(self): | |
- return [platform for platform, task in | |
- self.task_per_platform.iteritems() if task is None] | |
class TimeoutWatcher: | |
MAX_WAITING_TIME = timedelta(days=1) | |
def __init__(self, start_timestamp): | |
self.start_timestamp = parser.parse(start_timestamp) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment