Skip to content

Instantly share code, notes, and snippets.

@rail
Created July 14, 2017 21:11
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rail/e9829640d592587ecf139d90e29e24b8 to your computer and use it in GitHub Desktop.
Save rail/e9829640d592587ecf139d90e29e24b8 to your computer and use it in GitHub Desktop.
diff --git a/buildfarm/release/release-runner.py b/buildfarm/release/release-runner.py
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -503,17 +503,18 @@ def main(options):
"l10n_changesets": release['l10n_changesets'],
"extra_balrog_submitter_params": extra_balrog_submitter_params + " --product " + release["product"].capitalize(),
"publish_to_balrog_channels": publish_to_balrog_channels,
"snap_enabled": branchConfig.get("snap_enabled", {}).get(release["product"], False),
"update_verify_channel": branchConfig.get("update_verify_channel", {}).get(release["product"]),
"update_verify_requires_cdn_push": branchConfig.get("update_verify_requires_cdn_push", False),
}
- validate_graph_kwargs(queue, gpg_key_path, **kwargs)
+ # TODO: en-US validation for multiple tasks
+ # validate_graph_kwargs(queue, gpg_key_path, **kwargs)
graph = make_task_graph_strict_kwargs(**kwargs)
rr.update_status(release, "Submitting task graph")
log.info("Task graph generated!")
import pprint
log.debug(pprint.pformat(graph, indent=4, width=160))
print(scheduler.createTaskGraph(graph_id, graph))
rr.mark_as_completed(release)
diff --git a/lib/python/kickoff/__init__.py b/lib/python/kickoff/__init__.py
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -149,38 +149,59 @@ def get_platform_locales(l10n_changesets
return [l for l in l10n_changesets.keys() if l != ignore]
def get_l10n_config(index, product, branch, revision, platforms,
l10n_platforms, l10n_changesets, tc_task_indexes):
l10n_platform_configs = {}
for platform in l10n_platforms:
- route = tc_task_indexes[platform]['signed'].format(rev=revision)
+ # mar_tools lives in the unsigned task
+ unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
+ unsigned_task = index.findTask(unsigned_route)
+ if tc_task_indexes[platform]['ci_system'] == 'bb':
+ mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+ taskid=unsigned_task["taskId"]
+ )
+ else:
+ mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format(
+ taskid=unsigned_task["taskId"]
+ )
+
+ # en-US binary lives all over the places!
+ if platform.startswith("linux"):
+ route = tc_task_indexes[platform]['signed'].format(rev=revision)
+ elif platform.startswith("mac"):
+ if tc_task_indexes[platform]['ci_system'] == 'bb':
+ route = tc_task_indexes[platform]['signed'].format(rev=revision)
+ else:
+ route = tc_task_indexes[platform]['repackage'].format(rev=revision)
+ elif platform.startswith("win"):
+ # BB has all binaries in one task
+ # TC generates target.zip in this task, the installer is set below
+ route = tc_task_indexes[platform]['signed'].format(rev=revision)
signed_task = index.findTask(route)
en_us_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
taskid=signed_task["taskId"]
)
- if tc_task_indexes[platform]['unsigned'] != tc_task_indexes[platform]['signed']:
- # TC based builds use different tasks for en-US (signed) and
- # martools (unsigned)
- unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
- unsigned_task = index.findTask(unsigned_route)
- mar_tools_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build/host/bin".format(
- taskid=unsigned_task["taskId"]
- )
- else:
- mar_tools_url = en_us_binary_url
l10n_platform_configs[platform] = {
"locales": get_platform_locales(l10n_changesets, platform),
"en_us_binary_url": en_us_binary_url,
"mar_tools_url": mar_tools_url,
"chunks": platforms[platform].get("l10n_chunks", 10),
}
+ # Windows installer is a different beast
+ if platform.startswith("win") and tc_task_indexes[platform]['ci_system'] == 'tc':
+ route = tc_task_indexes[platform]['repackage'].format(rev=revision)
+ installer_task = index.findTask(route)
+ en_us_installer_binary_url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+ taskid=installer_task["taskId"]
+ )
+ l10n_platform_configs[platform]['en_us_installer_binary_url'] = en_us_installer_binary_url
return {
"platforms": l10n_platform_configs,
"changesets": l10n_changesets,
}
def get_en_US_config(index, product, branch, revision, platforms,
@@ -189,17 +210,23 @@ def get_en_US_config(index, product, bra
for platform in platforms:
signed_route = tc_task_indexes[platform]['signed'].format(rev=revision)
signed_task = index.findTask(signed_route)
unsigned_route = tc_task_indexes[platform]['unsigned'].format(rev=revision)
unsigned_task = index.findTask(unsigned_route)
platform_configs[platform] = {
"signed_task_id": signed_task["taskId"],
"unsigned_task_id": unsigned_task["taskId"],
+ "ci_system": tc_task_indexes[platform]["ci_system"],
}
+ for t in ("repackage", "repackage-signing"):
+ if t in tc_task_indexes[platform]:
+ route = tc_task_indexes[platform][t].format(rev=revision)
+ task = index.findTask(route)
+ platform_configs[platform]['{}_task_id'.format(t)] = task['taskId']
return {
"platforms": platform_configs,
}
# FIXME: the following function should be removed and we should use
# next_version provided by ship-it
diff --git a/lib/python/kickoff/build_status.py b/lib/python/kickoff/build_status.py
--- a/lib/python/kickoff/build_status.py
+++ b/lib/python/kickoff/build_status.py
@@ -41,71 +41,71 @@ class LoggedError(Exception):
class EnUsBuildsWatcher:
# TODO: Bug 1300147 as well
def __init__(self, index, release_name, submitted_at, revision,
platforms, queue, tc_task_indexes):
self.taskcluster_index = index
self.release_name = release_name
self.revision = revision
- self.task_per_platform = {p: None for p in platforms}
+ self.platforms_to_check = list(platforms)
self.queue = queue
self.tc_task_indexes = tc_task_indexes
self._timeout_watcher = TimeoutWatcher(start_timestamp=submitted_at)
def are_builds_completed(self):
if self._timeout_watcher.timed_out:
raise TimeoutWatcher.TimeoutError(
self.release_name, self._timeout_watcher.start_timestamp)
self._fetch_completed_tasks()
+ return len(self.platforms_to_check) == 0
- return len(self._platforms_with_no_task) == 0
+ def _all_tasks_ready(self, platform):
+ for t in ('signed', 'unsigned', 'repackage', 'repackage-signing'):
+
+ if t not in self.tc_task_indexes[platform]:
+ log.debug("Ignoring `%s` index type for %s", t, platform)
+ continue
+
+ # Tasks are always completed if they are referenced in the index
+ # https://docs.taskcluster.net/reference/core/index Assuming that
+ # the signed tasks are completed after their unsigned counterparts
+ route = self.tc_task_indexes[platform][t].format(rev=self.revision)
+ task_id = self.taskcluster_index.findTask(route)['taskId']
+ # Bug 1307326 - consider only tasks indexed with rank > 0.
+ # If `rank` is unknown use tier-1 tasks.
+ task = self.queue.task(task_id)
+ rank = task["extra"].get("index", {}).get("rank")
+ tier = task["extra"].get("treeherder", {}).get("tier")
+ if rank is None:
+ eligible = tier == 1
+ else:
+ eligible = rank != 0
+ if not eligible:
+ log.debug("Ignoring task %s because rank (%s) or tier (%s)",
+ task_id, rank, tier)
+ # Do not bother all tasks if some are not finished yet
+ return False
+
+ return True
def _fetch_completed_tasks(self):
- platforms_with_no_task = self._platforms_with_no_task
log.debug('Release "%s" still has to find tasks for %s',
- self.release_name, platforms_with_no_task)
+ self.release_name, self.platforms_to_check)
- for platform in platforms_with_no_task:
+ for platform in list(self.platforms_to_check):
try:
- # Tasks are always completed if they are referenced in the
- # index https://docs.taskcluster.net/reference/core/index
- # Assuming that the signed tasks are completed after their
- # unsigned counterparts
- route = self.tc_task_indexes[platform]['signed'].format(
- rev=self.revision)
- task_id = self.taskcluster_index.findTask(route)['taskId']
- # Bug 1307326 - consider only tasks indexed with rank > 0.
- # If `rank` is unknown use tier-1 tasks.
- task = self.queue.task(task_id)
- rank = task["extra"].get("index", {}).get("rank")
- tier = task["extra"].get("treeherder", {}).get("tier")
- if rank is None:
- eligible = tier == 1
- else:
- eligible = rank != 0
- if not eligible:
- log.debug("Ignoring task %s because rank (%s) or tier (%s)",
- task_id, rank, tier)
- continue
+ if self._all_tasks_ready(platform):
+ log.info("All tasks for %s are ready", platform)
+ self.platforms_to_check.remove(platform)
except TaskclusterRestFailure:
log.debug('Task for platform "%s" is not yet created for '
'release "%s"', platform, self.release_name)
- continue
-
- log.debug('Task "%s" was found for release "%s" and platform "%s"',
- task_id, self.release_name, platform)
- self.task_per_platform[platform] = task_id
-
- @property
- def _platforms_with_no_task(self):
- return [platform for platform, task in
- self.task_per_platform.iteritems() if task is None]
class TimeoutWatcher:
MAX_WAITING_TIME = timedelta(days=1)
def __init__(self, start_timestamp):
self.start_timestamp = parser.parse(start_timestamp)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment