Skip to content

Instantly share code, notes, and snippets.

@rail
Created February 15, 2018 13:56
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rail/01caca703ca8c1d11e99f92b88fc2490 to your computer and use it in GitHub Desktop.
Save rail/01caca703ca8c1d11e99f92b88fc2490 to your computer and use it in GitHub Desktop.
diff --git a/taskcluster/ci/release-bouncer-check/kind.yml b/taskcluster/ci/release-bouncer-check/kind.yml
index 3c8bfb2341e7..7daf8a9008ba 100644
--- a/taskcluster/ci/release-bouncer-check/kind.yml
+++ b/taskcluster/ci/release-bouncer-check/kind.yml
@@ -10,6 +10,7 @@ kind-dependencies:
transforms:
- taskgraph.transforms.release_deps:transforms
- taskgraph.transforms.bouncer_check:transforms
+ - taskgraph.transforms.release_notifications:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
@@ -25,12 +26,6 @@ job-defaults:
run:
using: run-task
sparse-profile: mozharness
- # 'release' index type interpolates the routes placeholders
- index:
- type: release
- routes:
- - index.releases.v1.{branch}.latest.{product}.latest.bouncer_check
- - index.releases.v1.{branch}.{revision}.{product}.{underscore_version}.build{build_number}.bouncer_check
attributes:
build_platform: linux64
build_type: opt
@@ -39,53 +34,12 @@ job-defaults:
kind: test
tier: 1
- notifications:
- completed:
- subject: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- message: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- plugins:
- by-project:
- mozilla-beta: ["log_collect"]
- mozilla-release: ["log_collect"]
- default: []
-
- failed:
- subject: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- message: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- plugins:
- by-project:
- mozilla-beta: ["log_collect", "ses"]
- mozilla-release: ["log_collect", "ses"]
- default: ["ses"]
- emails:
- by-project:
- mozilla-beta: ["release-automation-notifications@mozilla.com"]
- mozilla-release: ["release-automation-notifications@mozilla.com"]
- try: ["{task_def[metadata][owner]}"]
- maple: ["release+tcstaging@mozilla.com"]
- default: []
-
- exception:
- subject: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- message: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task"
- plugins:
- by-project:
- mozilla-beta: ["log_collect", "ses"]
- mozilla-release: ["log_collect", "ses"]
- default: ["ses"]
- emails:
- by-project:
- mozilla-beta: ["release-automation-notifications@mozilla.com"]
- mozilla-release: ["release-automation-notifications@mozilla.com"]
- try: ["{task_def[metadata][owner]}"]
- maple: ["release+tcstaging@mozilla.com"]
- default: []
-
jobs:
firefox:
shipping-product: firefox
index:
product: firefox
+ job-name: firefox-bouncer-check
run:
config:
by-project:
@@ -104,6 +58,7 @@ jobs:
shipping-product: devedition
index:
product: devedition
+ job-name: devedition-bouncer-check
run:
config:
by-project:
diff --git a/taskcluster/taskgraph/transforms/bouncer_check.py b/taskcluster/taskgraph/transforms/bouncer_check.py
index 490011e4d66d..a9ddec89d216 100644
--- a/taskcluster/taskgraph/transforms/bouncer_check.py
+++ b/taskcluster/taskgraph/transforms/bouncer_check.py
@@ -4,6 +4,7 @@
from __future__ import absolute_import, print_function, unicode_literals
import copy
+import subprocess
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.scriptworker import get_release_config
@@ -74,5 +75,5 @@ def command_to_string(config, jobs):
"""Convert command to string to make it work properly with run-task"""
for job in jobs:
job = copy.deepcopy(job) # don't overwrite dict values here
- job["run"]["command"] = " ".join(job["run"]["command"])
+ job["run"]["command"] = subprocess.list2cmdline(job["run"]["command"])
yield job
diff --git a/testing/mozharness/scripts/release/uptake_monitoring.py b/testing/mozharness/scripts/release/uptake_monitoring.py
deleted file mode 100644
index 79f90a8fa722..000000000000
--- a/testing/mozharness/scripts/release/uptake_monitoring.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" uptake_monitoring.py
-
-A script to replace the old-fashion way of computing the uptake monitoring
-from the scheduler within the slaves.
-"""
-
-import os
-import sys
-import datetime
-import time
-import xml.dom.minidom
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-
-def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os):
- return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url,
- related_product, os)
-
-
-class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin):
- config_options = virtualenv_config_options
-
- def __init__(self, require_config_file=True):
- super(UptakeMonitoring, self).__init__(
- config_options=self.config_options,
- require_config_file=require_config_file,
- config={
- "virtualenv_modules": [
- "redo",
- "requests",
- ],
-
- "virtualenv_path": "venv",
- "credentials_file": "oauth.txt",
- "buildbot_json_path": "buildprops.json",
- "poll_interval": 60,
- "poll_timeout": 20*60,
- "min_uptake": 10000,
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- )
-
- def _pre_config_lock(self, rw_config):
- super(UptakeMonitoring, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config["properties"]
- for prop in ['tuxedo_server_url', 'version']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
- else:
- self.warning("%s could not be found within buildprops" % prop)
- return
- if props.get('partial_versions'):
- partials = [v.strip() for v in props["partial_versions"].split(",")]
- self.config["partial_versions"] = [v.split("build")[0] for v in partials]
- self.config["platforms"] = [p.strip() for p in
- props["platforms"].split(",")]
-
- def _get_product_uptake(self, tuxedo_server_url, auth,
- related_product, os):
- from redo import retry
- import requests
-
- url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
- self.info("Requesting {} from tuxedo".format(url))
-
- def get_tuxedo_page():
- r = requests.get(url, auth=auth,
- verify=False, timeout=60)
- r.raise_for_status()
- return r.content
-
- def calculateUptake(page):
- doc = xml.dom.minidom.parseString(page)
- uptake_values = []
-
- for element in doc.getElementsByTagName('available'):
- for node in element.childNodes:
- if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
- node.data.isdigit():
- uptake_values.append(int(node.data))
- if not uptake_values:
- uptake_values = [0]
- return min(uptake_values)
-
- page = retry(get_tuxedo_page)
- uptake = calculateUptake(page)
- self.info("Current uptake for {} is {}".format(related_product, uptake))
- return uptake
-
- def _get_release_uptake(self, auth):
- assert isinstance(self.config["platforms"], (list, tuple))
-
- # handle the products first
- tuxedo_server_url = self.config["tuxedo_server_url"]
- version = self.config["version"]
- dl = []
-
- for product, info in self.config["products"].iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- related_product = product_template % {"version": version}
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- # handle the partials as well
- prev_versions = self.config.get("partial_versions", [])
- for product, info in self.config.get("partials", {}).iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- for prev_version in prev_versions:
- subs = {
- "version": version,
- "prev_version": prev_version
- }
- related_product = product_template % subs
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- return min(dl)
-
- def monitor_uptake(self):
- credentials_file = os.path.join(os.getcwd(),
- self.config["credentials_file"])
- credentials = {}
- execfile(credentials_file, credentials)
- auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
- self.info("Starting the loop to determine the uptake monitoring ...")
-
- start_time = datetime.datetime.now()
- while True:
- delta = (datetime.datetime.now() - start_time).seconds
- if delta > self.config["poll_timeout"]:
- self.error("Uptake monitoring sadly timed-out")
- raise Exception("Time-out during uptake monitoring")
-
- uptake = self._get_release_uptake(auth)
- self.info("Current uptake value to check is {}".format(uptake))
-
- if uptake >= self.config["min_uptake"]:
- self.info("Uptake monitoring is complete!")
- break
- else:
- self.info("Mirrors not yet updated, sleeping for a bit ...")
- time.sleep(self.config["poll_interval"])
-
-
-if __name__ == '__main__':
- myScript = UptakeMonitoring()
- myScript.run_and_exit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment