Created
February 15, 2018 13:56
-
-
Save rail/01caca703ca8c1d11e99f92b88fc2490 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/taskcluster/ci/release-bouncer-check/kind.yml b/taskcluster/ci/release-bouncer-check/kind.yml | |
index 3c8bfb2341e7..7daf8a9008ba 100644 | |
--- a/taskcluster/ci/release-bouncer-check/kind.yml | |
+++ b/taskcluster/ci/release-bouncer-check/kind.yml | |
@@ -10,6 +10,7 @@ kind-dependencies: | |
transforms: | |
- taskgraph.transforms.release_deps:transforms | |
- taskgraph.transforms.bouncer_check:transforms | |
+ - taskgraph.transforms.release_notifications:transforms | |
- taskgraph.transforms.job:transforms | |
- taskgraph.transforms.task:transforms | |
@@ -25,12 +26,6 @@ job-defaults: | |
run: | |
using: run-task | |
sparse-profile: mozharness | |
- # 'release' index type interpolates the routes placeholders | |
- index: | |
- type: release | |
- routes: | |
- - index.releases.v1.{branch}.latest.{product}.latest.bouncer_check | |
- - index.releases.v1.{branch}.{revision}.{product}.{underscore_version}.build{build_number}.bouncer_check | |
attributes: | |
build_platform: linux64 | |
build_type: opt | |
@@ -39,53 +34,12 @@ job-defaults: | |
kind: test | |
tier: 1 | |
- notifications: | |
- completed: | |
- subject: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- message: "COMPLETED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- plugins: | |
- by-project: | |
- mozilla-beta: ["log_collect"] | |
- mozilla-release: ["log_collect"] | |
- default: [] | |
- | |
- failed: | |
- subject: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- message: "FAILED: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- plugins: | |
- by-project: | |
- mozilla-beta: ["log_collect", "ses"] | |
- mozilla-release: ["log_collect", "ses"] | |
- default: ["ses"] | |
- emails: | |
- by-project: | |
- mozilla-beta: ["release-automation-notifications@mozilla.com"] | |
- mozilla-release: ["release-automation-notifications@mozilla.com"] | |
- try: ["{task_def[metadata][owner]}"] | |
- maple: ["release+tcstaging@mozilla.com"] | |
- default: [] | |
- | |
- exception: | |
- subject: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- message: "EXCEPTION: [{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]}] {task_def[metadata][name]} task" | |
- plugins: | |
- by-project: | |
- mozilla-beta: ["log_collect", "ses"] | |
- mozilla-release: ["log_collect", "ses"] | |
- default: ["ses"] | |
- emails: | |
- by-project: | |
- mozilla-beta: ["release-automation-notifications@mozilla.com"] | |
- mozilla-release: ["release-automation-notifications@mozilla.com"] | |
- try: ["{task_def[metadata][owner]}"] | |
- maple: ["release+tcstaging@mozilla.com"] | |
- default: [] | |
- | |
jobs: | |
firefox: | |
shipping-product: firefox | |
index: | |
product: firefox | |
+ job-name: firefox-bouncer-check | |
run: | |
config: | |
by-project: | |
@@ -104,6 +58,7 @@ jobs: | |
shipping-product: devedition | |
index: | |
product: devedition | |
+ job-name: devedition-bouncer-check | |
run: | |
config: | |
by-project: | |
diff --git a/taskcluster/taskgraph/transforms/bouncer_check.py b/taskcluster/taskgraph/transforms/bouncer_check.py | |
index 490011e4d66d..a9ddec89d216 100644 | |
--- a/taskcluster/taskgraph/transforms/bouncer_check.py | |
+++ b/taskcluster/taskgraph/transforms/bouncer_check.py | |
@@ -4,6 +4,7 @@ | |
from __future__ import absolute_import, print_function, unicode_literals | |
import copy | |
+import subprocess | |
from taskgraph.transforms.base import TransformSequence | |
from taskgraph.util.scriptworker import get_release_config | |
@@ -74,5 +75,5 @@ def command_to_string(config, jobs): | |
"""Convert command to string to make it work properly with run-task""" | |
for job in jobs: | |
job = copy.deepcopy(job) # don't overwrite dict values here | |
- job["run"]["command"] = " ".join(job["run"]["command"]) | |
+ job["run"]["command"] = subprocess.list2cmdline(job["run"]["command"]) | |
yield job | |
diff --git a/testing/mozharness/scripts/release/uptake_monitoring.py b/testing/mozharness/scripts/release/uptake_monitoring.py | |
deleted file mode 100644 | |
index 79f90a8fa722..000000000000 | |
--- a/testing/mozharness/scripts/release/uptake_monitoring.py | |
+++ /dev/null | |
@@ -1,189 +0,0 @@ | |
-#!/usr/bin/env python | |
-# lint_ignore=E501 | |
-# ***** BEGIN LICENSE BLOCK ***** | |
-# This Source Code Form is subject to the terms of the Mozilla Public | |
-# License, v. 2.0. If a copy of the MPL was not distributed with this file, | |
-# You can obtain one at http://mozilla.org/MPL/2.0/. | |
-# ***** END LICENSE BLOCK ***** | |
-""" uptake_monitoring.py | |
- | |
-A script to replace the old-fashion way of computing the uptake monitoring | |
-from the scheduler within the slaves. | |
-""" | |
- | |
-import os | |
-import sys | |
-import datetime | |
-import time | |
-import xml.dom.minidom | |
- | |
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0]))) | |
- | |
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options | |
-from mozharness.base.script import BaseScript | |
-from mozharness.mozilla.buildbot import BuildbotMixin | |
- | |
- | |
-def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os): | |
- return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url, | |
- related_product, os) | |
- | |
- | |
-class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin): | |
- config_options = virtualenv_config_options | |
- | |
- def __init__(self, require_config_file=True): | |
- super(UptakeMonitoring, self).__init__( | |
- config_options=self.config_options, | |
- require_config_file=require_config_file, | |
- config={ | |
- "virtualenv_modules": [ | |
- "redo", | |
- "requests", | |
- ], | |
- | |
- "virtualenv_path": "venv", | |
- "credentials_file": "oauth.txt", | |
- "buildbot_json_path": "buildprops.json", | |
- "poll_interval": 60, | |
- "poll_timeout": 20*60, | |
- "min_uptake": 10000, | |
- }, | |
- all_actions=[ | |
- "create-virtualenv", | |
- "activate-virtualenv", | |
- "monitor-uptake", | |
- ], | |
- default_actions=[ | |
- "create-virtualenv", | |
- "activate-virtualenv", | |
- "monitor-uptake", | |
- ], | |
- ) | |
- | |
- def _pre_config_lock(self, rw_config): | |
- super(UptakeMonitoring, self)._pre_config_lock(rw_config) | |
- # override properties from buildbot properties here as defined by | |
- # taskcluster properties | |
- self.read_buildbot_config() | |
- if not self.buildbot_config: | |
- self.warning("Skipping buildbot properties overrides") | |
- return | |
- props = self.buildbot_config["properties"] | |
- for prop in ['tuxedo_server_url', 'version']: | |
- if props.get(prop): | |
- self.info("Overriding %s with %s" % (prop, props[prop])) | |
- self.config[prop] = props.get(prop) | |
- else: | |
- self.warning("%s could not be found within buildprops" % prop) | |
- return | |
- if props.get('partial_versions'): | |
- partials = [v.strip() for v in props["partial_versions"].split(",")] | |
- self.config["partial_versions"] = [v.split("build")[0] for v in partials] | |
- self.config["platforms"] = [p.strip() for p in | |
- props["platforms"].split(",")] | |
- | |
- def _get_product_uptake(self, tuxedo_server_url, auth, | |
- related_product, os): | |
- from redo import retry | |
- import requests | |
- | |
- url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os) | |
- self.info("Requesting {} from tuxedo".format(url)) | |
- | |
- def get_tuxedo_page(): | |
- r = requests.get(url, auth=auth, | |
- verify=False, timeout=60) | |
- r.raise_for_status() | |
- return r.content | |
- | |
- def calculateUptake(page): | |
- doc = xml.dom.minidom.parseString(page) | |
- uptake_values = [] | |
- | |
- for element in doc.getElementsByTagName('available'): | |
- for node in element.childNodes: | |
- if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \ | |
- node.data.isdigit(): | |
- uptake_values.append(int(node.data)) | |
- if not uptake_values: | |
- uptake_values = [0] | |
- return min(uptake_values) | |
- | |
- page = retry(get_tuxedo_page) | |
- uptake = calculateUptake(page) | |
- self.info("Current uptake for {} is {}".format(related_product, uptake)) | |
- return uptake | |
- | |
- def _get_release_uptake(self, auth): | |
- assert isinstance(self.config["platforms"], (list, tuple)) | |
- | |
- # handle the products first | |
- tuxedo_server_url = self.config["tuxedo_server_url"] | |
- version = self.config["version"] | |
- dl = [] | |
- | |
- for product, info in self.config["products"].iteritems(): | |
- if info.get("check_uptake"): | |
- product_template = info["product-name"] | |
- related_product = product_template % {"version": version} | |
- | |
- enUS_platforms = set(self.config["platforms"]) | |
- paths_platforms = set(info["paths"].keys()) | |
- platforms = enUS_platforms.intersection(paths_platforms) | |
- | |
- for platform in platforms: | |
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform') | |
- dl.append(self._get_product_uptake(tuxedo_server_url, auth, | |
- related_product, bouncer_platform)) | |
- # handle the partials as well | |
- prev_versions = self.config.get("partial_versions", []) | |
- for product, info in self.config.get("partials", {}).iteritems(): | |
- if info.get("check_uptake"): | |
- product_template = info["product-name"] | |
- for prev_version in prev_versions: | |
- subs = { | |
- "version": version, | |
- "prev_version": prev_version | |
- } | |
- related_product = product_template % subs | |
- | |
- enUS_platforms = set(self.config["platforms"]) | |
- paths_platforms = set(info["paths"].keys()) | |
- platforms = enUS_platforms.intersection(paths_platforms) | |
- | |
- for platform in platforms: | |
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform') | |
- dl.append(self._get_product_uptake(tuxedo_server_url, auth, | |
- related_product, bouncer_platform)) | |
- return min(dl) | |
- | |
- def monitor_uptake(self): | |
- credentials_file = os.path.join(os.getcwd(), | |
- self.config["credentials_file"]) | |
- credentials = {} | |
- execfile(credentials_file, credentials) | |
- auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword']) | |
- self.info("Starting the loop to determine the uptake monitoring ...") | |
- | |
- start_time = datetime.datetime.now() | |
- while True: | |
- delta = (datetime.datetime.now() - start_time).seconds | |
- if delta > self.config["poll_timeout"]: | |
- self.error("Uptake monitoring sadly timed-out") | |
- raise Exception("Time-out during uptake monitoring") | |
- | |
- uptake = self._get_release_uptake(auth) | |
- self.info("Current uptake value to check is {}".format(uptake)) | |
- | |
- if uptake >= self.config["min_uptake"]: | |
- self.info("Uptake monitoring is complete!") | |
- break | |
- else: | |
- self.info("Mirrors not yet updated, sleeping for a bit ...") | |
- time.sleep(self.config["poll_interval"]) | |
- | |
- | |
-if __name__ == '__main__': | |
- myScript = UptakeMonitoring() | |
- myScript.run_and_exit() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment