jenkins-chef-ci-profiler.py

#!/usr/bin/env python3
import sys
import argparse
from collections import defaultdict
from datetime import datetime,timedelta
import requests
import logging
import traceback

class JenkinsJobProfiler(object):
    def __init__(self, repository, pr, build, username, password):
        self.url = f'https://kube-jenkins.inday.io/chef/job/cookbooks/job/{repository}/job/{pr}/{build}/consoleText'
        self.username = username
        self.password = password
        self.bucket_counts = [2, 4, 8, 16, 32, 64, 128, 256, 512]
        self.buckets = defaultdict(list)

    def fetch_log(self):
        res = requests.get(self.url, auth=(self.username, self.password))
        if res.status_code != 200:
            raise Exception('Something went wrong')


        self.lines = res.text.splitlines()
        if len(self.lines) <= 1:
            raise Exception('Insufficient lines')

    def elapsed(self, a, b):
        da = datetime.fromisoformat(a[1:20])
        db = datetime.fromisoformat(b[1:20])
        return (db - da).seconds

    def nearest_power2(self, n):
        n = n - 1
        while n & n - 1:
            n = n & n - 1
        return n << 1

    def bucketize(self):
        '''
        legends tell if you stare into this long enough to read it,
        you'll require anger management sessions. You've been warned.
        '''
        a = (0, self.lines[0])
        for b, line in enumerate(self.lines[1:]):
            delta = self.elapsed(a[1], line)

            if delta < self.bucket_counts[0]:
                continue

            bucket = self.nearest_power2(delta)
            if bucket >= self.bucket_counts[-1]:
                bucket = self.bucket_counts[-1]

            self.buckets[bucket].append((a[0], delta))
            a = (b, line)

    def reportize(self):
        for bucket in reversed(self.bucket_counts):
            print(f'{bucket} => {len(self.buckets[bucket])}')

    def run(self):
        try:
            logging.info(f'Fetching log from {self.url}')
            self.fetch_log()
            logging.info(f'Preparing histogram')
            self.bucketize()
            logging.info(f'Preparing report')
            self.reportize()
            logging.info(f'Saving report')
            self.save()
        except Exception as e:
            logging.error(f'Exception: {e}')
            print(repr(traceback.format_stack()))
            exit(1)

    def save(self):
        for bucket_index in list(reversed(self.bucket_counts)):
            for bucket in self.buckets[bucket_index]:
                try:
                    line_index, elapsed = bucket
                except Exception as e:
                    logging.info(f'somethign went awry {bucket} {self.buckets}')
                    continue

                try:
                    elapsed = str(timedelta(seconds=elapsed))
                except Exception as e:
                    import ipdb; ipdb.set_trace()
                print(f'## elapsed: {elapsed}')
                # previous line
                #print(f'{self.lines[line_index - 1]}')
                #print(f'{self.lines[line_index]}')

def logging_setup(handlers=[]):
    root = logging.getLogger()
    root.setLevel(logging.DEBUG)

    # disable some stuff that piggy backs
    logging.getLogger("blib2to3").setLevel(logging.ERROR)

    if isinstance(handlers, list) and not len(handlers):
        handlers.append(sys.stdout)

    for handler in handlers:
        streamhandler = logging.StreamHandler(handler)
        streamhandler.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        streamhandler.setFormatter(formatter)
        root.addHandler(streamhandler)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('-r', '--repo', nargs='?', type=str, default='oms')
    parser.add_argument('-P', '--pr', nargs='?', type=str, default='OMSDEPLOY-4031')
    parser.add_argument('-b', '--build', nargs='?', type=int, default=1)
    parser.add_argument('-u', '--username', nargs='?', type=str, default='omsdeploy')
    parser.add_argument('-p', '--password', type=str)
    return parser.parse_args()


def main():
    args = parse_args()
    logging_setup()
    profiler = JenkinsJobProfiler(args.repo, args.pr, args.build, args.username, args.password)
    profiler.run()

if __name__ == "__main__":
    main()