Script meant to measure fedmsg reliability.

testing-koji-fedmsg.py

#!/usr/bin/env python
""" Collect data on fedmsg reliability.

The gist is that we start up and go into a loop.  Each time we wake up we:
    - Ask koji for all the builds that were started in the last hour.  This
      includes builds that eventually fail or are in progress.
    - Ask fedmsg for all the koji events from the past error.  Throw out all
      the ones except the 'build start' events.
    - Compare the number of new koji builds to the number of fedmsg new build
      messages.  They should be the same.
    - Print out something that can be plotted by gnuplot .CSV style.

:Author: Ralph Bean <rbean@redhat.com>

"""


import datetime
import time
import sys

import koji
import requests

url = "https://koji.fedoraproject.org/koji/buildinfo?buildID=%s"


def get_koji_results(earlier):
    # Get koji build in that period
    c = koji.ClientSession("https://koji.fedoraproject.org/kojihub")
    results = c.listBuilds(createdAfter=earlier)

    build_ids = [result['build_id'] for result in results]
    return set(build_ids)


def get_fedmsg_results(earlier):
    def _make_query(page=0):
        """ Get fedmsg messages in that period """
        resp = requests.get(
            "https://apps.fedoraproject.org/datagrepper/raw",
            params=dict(
                start=earlier,
                topic="org.fedoraproject.prod.buildsys.build.state.change",
                rows_per_page=100,  # This is going to fail if we have more
            )
        )
        data = resp.json()
        return data

    # Grab the first page of results
    data = _make_query()
    messages = data['raw_messages']

    # Grab and smash subsequent pages if there are any
    for page in range(1, data['pages']):
        data = _make_query(page=page)
        messages.extend(data['raw_messages'])

    build_ids = []
    for msg in messages:

        # Throw away all messages that are not "new build" messages.
        if msg['msg']['new'] != 0:
            continue

        build_id = msg['msg']['build_id']
        build_ids.append(build_id)

    return set(build_ids)


def debug(timestamp, koji_results, fedmsg_results):
    for item in koji_results:
        if item not in fedmsg_results:
            print "* For", timestamp, "fedmsg did not have", url % item

    for item in fedmsg_results:
        if item not in koji_results:
            print "* For", timestamp, "koji did not have", url % item


if __name__ == '__main__':
    sleep_interval = 0.25 * 60  # 0.25 minutes
    scan_interval = datetime.timedelta(hours=1)

    print "time, koji results, fedmsg results"
    while True:
        now = time.time()

        earlier_dt = datetime.datetime.now() - scan_interval
        earlier = time.mktime(earlier_dt.timetuple())

        koji_results = get_koji_results(earlier)
        fedmsg_results = get_fedmsg_results(earlier)

        print "%i,%i,%i" % (now, len(koji_results), len(fedmsg_results))
        debug(earlier_dt, koji_results, fedmsg_results)
        sys.stdout.flush()

        time.sleep(sleep_interval)