Collect MQ statistics with MQWeb and MQ triggering

mqstats.py

'''
 Get/browse all queue statistic messages from a queue (default is SYSTEM.ADMIN.STATISTICS.QUEUE).
 MQWeb (http://www.mqweb.org) is used to translate the MQADMIN message to JSON.
 It is assumed that MQWeb is running on localhost.
'''

import json
import httplib
import socket
import argparse
import sys
import datetime
import os

# We keep some statistic information of our script
stats = {}

class RunException(Exception):
    '''
    Our custom exception which is raised when something fails in the run method.
    '''
    pass

def request(url, port):
    '''
    Send a request to MQWeb and return the answer in JSON format.
    '''
    conn = httplib.HTTPConnection('localhost', port)
    conn.request('GET', url)
    res = conn.getresponse()
    return json.loads(res.read())

def createJSON(parameters, statistics):
    '''
    Create a JSON representation of the statistics
    '''
    return {
        'startDate': parameters['IntervalStartDate']['value'],
        'startTime': parameters['IntervalStartTime']['value'],
        'endDate': parameters['IntervalEndDate']['value'],
        'endTime': parameters['IntervalEndTime']['value'],
        'queue': statistics['QName']['value'],
        'queuemanager': parameters['QMgrName']['value'],
        'avgTimeOnQ': {
            'non_persistent': statistics['AvgTimeOnQ']['value'][0],
            'persistent': statistics['AvgTimeOnQ']['value'][1]
        },
        'expiredMsgCount': statistics['ExpiredMsgCount']['value'],
        'purgeCount': statistics['PurgeCount']['value'],
        'nonQueuedMsgCount': statistics['NonQueuedMsgCount']['value'],
        'depth' : {
            'min': statistics['QMinDepth']['value'],
            'max': statistics['QMaxDepth']['value']
        },
        'get' : {
            'count': statistics['GetCount']['value'][0] + statistics['GetCount']['value'][1],
            'bytes': statistics['GetBytes']['value'][0] + statistics['GetBytes']['value'][1],
            'fail': statistics['GetFailCount']['value']
        },
        'put' : {
            'count': statistics['PutCount']['value'][0] + statistics['PutCount']['value'][1] + statistics['Put1Count']['value'][0] + statistics['Put1Count']['value'][1],
            'bytes': statistics['PutBytes']['value'][0] + statistics['PutBytes']['value'][1],
            'fail': statistics['PutFailCount']['value'] + statistics['Put1FailCount']['value']
        },
        'browse' : {
            'count': statistics['BrowseCount']['value'][0] + statistics['BrowseCount']['value'][1],
            'bytes': statistics['BrowseBytes']['value'][0] + statistics['BrowseBytes']['value'][1],
            'fail': statistics['BrowseFailCount']['value']
        }
    }

def writeToFile(path, qmgr, output):
    '''
    Write the output to <path>/<qmgr>_YYYY-MM-DD.log
    '''
    now = datetime.datetime.now()
    uniqueFilename = path + os.sep + qmgr + '_' + now.strftime('%Y-%m-%d') + '.log'
    try:
        with open(uniqueFilename, "a+") as outputFile:
            outputFile.write(output)
            outputFile.write(os.linesep)
    except:
        raise RunException("Can't open file or write to " + uniqueFilename + ". Check the path and permissions.")

def run(qmgr, action = 'get', queue = 'SYSTEM.ADMIN.STATISTICS.QUEUE', port = 8081, path = None):
    '''
    Get(or browse) all messages from a queue. All messages are retrieved
    one by one. When a problem occurs with a message retrieved from MQWeb,
    the script will end (leaving the other messages on the queue) and the
    current message will be lost.

    When a message is not an admin message, it will be discarded.

    Each message can contain statistics for multiple queues. Each queue will
    result in a separate JSON structure and is written to stdout or a
    file (when path argument is set).
    '''
    size = 1024 * 32 # 32K is the max. size we use for each message, will this be enough?

    url = "/api/message/" + action + "/" + qmgr + '/' + queue + '?limit=1&size=' + str(size)

    try:
        while True:
            count = 0
            result = request(url, port)

            if 'error' in result:
                raise RunException(qmgr + ' - Received a WebSphere MQ error: ' +
                    str(result['error']['reason']['code'])
                )

            for message in result['data']:
                count += 1
                if 'admin' in message:
                    parameters = message['admin']['parameters']
                    if 'QStatisticsData' in parameters:
                        for statistics in parameters['QStatisticsData']['value']:
                            if not parameters['QMgrName']['value'] in stats:
                                stats[parameters['QMgrName']['value']] = 0
                            output = createJSON(parameters, statistics)
                            if path:
                                writeToFile(path, parameters['QMgrName']['value'], json.dumps(output))
                            else:
                                print(json.dumps(json))
                            stats[parameters['QMgrName']['value']] += 1

            if count == 0:
                break
    except httplib.HTTPException as e:
        raise RunException('An HTTP error occurred while getting/browsing messages: ' +
            e.errno + e.strerror
            )
    except socket.error as e:
        raise RunException(e.strerror + os.linesep + 'Is the MQWeb daemon running?')

# Check if the script is run standalone
if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='''
            mqstats - MQ statistic collector - Get all statistic messages from a
            queue and output them as JSON.
            ''',
        epilog='''By default the output is send to stdout.
            When no queuemanagers are passed, the script will inquire MQWeb to get
            all available queuemanagers.
            '''
    )
    parser.add_argument('-p', '--port',
        help ='The port used by the MQWeb HTTP server (default: %(default)s).',
        type = int,
        default = 8081)
    parser.add_argument('-o', '--output',
        help = 'The name of a folder to write a file with the ouput.',
        type = str)
    parser.add_argument('-m', '--queuemanagers',
        help = 'A comma delimited list with all queuemanagers to process.',
        type = str)
    parser.add_argument('-q', '--queue',
        help = 'The name of the queue to get/browse the statistic messages.',
        type = str,
        default= 'SYSTEM.ADMIN.STATISTICS.QUEUE')
    parser.add_argument('-b', '--browse',
        help = 'Browse the messages instead of get',
        action ='store_true')
    args = parser.parse_args()

    # Default action is get (means that all messages will be deleted from the queue)
    # This can be overruled by using argument -b or --browse
    action = 'get'
    if args.browse:
        action = 'browse'

    if args.output:
        if not os.path.isdir(args.output):
            sys.exit(args.output + ' is not a directory!')

    # When a comma-delimited list of queuemanagers is passed, we use them.
    # If not, we try to get a list of queuemanagers from MQWeb.
    if args.queuemanagers:
        qmgrs = args.queuemanagers.split(',')
    else:
        try:
            result = request("/api/mqweb/list", args.port)
            qmgrs = result['data']
        except httplib.HTTPException as e:
            sys.exit('An HTTP error occurred while listing all queuemanagers: ' +
               e.errno + e.strerror
               )
        except socket.error as e:
            sys.exit(e.strerror + os.linesep + 'Is the MQWeb daemon running?')

    # Run the code for each queuemanager
    for qmgr in qmgrs:
        try:
            run(qmgr, action, args.queue, args.port, args.output)
        except RunException as e:
            sys.exit(str(e))

    # Dump statistical information about the run of our script
    total = 0
    for qmgr in stats:
        total += stats[qmgr]
        print(qmgr + ': ' + str(stats[qmgr]))
    print(str(total) + " message(s) processed.")

mqstatstrigger.py

#!/usr/bin/env python
# This script can be used to call the mqstats.py script from a MQ trigger
import sys;
import mqstats;

# Extracts strings from data using the record description. For each field
# a tuple must be used with a value: (name, length). A dictionary is returned.
def extract(data, record):
    instance = {}
    pos = 0
    for column in record:
        name, length = column
        instance[name] = data[pos:pos+length].strip()
        pos += length
    return instance

if len(sys.argv) < 2:
    sys.exit('No trigger data passed to script!')

# The only argument passed to this script is a MQTMC2 structure.
input = extract(sys.argv[1], [
    ('structId', 4),
    ('version', 4),
    ('qName', 48),
    ('processName', 48),
    ('triggerData', 64),
    ('applType', 4),
    ('applId', 256),
    ('envData', 128),
    ('userData', 128),
    ('qmgr', 48)
])

# Process the userdata. Which is a semi-colon delimited string with output
# and port parameters. By default output is set to /var/tmp and the MQWeb port is
# set to 8081
input['output'] = '/var/tmp'
input['port'] = 8081
if len(input['userData']) > 0:
    userDataParts = input['userData'].split(';')
    for userDataPart in userDataParts:
        key, value = userDataPart.split('=', 2)
        input[key] = value

mqstats.run(qmgr=input['qmgr'], queue=input['qName'], path=input['output'], port=input['port'])