Collect MQ statistics with MQWeb and MQ triggering
''' | |
Get/browse all queue statistic messages from a queue (default is SYSTEM.ADMIN.STATISTICS.QUEUE). | |
MQWeb (http://www.mqweb.org) is used to translate the MQADMIN message to JSON. | |
It is assumed that MQWeb is running on localhost. | |
''' | |
import json | |
import httplib | |
import socket | |
import argparse | |
import sys | |
import datetime | |
import os | |
# We keep some statistic information of our script | |
stats = {} | |
class RunException(Exception): | |
''' | |
Our custom exception which is raised when something fails in the run method. | |
''' | |
pass | |
def request(url, port): | |
''' | |
Send a request to MQWeb and return the answer in JSON format. | |
''' | |
conn = httplib.HTTPConnection('localhost', port) | |
conn.request('GET', url) | |
res = conn.getresponse() | |
return json.loads(res.read()) | |
def createJSON(parameters, statistics): | |
''' | |
Create a JSON representation of the statistics | |
''' | |
return { | |
'startDate': parameters['IntervalStartDate']['value'], | |
'startTime': parameters['IntervalStartTime']['value'], | |
'endDate': parameters['IntervalEndDate']['value'], | |
'endTime': parameters['IntervalEndTime']['value'], | |
'queue': statistics['QName']['value'], | |
'queuemanager': parameters['QMgrName']['value'], | |
'avgTimeOnQ': { | |
'non_persistent': statistics['AvgTimeOnQ']['value'][0], | |
'persistent': statistics['AvgTimeOnQ']['value'][1] | |
}, | |
'expiredMsgCount': statistics['ExpiredMsgCount']['value'], | |
'purgeCount': statistics['PurgeCount']['value'], | |
'nonQueuedMsgCount': statistics['NonQueuedMsgCount']['value'], | |
'depth' : { | |
'min': statistics['QMinDepth']['value'], | |
'max': statistics['QMaxDepth']['value'] | |
}, | |
'get' : { | |
'count': statistics['GetCount']['value'][0] + statistics['GetCount']['value'][1], | |
'bytes': statistics['GetBytes']['value'][0] + statistics['GetBytes']['value'][1], | |
'fail': statistics['GetFailCount']['value'] | |
}, | |
'put' : { | |
'count': statistics['PutCount']['value'][0] + statistics['PutCount']['value'][1] + statistics['Put1Count']['value'][0] + statistics['Put1Count']['value'][1], | |
'bytes': statistics['PutBytes']['value'][0] + statistics['PutBytes']['value'][1], | |
'fail': statistics['PutFailCount']['value'] + statistics['Put1FailCount']['value'] | |
}, | |
'browse' : { | |
'count': statistics['BrowseCount']['value'][0] + statistics['BrowseCount']['value'][1], | |
'bytes': statistics['BrowseBytes']['value'][0] + statistics['BrowseBytes']['value'][1], | |
'fail': statistics['BrowseFailCount']['value'] | |
} | |
} | |
def writeToFile(path, qmgr, output): | |
''' | |
Write the output to <path>/<qmgr>_YYYY-MM-DD.log | |
''' | |
now = datetime.datetime.now() | |
uniqueFilename = path + os.sep + qmgr + '_' + now.strftime('%Y-%m-%d') + '.log' | |
try: | |
with open(uniqueFilename, "a+") as outputFile: | |
outputFile.write(output) | |
outputFile.write(os.linesep) | |
except: | |
raise RunException("Can't open file or write to " + uniqueFilename + ". Check the path and permissions.") | |
def run(qmgr, action = 'get', queue = 'SYSTEM.ADMIN.STATISTICS.QUEUE', port = 8081, path = None): | |
''' | |
Get(or browse) all messages from a queue. All messages are retrieved | |
one by one. When a problem occurs with a message retrieved from MQWeb, | |
the script will end (leaving the other messages on the queue) and the | |
current message will be lost. | |
When a message is not an admin message, it will be discarded. | |
Each message can contain statistics for multiple queues. Each queue will | |
result in a separate JSON structure and is written to stdout or a | |
file (when path argument is set). | |
''' | |
size = 1024 * 32 # 32K is the max. size we use for each message, will this be enough? | |
url = "/api/message/" + action + "/" + qmgr + '/' + queue + '?limit=1&size=' + str(size) | |
try: | |
while True: | |
count = 0 | |
result = request(url, port) | |
if 'error' in result: | |
raise RunException(qmgr + ' - Received a WebSphere MQ error: ' + | |
str(result['error']['reason']['code']) | |
) | |
for message in result['data']: | |
count += 1 | |
if 'admin' in message: | |
parameters = message['admin']['parameters'] | |
if 'QStatisticsData' in parameters: | |
for statistics in parameters['QStatisticsData']['value']: | |
if not parameters['QMgrName']['value'] in stats: | |
stats[parameters['QMgrName']['value']] = 0 | |
output = createJSON(parameters, statistics) | |
if path: | |
writeToFile(path, parameters['QMgrName']['value'], json.dumps(output)) | |
else: | |
print(json.dumps(json)) | |
stats[parameters['QMgrName']['value']] += 1 | |
if count == 0: | |
break | |
except httplib.HTTPException as e: | |
raise RunException('An HTTP error occurred while getting/browsing messages: ' + | |
e.errno + e.strerror | |
) | |
except socket.error as e: | |
raise RunException(e.strerror + os.linesep + 'Is the MQWeb daemon running?') | |
# Check if the script is run standalone | |
if __name__ == '__main__': | |
parser = argparse.ArgumentParser( | |
description=''' | |
mqstats - MQ statistic collector - Get all statistic messages from a | |
queue and output them as JSON. | |
''', | |
epilog='''By default the output is send to stdout. | |
When no queuemanagers are passed, the script will inquire MQWeb to get | |
all available queuemanagers. | |
''' | |
) | |
parser.add_argument('-p', '--port', | |
help ='The port used by the MQWeb HTTP server (default: %(default)s).', | |
type = int, | |
default = 8081) | |
parser.add_argument('-o', '--output', | |
help = 'The name of a folder to write a file with the ouput.', | |
type = str) | |
parser.add_argument('-m', '--queuemanagers', | |
help = 'A comma delimited list with all queuemanagers to process.', | |
type = str) | |
parser.add_argument('-q', '--queue', | |
help = 'The name of the queue to get/browse the statistic messages.', | |
type = str, | |
default= 'SYSTEM.ADMIN.STATISTICS.QUEUE') | |
parser.add_argument('-b', '--browse', | |
help = 'Browse the messages instead of get', | |
action ='store_true') | |
args = parser.parse_args() | |
# Default action is get (means that all messages will be deleted from the queue) | |
# This can be overruled by using argument -b or --browse | |
action = 'get' | |
if args.browse: | |
action = 'browse' | |
if args.output: | |
if not os.path.isdir(args.output): | |
sys.exit(args.output + ' is not a directory!') | |
# When a comma-delimited list of queuemanagers is passed, we use them. | |
# If not, we try to get a list of queuemanagers from MQWeb. | |
if args.queuemanagers: | |
qmgrs = args.queuemanagers.split(',') | |
else: | |
try: | |
result = request("/api/mqweb/list", args.port) | |
qmgrs = result['data'] | |
except httplib.HTTPException as e: | |
sys.exit('An HTTP error occurred while listing all queuemanagers: ' + | |
e.errno + e.strerror | |
) | |
except socket.error as e: | |
sys.exit(e.strerror + os.linesep + 'Is the MQWeb daemon running?') | |
# Run the code for each queuemanager | |
for qmgr in qmgrs: | |
try: | |
run(qmgr, action, args.queue, args.port, args.output) | |
except RunException as e: | |
sys.exit(str(e)) | |
# Dump statistical information about the run of our script | |
total = 0 | |
for qmgr in stats: | |
total += stats[qmgr] | |
print(qmgr + ': ' + str(stats[qmgr])) | |
print(str(total) + " message(s) processed.") |
#!/usr/bin/env python | |
# This script can be used to call the mqstats.py script from a MQ trigger | |
import sys; | |
import mqstats; | |
# Extracts strings from data using the record description. For each field | |
# a tuple must be used with a value: (name, length). A dictionary is returned. | |
def extract(data, record): | |
instance = {} | |
pos = 0 | |
for column in record: | |
name, length = column | |
instance[name] = data[pos:pos+length].strip() | |
pos += length | |
return instance | |
if len(sys.argv) < 2: | |
sys.exit('No trigger data passed to script!') | |
# The only argument passed to this script is a MQTMC2 structure. | |
input = extract(sys.argv[1], [ | |
('structId', 4), | |
('version', 4), | |
('qName', 48), | |
('processName', 48), | |
('triggerData', 64), | |
('applType', 4), | |
('applId', 256), | |
('envData', 128), | |
('userData', 128), | |
('qmgr', 48) | |
]) | |
# Process the userdata. Which is a semi-colon delimited string with output | |
# and port parameters. By default output is set to /var/tmp and the MQWeb port is | |
# set to 8081 | |
input['output'] = '/var/tmp' | |
input['port'] = 8081 | |
if len(input['userData']) > 0: | |
userDataParts = input['userData'].split(';') | |
for userDataPart in userDataParts: | |
key, value = userDataPart.split('=', 2) | |
input[key] = value | |
mqstats.run(qmgr=input['qmgr'], queue=input['qName'], path=input['output'], port=input['port']) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment