Collect MQ statistics with MQWeb and MQ triggering
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
''' | |
Get/browse all queue statistic messages from a queue (default is SYSTEM.ADMIN.STATISTICS.QUEUE). | |
MQWeb (http://www.mqweb.org) is used to translate the MQADMIN message to JSON. | |
It is assumed that MQWeb is running on localhost. | |
''' | |
import json | |
import httplib | |
import socket | |
import argparse | |
import sys | |
import datetime | |
import os | |
# We keep some statistic information of our script | |
stats = {} | |
class RunException(Exception): | |
''' | |
Our custom exception which is raised when something fails in the run method. | |
''' | |
pass | |
def request(url, port): | |
''' | |
Send a request to MQWeb and return the answer in JSON format. | |
''' | |
conn = httplib.HTTPConnection('localhost', port) | |
conn.request('GET', url) | |
res = conn.getresponse() | |
return json.loads(res.read()) | |
def createJSON(parameters, statistics): | |
''' | |
Create a JSON representation of the statistics | |
''' | |
return { | |
'startDate': parameters['IntervalStartDate']['value'], | |
'startTime': parameters['IntervalStartTime']['value'], | |
'endDate': parameters['IntervalEndDate']['value'], | |
'endTime': parameters['IntervalEndTime']['value'], | |
'queue': statistics['QName']['value'], | |
'queuemanager': parameters['QMgrName']['value'], | |
'avgTimeOnQ': { | |
'non_persistent': statistics['AvgTimeOnQ']['value'][0], | |
'persistent': statistics['AvgTimeOnQ']['value'][1] | |
}, | |
'expiredMsgCount': statistics['ExpiredMsgCount']['value'], | |
'purgeCount': statistics['PurgeCount']['value'], | |
'nonQueuedMsgCount': statistics['NonQueuedMsgCount']['value'], | |
'depth' : { | |
'min': statistics['QMinDepth']['value'], | |
'max': statistics['QMaxDepth']['value'] | |
}, | |
'get' : { | |
'count': statistics['GetCount']['value'][0] + statistics['GetCount']['value'][1], | |
'bytes': statistics['GetBytes']['value'][0] + statistics['GetBytes']['value'][1], | |
'fail': statistics['GetFailCount']['value'] | |
}, | |
'put' : { | |
'count': statistics['PutCount']['value'][0] + statistics['PutCount']['value'][1] + statistics['Put1Count']['value'][0] + statistics['Put1Count']['value'][1], | |
'bytes': statistics['PutBytes']['value'][0] + statistics['PutBytes']['value'][1], | |
'fail': statistics['PutFailCount']['value'] + statistics['Put1FailCount']['value'] | |
}, | |
'browse' : { | |
'count': statistics['BrowseCount']['value'][0] + statistics['BrowseCount']['value'][1], | |
'bytes': statistics['BrowseBytes']['value'][0] + statistics['BrowseBytes']['value'][1], | |
'fail': statistics['BrowseFailCount']['value'] | |
} | |
} | |
def writeToFile(path, qmgr, output): | |
''' | |
Write the output to <path>/<qmgr>_YYYY-MM-DD.log | |
''' | |
now = datetime.datetime.now() | |
uniqueFilename = path + os.sep + qmgr + '_' + now.strftime('%Y-%m-%d') + '.log' | |
try: | |
with open(uniqueFilename, "a+") as outputFile: | |
outputFile.write(output) | |
outputFile.write(os.linesep) | |
except: | |
raise RunException("Can't open file or write to " + uniqueFilename + ". Check the path and permissions.") | |
def run(qmgr, action = 'get', queue = 'SYSTEM.ADMIN.STATISTICS.QUEUE', port = 8081, path = None): | |
''' | |
Get(or browse) all messages from a queue. All messages are retrieved | |
one by one. When a problem occurs with a message retrieved from MQWeb, | |
the script will end (leaving the other messages on the queue) and the | |
current message will be lost. | |
When a message is not an admin message, it will be discarded. | |
Each message can contain statistics for multiple queues. Each queue will | |
result in a separate JSON structure and is written to stdout or a | |
file (when path argument is set). | |
''' | |
size = 1024 * 32 # 32K is the max. size we use for each message, will this be enough? | |
url = "/api/message/" + action + "/" + qmgr + '/' + queue + '?limit=1&size=' + str(size) | |
try: | |
while True: | |
count = 0 | |
result = request(url, port) | |
if 'error' in result: | |
raise RunException(qmgr + ' - Received a WebSphere MQ error: ' + | |
str(result['error']['reason']['code']) | |
) | |
for message in result['data']: | |
count += 1 | |
if 'admin' in message: | |
parameters = message['admin']['parameters'] | |
if 'QStatisticsData' in parameters: | |
for statistics in parameters['QStatisticsData']['value']: | |
if not parameters['QMgrName']['value'] in stats: | |
stats[parameters['QMgrName']['value']] = 0 | |
output = createJSON(parameters, statistics) | |
if path: | |
writeToFile(path, parameters['QMgrName']['value'], json.dumps(output)) | |
else: | |
print(json.dumps(json)) | |
stats[parameters['QMgrName']['value']] += 1 | |
if count == 0: | |
break | |
except httplib.HTTPException as e: | |
raise RunException('An HTTP error occurred while getting/browsing messages: ' + | |
e.errno + e.strerror | |
) | |
except socket.error as e: | |
raise RunException(e.strerror + os.linesep + 'Is the MQWeb daemon running?') | |
# Check if the script is run standalone | |
if __name__ == '__main__': | |
parser = argparse.ArgumentParser( | |
description=''' | |
mqstats - MQ statistic collector - Get all statistic messages from a | |
queue and output them as JSON. | |
''', | |
epilog='''By default the output is send to stdout. | |
When no queuemanagers are passed, the script will inquire MQWeb to get | |
all available queuemanagers. | |
''' | |
) | |
parser.add_argument('-p', '--port', | |
help ='The port used by the MQWeb HTTP server (default: %(default)s).', | |
type = int, | |
default = 8081) | |
parser.add_argument('-o', '--output', | |
help = 'The name of a folder to write a file with the ouput.', | |
type = str) | |
parser.add_argument('-m', '--queuemanagers', | |
help = 'A comma delimited list with all queuemanagers to process.', | |
type = str) | |
parser.add_argument('-q', '--queue', | |
help = 'The name of the queue to get/browse the statistic messages.', | |
type = str, | |
default= 'SYSTEM.ADMIN.STATISTICS.QUEUE') | |
parser.add_argument('-b', '--browse', | |
help = 'Browse the messages instead of get', | |
action ='store_true') | |
args = parser.parse_args() | |
# Default action is get (means that all messages will be deleted from the queue) | |
# This can be overruled by using argument -b or --browse | |
action = 'get' | |
if args.browse: | |
action = 'browse' | |
if args.output: | |
if not os.path.isdir(args.output): | |
sys.exit(args.output + ' is not a directory!') | |
# When a comma-delimited list of queuemanagers is passed, we use them. | |
# If not, we try to get a list of queuemanagers from MQWeb. | |
if args.queuemanagers: | |
qmgrs = args.queuemanagers.split(',') | |
else: | |
try: | |
result = request("/api/mqweb/list", args.port) | |
qmgrs = result['data'] | |
except httplib.HTTPException as e: | |
sys.exit('An HTTP error occurred while listing all queuemanagers: ' + | |
e.errno + e.strerror | |
) | |
except socket.error as e: | |
sys.exit(e.strerror + os.linesep + 'Is the MQWeb daemon running?') | |
# Run the code for each queuemanager | |
for qmgr in qmgrs: | |
try: | |
run(qmgr, action, args.queue, args.port, args.output) | |
except RunException as e: | |
sys.exit(str(e)) | |
# Dump statistical information about the run of our script | |
total = 0 | |
for qmgr in stats: | |
total += stats[qmgr] | |
print(qmgr + ': ' + str(stats[qmgr])) | |
print(str(total) + " message(s) processed.") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# This script can be used to call the mqstats.py script from a MQ trigger | |
import sys; | |
import mqstats; | |
# Extracts strings from data using the record description. For each field | |
# a tuple must be used with a value: (name, length). A dictionary is returned. | |
def extract(data, record): | |
instance = {} | |
pos = 0 | |
for column in record: | |
name, length = column | |
instance[name] = data[pos:pos+length].strip() | |
pos += length | |
return instance | |
if len(sys.argv) < 2: | |
sys.exit('No trigger data passed to script!') | |
# The only argument passed to this script is a MQTMC2 structure. | |
input = extract(sys.argv[1], [ | |
('structId', 4), | |
('version', 4), | |
('qName', 48), | |
('processName', 48), | |
('triggerData', 64), | |
('applType', 4), | |
('applId', 256), | |
('envData', 128), | |
('userData', 128), | |
('qmgr', 48) | |
]) | |
# Process the userdata. Which is a semi-colon delimited string with output | |
# and port parameters. By default output is set to /var/tmp and the MQWeb port is | |
# set to 8081 | |
input['output'] = '/var/tmp' | |
input['port'] = 8081 | |
if len(input['userData']) > 0: | |
userDataParts = input['userData'].split(';') | |
for userDataPart in userDataParts: | |
key, value = userDataPart.split('=', 2) | |
input[key] = value | |
mqstats.run(qmgr=input['qmgr'], queue=input['qName'], path=input['output'], port=input['port']) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment