Skip to content

Instantly share code, notes, and snippets.

@deniszh

deniszh/1.py

Created May 29, 2016 16:34
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save deniszh/431e916c786f630928b7a9f5f7a0a4fd to your computer and use it in GitHub Desktop.
Save deniszh/431e916c786f630928b7a9f5f7a0a4fd to your computer and use it in GitHub Desktop.
Download ZIP
Raw
1.py
#!/usr/bin/env python
# coding=utf-8
#
# collectd-iostat-python
# ======================
#
# Collectd-iostat-python is an iostat plugin for collectd that allows you to
# graph Linux iostat metrics in Graphite or other output formats that are
# supported by collectd.
#
# https://github.com/powdahound/redis-collectd-plugin
# - was used as template
# https://github.com/keirans/collectd-iostat/
# - was used as inspiration and contains some code from
# https://bitbucket.org/jakamkon/python-iostat
# - by Kuba Kończyk <jakamkon at users.sourceforge.net>
#
import signal
import string
import subprocess
import sys
__version__ = '0.0.3'
__author__ = 'denis.zhdanov@gmail.com'
class IOStatError(Exception):
pass
class CmdError(IOStatError):
pass
class ParseError(IOStatError):
pass
class IOStat(object):
def __init__(self, path='/usr/bin/iostat', interval=2, count=2, disks=[]):
self.path = path
self.interval = interval
self.count = count
self.disks = disks
def parse_diskstats(self, input):
"""
Parse iostat -d and -dx output.If there are more
than one series of statistics, get the last one.
By default parse statistics for all avaliable block devices.
@type input: C{string}
@param input: iostat output
@type disks: list of C{string}s
@param input: lists of block devices that
statistics are taken for.
@return: C{dictionary} contains per block device statistics.
Statistics are in form of C{dictonary}.
Main statistics:
tps Blk_read/s Blk_wrtn/s Blk_read Blk_wrtn
Extended staistics (available with post 2.5 kernels):
rrqm/s wrqm/s r/s w/s rsec/s wsec/s rkB/s wkB/s avgrq-sz \
avgqu-sz await svctm %util
See I{man iostat} for more details.
"""
dstats = {}
dsi = input.rfind('Device:')
if dsi == -1:
raise ParseError('Unknown input format: %r' % input)
ds = input[dsi:].splitlines()
hdr = ds.pop(0).split()[1:]
for d in ds:
if d:
d = d.split()
dev = d.pop(0)
if (dev in self.disks) or not self.disks:
dstats[dev] = dict([(k, float(v)) for k, v in zip(hdr, d)])
return dstats
def sum_dstats(self, stats, smetrics):
"""
Compute the summary statistics for chosen metrics.
"""
avg = {}
for disk, metrics in stats.iteritems():
for mname, metric in metrics.iteritems():
if mname not in smetrics:
continue
if mname in avg:
avg[mname] += metric
else:
avg[mname] = metric
return avg
def _run(self, options=None):
"""
Run iostat command.
"""
close_fds = 'posix' in sys.builtin_module_names
args = '%s %s %s %s %s' % (
self.path,
''.join(options),
self.interval,
self.count,
' '.join(self.disks))
return subprocess.Popen(
args,
bufsize=1,
shell=True,
stdout=subprocess.PIPE,
close_fds=close_fds)
@staticmethod
def _get_childs_data(child):
"""
Return child's data when avaliable.
"""
(stdout, stderr) = child.communicate()
ecode = child.poll()
if ecode != 0:
raise CmdError('Command %r returned %d' % (child.cmd, ecode))
return stdout
def get_diskstats(self):
"""
Get all avaliable disks statistics that we can get.
"""
#dstats = self._run(options=['-kNd'])
#extdstats = self._run(options=['-kNdx'])
#dsd = self._get_childs_data(dstats)
#edd = self._get_childs_data(extdstats)
dsd = """Linux 4.1.12-gentoo (clx-db1) 05/24/2016 x86_64 (12 CPU)
Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn
sda 12.93 88.89 30.58 1241919812 427271402
sdb 1322.35 1403.23 13797.26 19604599843 192761939040
clx--db1--vg-swap 0.12 0.20 0.26 2861300 3635740
clx--db1--vg-tmp 0.00 0.00 0.00 6845 34276
clx--db1--vg-var 0.63 6.31 4.49 88117685 62751656
clx--db1--vg-mysql 1143.61 78.84 12862.92 1101464313 179708276120
clx--db1--vg-pgsql 197.20 1401.04 959.92 19573910790 13411047616
drbd1 196.17 1395.62 954.30 19498315066 13332606912
drbd0 19.12 55.15 113.68 770446622 1588215328"""
edd = """Linux 4.1.12-gentoo (clx-db1) 05/24/2016 x86_64 (12 CPU)
Device: rrqm/s wrqm/s r/s w/s rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util
sda 0.00 0.24 9.04 3.89 88.89 30.58 18.48 0.12 9.06 6.27 15.55 3.26 4.21
sdb 0.00 6.44 79.30 1243.05 1403.23 13797.26 22.99 0.04 0.03 2.33 0.13 0.19 25.51
clx--db1--vg-swap 0.00 0.00 0.05 0.07 0.20 0.26 8.00 0.00 20.24 14.42 24.82 1.86 0.02
clx--db1--vg-tmp 0.00 0.00 0.00 0.00 0.00 0.00 8.14 0.00 3.04 5.77 2.55 1.39 0.00
clx--db1--vg-var 0.00 0.00 0.11 0.52 6.31 4.49 34.40 0.00 7.83 24.72 4.34 0.75 0.05
clx--db1--vg-mysql 0.00 0.00 3.75 1139.86 78.84 12862.92 22.63 0.01 0.01 6.48 0.26 0.13 14.86
clx--db1--vg-pgsql 0.00 0.00 84.05 113.14 1401.03 959.92 23.95 0.29 1.48 2.65 0.60 1.24 24.38
drbd1 0.00 0.00 84.00 112.17 1395.62 954.30 23.96 0.28 1.45 2.66 0.55 1.25 24.55
drbd0 0.00 0.00 2.46 16.66 55.15 113.68 17.66 0.02 1.28 6.00 0.59 0.49 0.93"""
ds = self.parse_diskstats(dsd)
eds = self.parse_diskstats(edd)
for dk, dv in ds.iteritems():
if dk in eds:
ds[dk].update(eds[dk])
return ds
class IOMon(object):
def __init__(self):
self.plugin_name = 'collectd-iostat-python'
self.iostat_path = '/usr/bin/iostat'
self.iostat_interval = 2
self.iostat_count = 2
self.iostat_disks = []
self.iostat_nice_names = False
self.iostat_disks_regex = ''
self.verbose_logging = False
self.names = {
'tps': {'t': 'transfers_per_second'},
'Blk_read/s': {'t': 'blocks_per_second', 'ti': 'read'},
'kB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e3},
'MB_read/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e6},
'Blk_wrtn/s': {'t': 'blocks_per_second', 'ti': 'write'},
'kB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e3},
'MB_wrtn/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e6},
'Blk_read': {'t': 'blocks', 'ti': 'read'},
'kB_read': {'t': 'bytes', 'ti': 'read', 'm': 10e3},
'MB_read': {'t': 'bytes', 'ti': 'read', 'm': 10e6},
'Blk_wrtn': {'t': 'blocks', 'ti': 'write'},
'kB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 10e3},
'MB_wrtn': {'t': 'bytes', 'ti': 'write', 'm': 10e6},
'rrqm/s': {'t': 'requests_merged_per_second', 'ti': 'read'},
'wrqm/s': {'t': 'requests_merged_per_second', 'ti': 'write'},
'r/s': {'t': 'per_second', 'ti': 'read'},
'w/s': {'t': 'per_second', 'ti': 'write'},
'rsec/s': {'t': 'sectors_per_second', 'ti': 'read'},
'rkB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e3},
'rMB/s': {'t': 'bytes_per_second', 'ti': 'read', 'm': 10e6},
'wsec/s': {'t': 'sectors_per_second', 'ti': 'write'},
'wkB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e3},
'wMB/s': {'t': 'bytes_per_second', 'ti': 'write', 'm': 10e6},
'avgrq-sz': {'t': 'avg_request_size'},
'avgqu-sz': {'t': 'avg_request_queue'},
'await': {'t': 'avg_wait_time'},
'r_await': {'t': 'avg_wait_time', 'ti': 'read'},
'w_await': {'t': 'avg_wait_time', 'ti': 'write'},
'svctm': {'t': 'avg_service_time'},
'%util': {'t': 'percent', 'ti': 'util'}
}
def log_verbose(self, msg):
if not self.verbose_logging:
return
collectd.info('%s plugin [verbose]: %s' % (self.plugin_name, msg))
def configure_callback(self, conf):
"""
Receive configuration block
"""
for node in conf.children:
val = str(node.values[0])
if node.key == 'Path':
self.iostat_path = val
elif node.key == 'Interval':
self.iostat_interval = int(float(val))
elif node.key == 'Count':
self.iostat_count = int(float(val))
elif node.key == 'Disks':
self.iostat_disks = val.split(',')
elif node.key == 'NiceNames':
self.iostat_nice_names = val in ['True', 'true']
elif node.key == 'DisksRegex':
self.iostat_disks_regex = val
elif node.key == 'PluginName':
self.plugin_name = val
elif node.key == 'Verbose':
self.verbose_logging = val in ['True', 'true']
else:
collectd.warning(
'%s plugin: Unknown config key: %s.' % (
self.plugin_name,
node.key))
self.log_verbose(
'Configured with iostat=%s, interval=%s, count=%s, disks=%s, '
'disks_regex=%s' % (
self.iostat_path,
self.iostat_interval,
self.iostat_count,
self.iostat_disks,
self.iostat_disks_regex))
def dispatch_value(self, plugin_instance, val_type, type_instance, value):
"""
Dispatch a value to collectd
"""
self.log_verbose(
'Sending value: %s-%s.%s=%s' % (
self.plugin_name,
plugin_instance,
'-'.join([val_type, type_instance]),
value))
val = collectd.Values()
val.plugin = self.plugin_name
val.plugin_instance = plugin_instance
val.type = val_type
if len(type_instance):
val.type_instance = type_instance
val.values = [value, ]
val.meta={'0': True}
val.dispatch()
def read_callback(self):
"""
Collectd read callback
"""
self.log_verbose('Read callback called')
iostat = IOStat(
path=self.iostat_path,
interval=self.iostat_interval,
count=self.iostat_count,
disks=self.iostat_disks)
ds = iostat.get_diskstats()
if not ds:
self.log_verbose('%s plugin: No info received.' % self.plugin_name)
return
for disk in ds:
for name in ds[disk]:
if self.iostat_nice_names and name in self.names:
val_type = self.names[name]['t']
if 'ti' in self.names[name]:
type_instance = self.names[name]['ti']
else:
type_instance = ''
value = ds[disk][name]
if 'm' in self.names[name]:
value *= self.names[name]['m']
else:
val_type = 'gauge'
tbl = string.maketrans('/-%', '___')
type_instance = name.translate(tbl)
value = ds[disk][name]
self.dispatch_value(
disk, val_type, type_instance, value)
def restore_sigchld():
"""
Restore SIGCHLD handler for python <= v2.6
It will BREAK exec plugin!!!
See https://github.com/deniszh/collectd-iostat-python/issues/2 for details
"""
if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
signal.signal(signal.SIGCHLD, signal.SIG_DFL)
if __name__ == '__main__':
iostat = IOStat()
ds = iostat.get_diskstats()
for disk in ds:
for metric in ds[disk]:
tbl = string.maketrans('/-%', '___')
metric_name = metric.translate(tbl)
print("%s.%s:%s" % (disk, metric_name, ds[disk][metric]))
sys.exit(0)
else:
import collectd
iomon = IOMon()
# Register callbacks
collectd.register_init(restore_sigchld)
collectd.register_config(iomon.configure_callback)
collectd.register_read(iomon.read_callback)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment