gistfile1.txt

#!/usr/bin/env python
# originally from git@github.com:dlrust/python-memcached-stats.git
import re, telnetlib, sys, time, math
import cStringIO as StringIO
from collections import defaultdict

class Transforms:
    _transforms = [
        (re.compile(ur'[_/](\d+)[_/]?'), '_?_'),
        (re.compile(ur':([a-f0-9]{32})$'), ':?'),
        (re.compile(ur'_?([a-f0-9]{32,40})$'), '?'),
    ]

    @classmethod
    def find_replace(self, key):
        for find, replace in self._transforms:
            key = re.sub(find, replace, key)
        return key

    @classmethod
    def nop(self, key): return key


class MemcachedStats:
    _client = None
    _key_regex = re.compile(ur'ITEM (.*) \[(.*) b; (.*) s\]')
    _slab_regex = re.compile(ur'STAT items:(.*):number')
    _stat_regex = re.compile(ur"STAT (.*) (.*)\r")

    def __init__(self, host='localhost', port='11211', sort = True, limit = 1000, transform = Transforms.nop):
        self.host = host
        self.port = port
        self.sort = sort
        self.limit = limit
        self.transform = transform

    @property
    def client(self):
        if self._client is None:
            self._client = telnetlib.Telnet(self.host, self.port)
        return self._client

    def command(self, cmd):
        ' Write a command to telnet and return the response '
        res_buffer = StringIO.StringIO()
        self.client.write("%s\n" % cmd)
        while True:
            resp = self.client.read_very_eager()
            fin = resp[-5:]
            res_buffer.write(resp)
            if fin == "END\r\n": break

        return res_buffer.getvalue()

    def slab_ids(self):
        ' Return a list of slab ids in use '
        return self._slab_regex.findall(self.command('stats items'))

    def stats(self):
        ' Return a dict containing memcached stats '
        return dict(self._stat_regex.findall(self.command('stats')))

    def key_details(self):
        ' Return a list of tuples containing keys and details '
        cmd = 'stats cachedump %s %s'
        return [ [self.transform(key), int(size), int(expiry)] for id in self.slab_ids()
            for key, size, expiry in self._key_regex.findall(self.command(cmd % (id, self.limit)))]

    def keys(self):
        ' Return a list of keys in use '
        keys = [ k for k, e, s in self.key_details() ]
        if self.sort:
            return sorted(set(keys))
        else:
            return set(keys)

    def ntile(self, list, ntile): return list[int(math.floor(len(list) * ntile/100.0))]

    def stats_for_key(self, list):
        return {
            "average": sum(list) / len(list),
            "95th": self.ntile(list, 95),
            "min": min(list),
            "max": max(list),
            "median": self.ntile(list, 50),
        }

    def aggregate(self):
        now = int(time.time())
        total = 0
        data = defaultdict(lambda: dict(size=[], expiry=[], count=0))
        for key, size, expiry in self.key_details():
            data[key]['size'].append(size)
            data[key]['expiry'].append(expiry - now)
            data[key]['count'] += 1
            total += 1

        results = {}
        for key in data.keys():
            results[key] = {
            "size": self.stats_for_key(data[key]['size']),
            "expiry": self.stats_for_key(data[key]['expiry']),
            "weight": 100.0 * data[key]['count'] / total,
            }
        if self.sort:
            return sorted(results.iteritems(), key=lambda (k, v): v['weight'])
        else:
            return [ [k,v] for k,v in results.iteritems() ]

def main(argv=None):
    if not argv: argv = sys.argv
    host = argv[1] if len(argv) >= 2 else '127.0.0.1'
    port = argv[2] if len(argv) >= 3 else '11211'
    import pprint
    m = MemcachedStats(host, port, transform=Transforms.find_replace, limit=1 )
    print m.keys()
    pprint.pprint(m.aggregate())

if __name__ == '__main__':
    main()