MemcachedPool.py

import pylibmc
import Queue
import logging
import functools

"""
This is a transparent pool library that wraps a pylibmc client

from MemcachePool import mc
mc.get(key)
mc.set(key, value)

"""

import settings
def servers():
    return settings.get('memcached')

server_pool = Queue.Queue(maxsize=50) # of max cached connections; not max # of open connections

def newConnection():
    """setup a new memcached connection"""
    if callable(servers):
        return pylibmc.Client(servers())
    else:
        return pylibmc.Client(servers)

def pool(method):
    """
    this decorator implements a self growing pool and returns 
    connections to it
    If there is not a connection available in the pool create one
    If we can't return the connection to the pool, drop it 
    """
    @functools.wraps(method)
    def wrapper(*args, **kwargs):
        try:
            conn = server_pool.get_nowait()
        except Queue.Empty:
            conn = newConnection()
        val = None
        try:
            ## method is wrapperFunction in PoolClient
            key = args[0]
            if not isinstance(key, (str, list, tuple)):
                logging.warning('%r must be _utf8' % key)
            try:
                # convert to utf8 on the fly since memcached chokes on unicode objects
                args = [_utf8(arg) for arg in args]
                val = method(conn, *args, **kwargs)
            except:
                logging.exception('failed memcached call %s %s' % (str(args), str(kwargs)))
                val = None
        finally:
            try:
                server_pool.put_nowait(conn)
            except Queue.Full:
                pass
        return val
    return wrapper

class PoolClient(object):
    """
    This is a wrapper class that passes all methods on to the actual client
    getting a connection from the pool first, and returning it there after
    """
    def __getattribute__(self,name):
        @pool
        def wrapperFunction(*args, **kwargs):
            ## called by acquireAndRelease;
            ## args[0] == pylibmc.Client
            ## name == method to call
            ## args[1:] the original method arguments

            ## debug:
            # print "memcache", name, args[1]
            return getattr(args[0], name)(*args[1:], **kwargs)
        return wrapperFunction
            
mc = PoolClient()

def _utf8(s):
    if isinstance(s, unicode):
        return s.encode("utf-8")
    return s