A custom handler for Mailman which discards messages with a Message ID that has already been seen for a particular list. A cache of seen IDs is maintained in a pickled dictionary, which is periodically cleansed of old entries.

DeDuper.py

import os
import errno
import time
import cPickle

from Mailman.Errors import DiscardMessage
from Mailman.Logging.Syslog import syslog

def process(mlist, msg, msgdata):
    msgid = msg.get('message-id', None)
    if not msgid:
        return
    filename = os.path.join(mlist.fullpath(), 'message-ids.pck')
    try:
        fp = open(filename, 'r')
        pairs = cPickle.load(fp)
        if pairs.has_key(msgid):
            syslog('vette', '%s post from %s with message-id=%s is a duplicate',
                    mlist.internal_name(), msg.get_sender(), msgid)
            raise DiscardMessage
        else:
            pairs[msgid] = time.time()
            remove = [k for k,v in pairs.iteritems() if pairs[msgid]-v > 300]
            for k in remove: del pairs[k]
            fp = open(filename, 'w')
            cPickle.dump(pairs, fp)
    except IOError, e:
        if e.errno == errno.ENOENT:
            fp = open(filename, 'w')
            pairs = {}
            pairs[msgid] = time.time()
            cPickle.dump(pairs, fp)
    finally:
        fp.close()