Example script using several process in order to consume messages put in a message queue.

multiprocessqueue.py

#!/usr/bin/env python
# -*- coding:utf-8 -*-

"""
    Example script using several process in order to consume
    messages put in a message queue.
"""

from multiprocessing import Process, Queue, cpu_count

def push_msgs(queue, nb_msgs, nb_pullers):
    print "<pusher process starts>"
    for i in xrange(nb_msgs):
        queue.put([i, 42, 'hello'])
    for i in xrange(nb_pullers):
        queue.put("END")
    print "<pusher process exits>"


def pull_msgs(queue):
    print "<puller process starts>"
    treated_msgs = 0
    while True:
        msg = queue.get()
        if msg == "END":
            break
        treated_msgs += 1
    print "<puller process exit after %d msgs>" % treated_msgs


def main():
    print "<main process starts>"
    q = Queue()
    nb_messages = 10000
    nb_pullers = cpu_count() - 1

    msg_pusher_process = Process(target=push_msgs, args=(q, nb_messages, nb_pullers))
    msg_pusher_process.start()
    
    for i in xrange(nb_pullers):
        msg_puller_process = Process(target=pull_msgs, args=(q,))
        msg_puller_process.start()

    print "<main process exits>"


if __name__ == '__main__':
    main()