multi-cpu_bzip2_with_python.py

import os 
import sys 
from threading import Thread, Lock 
from Queue import Queue 

def report(message): 
     mutex.acquire() 
     print message 
     sys.stdout.flush() 
     mutex.release() 

class Compressor(Thread): 
     
     def __init__(self, in_queue, out_queue): 
         Thread.__init__(self) 
         self.in_queue = in_queue 
         self.out_queue = out_queue 
         
     def run(self): 
         while True: 
             path = self.in_queue.get() 
             sys.stdout.flush() 
             if path is None: 
                 break 
             report("Compressing %s" % path) 
             os.system("bzip2 %s" % path) 
             report("Done %s" %  path) 
             self.out_queue.put(path) 

in_queue = Queue() 

out_queue = Queue() 

mutex = Lock() 

THREAD_COUNT = 4 

worker_list = [] 

for i in range(THREAD_COUNT): 
     worker = Compressor(in_queue, out_queue) 
     worker.start() 
     worker_list.append(worker) 

for roots, dirlist, filelist in os.walk(os.curdir): 
     for file in [os.path.join(roots, filegot) for filegot in filelist]: 
         if "bz2" not in file: 
             in_queue.put(file) 

for i in range(THREAD_COUNT): 
     in_queue.put(None) 

for worker in worker_list: 
     worker.join()