A pipeline example

pipeline_example.py

from threading import Thread
from Queue import Queue
from time import sleep

q1 = Queue()
q2 = Queue()

# task generater
def producer1():
    for i in range(10):
        q1.put(i)
        
# downloader
def consumer1():
    while True:
        item = q1.get()
        f = download_f(item)
        sleep(1)
        q2.put(f)
        q1.task_done()
        
        
        
def download_f(item):
    print 'downloading {}'.format(item)
    return 'downloaded_{}'.format(item)

# restorer
def consumer2():
    while True:
        item = q2.get()
        v = process_f(item)
        print v
        q2.task_done()
        
def process_f(item):
    print 'restoring {}'.format(item)
    return 'restored_{}'.format(item)
        
    
if __name__ == '__main__':

    cons1 = Thread(target=consumer1)
    cons1.daemon=True
    cons1.start()
    
    cons2 = Thread(target=consumer2)
    cons2.daemon=True
    cons2.start()
    
    # main thread
    producer1()
    q1.join()
    q2.join()

    print 'finish'