Last active
February 22, 2019 02:35
-
-
Save krizex/0efa3b0ca64f1dffa16508df7a9ea26b to your computer and use it in GitHub Desktop.
A pipeline example
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from threading import Thread | |
from Queue import Queue | |
from time import sleep | |
q1 = Queue() | |
q2 = Queue() | |
# task generater | |
def producer1(): | |
for i in range(10): | |
q1.put(i) | |
# downloader | |
def consumer1(): | |
while True: | |
item = q1.get() | |
f = download_f(item) | |
sleep(1) | |
q2.put(f) | |
q1.task_done() | |
def download_f(item): | |
print 'downloading {}'.format(item) | |
return 'downloaded_{}'.format(item) | |
# restorer | |
def consumer2(): | |
while True: | |
item = q2.get() | |
v = process_f(item) | |
print v | |
q2.task_done() | |
def process_f(item): | |
print 'restoring {}'.format(item) | |
return 'restored_{}'.format(item) | |
if __name__ == '__main__': | |
cons1 = Thread(target=consumer1) | |
cons1.daemon=True | |
cons1.start() | |
cons2 = Thread(target=consumer2) | |
cons2.daemon=True | |
cons2.start() | |
# main thread | |
producer1() | |
q1.join() | |
q2.join() | |
print 'finish' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment