Skip to content

Instantly share code, notes, and snippets.

@dongguosheng
Last active December 19, 2015 17:19
Show Gist options
  • Save dongguosheng/5990639 to your computer and use it in GitHub Desktop.
Save dongguosheng/5990639 to your computer and use it in GitHub Desktop.
a simple threadpool in Python
import time
import threadpool
starttime = time.time()
def job(args):
print args[0]
print args[1]
def main():
urllist = [('http://soso7.gtimg.cn/sosopic/0/6704014884959496047/640', 1),
('http://soso1.gtimg.cn/sosopic/0/5484709051946749088/640', 2)]
# initialize a daemon thread pool for downloading pictures
print 'init daemon thread pool'
pool = threadpool.ThreadPool(job)
# feed the queue with urls
pool.feed_queue(urllist)
# wait for the queue
pool.wait_for_queue()
print 'Time : %s' % (time.time() - starttime)
if __name__ == '__main__':
main()
import threading
import Queue
import urllib
class WorkerThread(threading.Thread):
'''Worker thread waiting to process the job.'''
def __init__(self, queue, job):
threading.Thread.__init__(self)
self.queue = queue
self.daemon = True
self.start()
self.job = job
def run(self):
while(True):
args = self.queue.get()
# do something
self.job(args)
self.queue.task_done()
class ThreadPool(object):
'''Manage the threads.'''
def __init__(self, job, size=2):
self.queue = Queue.Queue()
self.threads = []
self.__init_thread_pool(size, job)
def __init_thread_pool(self, size, job):
'''Initialize a thread pool.'''
for i in range(size):
self.threads.append(WorkerThread(self.queue, job))
def feed_queue(self, args_list):
'''Feed the queue with data.'''
for args in args_list:
self.queue.put(args)
def wait_for_queue(self):
'''Wait for queue.'''
self.queue.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment