Skip to content

Instantly share code, notes, and snippets.

@jlyu
Created July 11, 2012 01:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jlyu/3087266 to your computer and use it in GitHub Desktop.
Save jlyu/3087266 to your computer and use it in GitHub Desktop.
from Queue import Queue
from threading import Thread
import time
import feedparser
num_fetch_threads = 2
enclosure_queue = Queue()
feed_urls = ['http://www.castsampler.com/cast/feed/rss/guest',]
def downloadEnclosures(i, q):
while True:
print '%s: Looking for the next enclosure' % i
url = q.get()
print '%s: Downloading:' % i, url
time.sleep(i + 2)
q.task_done()
for i in range(num_fetch_threads):
worker = Thread(target=downloadEnclosures, args=(i, enclosure_queue,))
worker.setDaemon(True)
worker.start()
for url in feed_urls:
response = feedparser.parse(url, agent='fetch_podcasts.py')
for entry in response['entries']:
for enclosure in entry.get('enclosures', []):
print 'Queuing:', enclosure['url']
enclosure_queue.put(enclosure['url'])
print '*** Main thread waiting'
enclosure_queue.join()
print '*** Done'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment