Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Script to fetch and CBC rss feed and count the words
#!/usr/bin/env python
import threading,thread
from lxml import etree, html
import urllib
from StringIO import StringIO
# mapper function
def mapper(document):
words = document.split()
for w in words:
#filter the string
w = w.replace("'","").replace('"','')
yield w, 1
#reduce function is just a sum up function
reduce = sum
# global words counter
_words_count = dict()
class MapReduceThread(threading.Thread):
def __init__(self, link):
threading.Thread.__init__(self) = link
def run(self):
words_count = dict()
page = html.parse(
# xpath to get body each paragraph body inside the div which has the id=storybody
content = page.xpath("//div[@id='storybody']/p/text()")
for doc in content:
#map each document(mapper implemented by each document! so we can thread this map and trigger the reduce afterwards)
map = mapper(doc)
for number,weight in map:
# set default return the number in the dictionary, and if the key is not there create a empty list()
words_count.setdefault(number, list()).append(weight)
a_lock = thread.allocate_lock()
with a_lock:
for word, group in words_count.items():
# try to get the number of that word! if is not there get 0
n = _words_count.setdefault(word, 0)
_words_count[word] = n + reduce(group)
def main():
# feed URL
URL = ''
# open the feed
file = urllib.urlopen(URL)
sio = StringIO(
# parse the content into a XML tree
tree = etree.parse(sio)
# xpath to find all links in the feed
arr = tree.xpath('/rss/channel/item/link')
for rr in arr:
threads = []
thread1 = MapReduceThread(rr.text)
print "Error: unable to start thread"
#wait for MapReduce in all documents
for t in threads:
#sort and get the first 50
words_arr = sorted(_words_count, key=_words_count.__getitem__, reverse=True)[:50]
#display them
count = 1
for w in words_arr:
print '%d %d %s' % (count,_words_count[w],w)
count += 1
if __name__ == '__main__':
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment