MapReduce top 100 word in Harry 6

gistfile1.txt

1	7558	the
2	4294	to
3	4053	and
4	3419	of
5	3314	he
6	3283	a
7	2617	harry
8	2449	said
9	2396	was
10	2249	his
11	2239	you
12	2134	i
13	1978	that
14	1923	in
15	1902	it
16	1658	had
17	1302	at
18	1241	as
19	1137	not
20	1104	but
21	1071	with
22	1032	him
23	986	for
24	941	on
25	876	dumbledore
26	850	her
27	805	have
28	799	ron
29	788	she
30	764	be
31	685	what
32	662	they
33	651	hermione
34	645	all
35	628	out
36	598	this
37	582	were
38	570	there
39	560	up	from
40	552	so
41	550	been
42	540	is
43	523	them
44	521	into
45	518	who
46	515	me
47	484	no
48	475	could
49	453	would
50	451	did
51	448	an
52	446	we
53	435	now
54	425	do
55	421	back
56	401	one
57	400	then
58	391	when
59	388	by
60	383	well	your
61	380	just
62	379	about
63	377	over
64	374	if
65	367	know
66	352	like
67	344	or
68	343	slughorn
69	339	snape
70	335	think	looked
71	333	are
72	329	very
73	326	malfoy	more	don't
74	317	see
75	311	time
76	306	their
77	305	though
78	297	down
79	294	my
80	289	around
81	284	again
82	279	got
83	277	professor
84	275	how
85	268	still
86	250	room
87	249	looking	thought
88	245	before
89	244	which	will	off
90	243	than
91	239	look
92	238	once
93	236	right
94	231	yes	can
95	230	it's	little
96	225	asked
97	221	face
98	218	ginny
99	211	get
100	210	hand

top100-map.py

#!/usr/bin/env python

import sys

for line in sys.stdin:
    line = line.strip()
    if line:
      word, count = line.split()
      print "%07d\t%s" % (1000000-int(count), word)

top100-reduce.py

#!/usr/bin/env python

import sys

current_words = []
current_count = None
pos = 0

for line in sys.stdin:

  line = line.strip()

  if not line:
    break
  count, word = line.split('\t', 1)

  count = 1000000-int(count)
  if(count != current_count):

    # print old list
    if current_count is not None and pos <= 100:
      print "%d\t%d\t%s" % (pos, current_count, "\t".join(current_words))

    # init new list
    pos += 1
    current_count = count
    current_words = []
  
  current_words.append(word)

# final list
if pos <= 100:
  print "%d\t%d\t%s" % (pos, current_count, "\t".join(current_words))


  

wordcount-map.py

#!/usr/bin/env python

import re
import sys

for line in sys.stdin:
    line = line.strip()
    
    words = re.findall('((?:[A-Za-z][^\s]*[A-Za-z])|(?:[A-Za-z]))', line.lower())

    for word in words:
        print "%s\t%d" % (word, 1)

wordcount-reduce.py

#!/usr/bin/env python

import sys

current_word = None
current_count = None

for line in sys.stdin:
  line = line.strip()

  word, count = line.split('\t', 1)

  count = int(count)

  if (word != current_word):
      if(current_word):
        print "%s\t%d" % (current_word, current_count)
      current_count = count
      current_word = word
  else:
    current_count += count

if(current_word):
  print "%s\t%d" % (current_word, current_count)