Skip to content

Instantly share code, notes, and snippets.

@natbusa
Last active November 2, 2024 11:12
Show Gist options
  • Save natbusa/9652200 to your computer and use it in GitHub Desktop.
Save natbusa/9652200 to your computer and use it in GitHub Desktop.
word count: mapper and reducer in python using hadoop streaming
#!/usr/bin/env python
import sys
# input comes from STDIN (standard input)
for line in sys.stdin:
#clean and split in words
linechars = [c for c in line.lower() if c.isalpha() or c==' ']
words = ''.join(linechars).strip().split()
#emit the key-balue pairs
for word in words:
print '%s\t%s' % (word, 1)
#!/usr/bin/env python
from operator import itemgetter
import sys
current_word = None
current_count = 0
word = None
# input comes from STDIN
for line in sys.stdin:
# remove leading and trailing whitespace
line = line.strip()
# parse the input we got from mapper.py
word, count = line.split('\t', 1)
# convert count (currently a string) to int
try:
count = int(count)
except ValueError:
# count was not a number, so silently
# ignore/discard this line
continue
# this IF-switch only works because Hadoop sorts map output
# by key (here: word) before it is passed to the reducer
if current_word == word:
current_count += count
else:
if current_word:
# write result to STDOUT
print '%s\t%s' % (current_word, current_count)
current_count = count
current_word = word
# do not forget to output the last word if needed!
if current_word == word:
print '%s\t%s' % (current_word, current_count)
$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-2.3.0.jar \
-mapper ./mapper.py
-reducer ./reducer.py
-input wordcount-input
-output wordcount-mapreduce-streaming-python-output
$HADOOP_HOME/bin/hadoop fs cat wordcount-mapreduce-streaming-python-output/*
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment