Skip to content

Instantly share code, notes, and snippets.

@malaikannan
Created December 4, 2015 05:05
Show Gist options
  • Save malaikannan/b6e5efee6b6a0c40b3f7 to your computer and use it in GitHub Desktop.
Save malaikannan/b6e5efee6b6a0c40b3f7 to your computer and use it in GitHub Desktop.
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.util.ReflectionUtils;
public class TestMapReduce {
public static class ColorMapper extends Mapper<Object, Text, Text, ColorWritable> {
private Text MyKey = new Text("A");
public void map(Object key, Text value, Context context) throws IOException, InterruptedException
{
String word = value.toString();
Context.write(MyKey, new ColorWritable(new Text(word)));
}
}
public static class ColorReducer extends Reducer<Text, ColorWritable, Text, Text>
{
public void reduce(Text key, Iterator<ColorWritable> values,Context context) throws IOException
{
try {
ArrayList<ColorWritable> list = new ArrayList<ColorWritable>();
Configuration conf = context.getConfiguration();
while (values.hasNext()) {
ColorWritable n = ReflectionUtils.newInstance(
ColorWritable.class, conf);
ReflectionUtils.copy(conf, values.next(), n);
list.add(n);
}
for (int i = 0; i < list.size(); i++) {
ColorWritable c = list.get(i);
output.collect(key, new Text(c.getColor()));
}
} catch (Exception e) {
System.out.println(e.toString());
}
}
}
public static void main(String[] args) {
try {
JobConf conf = new JobConf(TestMapReduce.class);
conf.setJobName("Iterator Comparable");
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(ColorWritable.class);
conf.setMapperClass(CountMapper.class);
conf.setReducerClass(CountReducer.class);
conf.setInputFormat(TextInputFormat.class);
conf.setOutputFormat(TextOutputFormat.class);
FileInputFormat.setInputPaths(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
JobClient.runJob(conf);
} catch (Exception e) {
System.out.println(e.toString());
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment