Skip to content

Instantly share code, notes, and snippets.

@harit-sunrun
Created August 2, 2012 22:37
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save harit-sunrun/3241462 to your computer and use it in GitHub Desktop.
Save harit-sunrun/3241462 to your computer and use it in GitHub Desktop.
This is basic template you would usually need to write a Map reduce program. Most part would remain same, except Job related inputs and input/output type classes
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
public class MapReduceClass extends Configured implements Tool {
public static class MapClass extends Mapper<LongWritable, Text, Text, LongWritable> {
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
// your map code goes here
}
}
public static class Reduce extends Reducer<Text, LongWritable, Text, Text> {
public void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
// your reduce function goes here
}
public int run(String args[]) throws Exception {
Job job = new Job();
job.setJarByClass(MapReduceClass.class);
job.setMapperClass(MapClass.class);
job.setReducerClass(Reduce.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// if your map output key/value classes are different than your input key/value classes
// see - http://stackoverflow.com/questions/11761135/hadoop-job-fails-while-reducing-java-io-ioexception-type-mismatch-in-value-fro
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
job.setJobName("MapReduceClass");
// uncomment the following if you just want to run Mapper and not Reducer
// job.setNumReduceTasks(0);
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
public static void main(String args[]) throws Exception {
int ret = ToolRunner.run(new MapReduceClass(), args);
System.exit(ret);
}
}
@jeremybeard
Copy link

Just missing a brace between 29 and 30.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment