Created
June 23, 2020 20:09
-
-
Save chanakaDe/a90836833acbb8a2d9113f8b8d3e1b61 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package stubs; | |
import java.io.IOException; | |
import org.apache.hadoop.conf.Configured; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.Path; | |
import org.apache.hadoop.util.Tool; | |
import org.apache.hadoop.util.ToolRunner; | |
import org.apache.hadoop.io.IntWritable; | |
import org.apache.hadoop.io.Text; | |
import org.apache.hadoop.mapreduce.Job; | |
import org.apache.hadoop.mapreduce.Mapper; | |
import org.apache.hadoop.mapreduce.Reducer; | |
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; | |
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; | |
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; | |
public class CitationHistogram extends Configured implements Tool { | |
public static class MapClass extends | |
Mapper<Text, Text, IntWritable, IntWritable> { | |
private final static IntWritable uno = new IntWritable(1); | |
private IntWritable citationCount = new IntWritable(); | |
@Override | |
public void map(Text key, Text value, Context context) | |
throws IOException, InterruptedException { | |
if (value.getLength() > 0) { | |
citationCount.set(Integer.parseInt(value.toString())); | |
context.write(citationCount, uno); | |
} | |
} | |
} | |
public static class Reduce extends | |
Reducer<IntWritable, IntWritable, IntWritable, IntWritable> { | |
@Override | |
public void reduce(IntWritable key, Iterable<IntWritable> values, | |
Context context) throws IOException, InterruptedException { | |
int count = 0; | |
for (IntWritable value : values) { | |
count += value.get(); | |
} | |
context.write(key, new IntWritable(count)); | |
} | |
} | |
public int run(String[] args) throws Exception { | |
Configuration conf = new Configuration(); | |
Job job = new Job(conf); | |
job.setJarByClass(CitationHistogram.class); | |
Path in = new Path(args[0]); | |
Path out = new Path(args[1]); | |
FileInputFormat.setInputPaths(job, in); | |
FileOutputFormat.setOutputPath(job, out); | |
job.setJobName("CitationHistogram"); | |
job.setMapperClass(MapClass.class); | |
job.setReducerClass(Reduce.class); | |
job.setOutputKeyClass(IntWritable.class); | |
job.setOutputValueClass(IntWritable.class); | |
job.setInputFormatClass(KeyValueTextInputFormat.class); | |
boolean success = job.waitForCompletion(true); | |
return success ? 0 : 1; | |
} | |
public static void main(String[] args) throws Exception { | |
int res = ToolRunner.run(new Configuration(), new CitationHistogram(), | |
args); | |
System.exit(res); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment