secret
Created

  • Download Gist
gistfile1.txt
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
Configuration conf = new Configuration();
 
conf.set("hbase.table.name", args[2]);
conf.set("mapreduce.input.keyvaluelinerecordreader.key.value.separator", "\t");
conf.set("key.value.separator.in.input.line", "\t");
conf.set("dfs.replication", 1);
 
// Load hbase-site.xml
HBaseConfiguration.addHbaseResources(conf);
 
Job job = new Job(conf, "HBase Bulk Import");
job.setJarByClass(HBaseMapper.class);
 
job.setMapperClass(HBaseMapper.class);
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class);
 
job.setInputFormatClass(KeyValueTextInputFormat.class);
 
HTable hTable = new HTable(conf, args[2]);
 
// Auto configure partitioner and reducer
HFileOutputFormat.configureIncrementalLoad(job, hTable);
 
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
 
job.waitForCompletion(true);
 
// Load generated HFiles into table
LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
loader.doBulkLoad(new Path(args[1]), hTable);

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.