Skip to content

Instantly share code, notes, and snippets.

@szydan
Created October 11, 2012 10:41
Show Gist options
  • Save szydan/3871561 to your computer and use it in GitHub Desktop.
Save szydan/3871561 to your computer and use it in GitHub Desktop.
how to copy file from hdfs to local disk without hadoop installation
/*
* Copy file from hdfs to local disk without hadoop installation
*
* params are something like
* hdfs://node01.sindice.net:8020 /user/bob/file.zip file.zip
*
* Credits for Josef Niedermeier
*
*/
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSdownloader extends Configured{
public static void main(String[] args) throws Exception {
if (args.length != 3) {
System.out.println("use: HDFSdownloader hdfs src dst");
System.exit(1);
}
HDFSdownloader dw = new HDFSdownloader();
dw.copy2local(args[0], args[1], args[2]);
}
private void copy2local(String hdfs, String src, String dst) throws IOException{
Configuration conf = new Configuration();
conf.set("fs.default.name", hdfs);
FileSystem.get(conf).copyToLocalFile(new Path(src), new Path(dst));
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment