Created
July 2, 2014 01:25
-
-
Save pppsunil/e78c9b5d9c06dd3fbb16 to your computer and use it in GitHub Desktop.
Sample HDFS Client that demonstrates how to read file from HDFS and how to set replication factor
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.spnotes.hadoop; | |
import org.apache.commons.io.IOUtils; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.FileSystem; | |
import org.apache.hadoop.fs.Path; | |
import org.apache.hadoop.hdfs.DistributedFileSystem; | |
import java.io.Console; | |
import java.io.IOException; | |
import java.io.InputStream; | |
import java.net.URI; | |
import java.util.Scanner; | |
/** | |
* Created by user on 7/1/14. | |
*/ | |
public class HelloHDFS { | |
public static void main(String[] argv) throws Exception{ | |
if(argv.length != 2){ | |
System.out.println("Use HelloHDFS read/write/replicate path"); | |
System.exit(-1); | |
} | |
String command = argv[0]; | |
String filePath = argv[1]; | |
Configuration conf = new Configuration(); | |
DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(URI.create(filePath),conf); | |
HelloHDFS hdfsClient = new HelloHDFS(); | |
if(command.equals("read")){ | |
hdfsClient.read(dfs,filePath); | |
}else if(command.equals("write")){ | |
hdfsClient.write(dfs, filePath); | |
}else if(command.equals("replicate")){ | |
hdfsClient.setReplication(dfs,filePath); | |
} | |
} | |
private void read(DistributedFileSystem fs, String path)throws IOException{ | |
System.out.println("Entering HelloHDFS.read()"); | |
InputStream in = null; | |
try{ | |
in = fs.open(new Path(path) ); | |
IOUtils.copy(in, System.out); | |
}finally{ | |
IOUtils.closeQuietly(in); | |
} | |
System.out.println("Entering HelloHDFS.read()"); | |
} | |
private void write(DistributedFileSystem fs, String filePath)throws IOException{ | |
System.out.println("Entering HelloHDFS.write()"); | |
// fs.open() | |
System.out.println("Entering HelloHDFS.wite()"); | |
} | |
private void setReplication(DistributedFileSystem dfs, String filePath)throws IOException{ | |
System.out.println("Entering HelloHDFS.setReplicationFactor()"); | |
Scanner reader = new Scanner(System.in); | |
System.out.println("Enter replication factor:"); | |
short replicationFactor = reader.nextShort(); | |
dfs.setReplication(new Path(filePath),replicationFactor); | |
System.out.println("Entering HelloHDFS.setReplicationFactor()"); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment