Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
LeoFS_Java_Client_Testing
bash-4.1$ ant
Buildfile: build.xml
run:
[java] Jul 18, 2014 10:15:47 AM com.amazonaws.http.HttpClientFactory createHttpClient
[java] INFO: Configuring Proxy. Proxy Host: localhost Proxy Port: 8080
[java] Bucket Creation Test [Start]
[java] Bucket Created Successfully
[java] -----List Buckets----
[java] Bucket:test Fri Jul 18 10:15:47 JST 2014
[java] Bucket Creation Test [End]
[java]
[java] Object Upload Test [Start]
[java] Successfully created text file
[java] Uploading a new object to S3 from a file
[java]
[java] Single Part File Uploaded Successfully
[java] Transfer: Uploading to test/testFile StateWaiting
[java] - Progress: 0Byte 0.0%
[java] File Uploaded Successfully
[java] Object Upload Test [End]
[java]
[java] HEAD Object Test [Start]
[java] Single Part Metadata => Etag :86e04e4744a20348dea89fd3e5ed988d ContentLength :33
[java] Digest(in hex format):: 86e04e4744a20348dea89fd3e5ed988d
[java] Single Part File MetaData Test passed Successfully
[java] Multi Part Metadata => Etag :86e04e4744a20348dea89fd3e5ed988d ContentLength :33
[java] Digest(in hex format):: 86e04e4744a20348dea89fd3e5ed988d
[java] Multi Part File MetaData Test passed Successfully
[java] HEAD Object Test [End]
[java]
[java] GET Object Test [Start]
[java] GET Object Test [End]
[java]
[java] COPY Object Test [Start]
[java] File copied successfully
[java] COPY Object Test [End]
[java] -----List objects----
[java] test-key Size:14
[java] testFile Size:33
[java] testFile.copy Size:33
[java] testFile.single Size:33
[java] DELETE Object Test [Start]
[java] -----List objects----
[java] test-key Deleted Successfully
[java] testFile Deleted Successfully
[java] testFile.copy Deleted Successfully
[java] testFile.single Deleted Successfully
[java] ALL File deleted Successfully
[java] DELETE Object Test [End]
[java]
[java] Bucket ACL Test [Start]
[java] #####Default ACL#####
[java] Owner ID : S3Owner [name=05236,id=05236]
[java] Grantee : null Permissions : FULL_CONTROL
[java] Bucket permission is private
[java]
[java] #####:public_read ACL#####
[java] Owner ID : S3Owner [name=05236,id=05236]
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : READ_ACP
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : READ
[java] Bucket permission is public_read
[java]
[java] #####:public_read_write ACL#####
[java] Owner ID : S3Owner [name=05236,id=05236]
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : WRITE
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : WRITE_ACP
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : READ_ACP
[java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers] Permissions : READ
[java] Bucket permission is public_read_write
[java]
[java] #####:private ACL#####
[java] Owner ID : S3Owner [name=05236,id=05236]
[java] Grantee : null Permissions : FULL_CONTROL
[java] Bucket permission is private
[java] Bucket ACL Test [End]
[java]
[java] DELETE Bucket Test [Start]
[java] Bucket deleted Successfully
[java] DELETE Bucket Test [End]
BUILD SUCCESSFUL
Total time: 1 second
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.Grant;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
public class LeoFSSample {
public static void main(String[] args) throws IOException {
/* ---------------------------------------------------------
* You need to set 'Proxy host', 'Proxy port' and 'Protocol'
* --------------------------------------------------------- */
ClientConfiguration config = new ClientConfiguration();
config.setProxyHost("localhost"); // LeoFS Gateway's Host
config.setProxyPort(8080); // LeoFS Gateway's Port
config.withProtocol(Protocol.HTTP);
final String accessKeyId = "05236";
final String secretAccessKey = "802562235";
AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
AmazonS3 s3 = new AmazonS3Client(credentials, config);
final String bucketName = "test";
final String key = "test-key";
final String fileName = "testFile";
try {
// Create a bucket
System.out.println("Bucket Creation Test [Start]");
s3.createBucket(bucketName);
if ( s3.doesBucketExist(bucketName) ) {
System.out.println("Bucket Created Successfully");
} else {
throw new IOException("Bucket Creation Failed");
}
// Retrieve list of buckets
System.out.println("-----List Buckets----");
for ( Bucket bucket : s3.listBuckets() ) {
System.out.println("Bucket:" + bucket.getName() + "\t" + bucket.getCreationDate());
}
System.out.println("Bucket Creation Test [End]\n");
System.out.println("Object Upload Test [Start]");
// PUT an object into the LeoFS
s3.putObject(new PutObjectRequest(bucketName, key, createFile()));
if ( !doesFileExist(s3, bucketName, key) ) {
throw new IOException("Text File could not Created Successfully");
} else {
System.out.println("Successfully created text file");
}
// File Upload to LeoFS using single part upload method
String filePath= "../temp_data/" + fileName;
System.out.println("Uploading a new object to S3 from a file\n");
File file = new File(filePath);
s3.putObject(new PutObjectRequest(bucketName, file.getName()+".single", file));
if ( !doesFileExist(s3, bucketName, fileName+".single") ) {
throw new IOException("Single Part File could not Uploaded Successfully");
} else {
System.out.println("Single Part File Uploaded Successfully");
}
/* Multipart upload allows you to upload a single object as a set of parts.Each part is a contiguous
portion of the object's data. You can upload these object parts independently and in any order.
If transmission of any part fails, you can retransmit that part without affecting other parts.
After all parts of your object are uploaded, LeoFS assembles these parts and creates the object.
In general, when your object size reaches 100 MB, you should consider using multipart uploads instead
of uploading the object in a single operation. Advantages : Improved throughput, Quick recovery from
any network issues, Pause and resume object uploads Begin an upload before you know the final object size. */
// File Upload to LeoFS using multipart upload method
TransferManager tx = new TransferManager(s3);
Upload upload = tx.upload(bucketName, file.getName(), file);
System.out.println("Transfer: " + upload.getDescription() + "\t" + "State" + upload.getState());
// You can poll your transfer's status to check its progress
while ( upload.isDone() == false ) {
System.out.println(" - Progress: " + upload.getProgress().getBytesTransferred()
+ "Byte \t" + upload.getProgress().getPercentTransferred() + "%" );
Thread.sleep(100);
}
upload.waitForCompletion();
tx.shutdownNow(Boolean.FALSE.booleanValue());
if ( !doesFileExist(s3, bucketName, fileName) ) {
throw new IOException("Multi-part File could not Uploaded Successfully");
} else {
System.out.println("File Uploaded Successfully");
}
System.out.println("Object Upload Test [End] \n");
/* Files in Amazon S3 & LeoFS are called "objects" and are stored in buckets. A specific object is
referred to by its key (i.e., name) and holds data. Here, we create a new object with
the key name, HEAD request is Metadata of that object. e.g. Size, etag, Content_type etc.
For more information http://boto.readthedocs.org/en/latest/s3_tut.html#storing-data */
// Head Object
System.out.println("HEAD Object Test [Start]");
ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, fileName+".single");
System.out.println("Single Part Metadata => Etag :" + objectMetadata.getETag() + " \tContentLength :" + objectMetadata.getContentLength() );
if ( file.length() == objectMetadata.getContentLength() && objectMetadata.getETag() == MD5(filePath) ) {
throw new IOException("Sigle Part File Metadata could not match");
} else {
System.out.println("Single Part File MetaData Test passed Successfully");
}
objectMetadata = s3.getObjectMetadata(bucketName, fileName);
System.out.println("Multi Part Metadata => Etag :" + objectMetadata.getETag() + " \tContentLength :" + objectMetadata.getContentLength());
if ( file.length() == objectMetadata.getContentLength() && objectMetadata.getETag() == MD5(filePath) ) {
throw new IOException("Multi Part File Metadata could not match");
} else {
System.out.println("Multi Part File MetaData Test passed Successfully");
}
System.out.println("HEAD Object Test [End]\n");
// File Download from LeoFS
System.out.println("GET Object Test [Start]");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, fileName));
dumpInputStream(object.getObjectContent(),fileName+".copy");
File newfile = new File(fileName+".copy");
if ( file.length() != newfile.length() ) {
throw new IOException("Downloaded File content-length is not equal");
}
System.out.println("GET Object Test [End]\n");
// File copy bucket internally
System.out.println("COPY Object Test [Start]");
s3.copyObject( bucketName, file.getName(), bucketName, fileName+".copy");
if (!doesFileExist(s3, bucketName, fileName+".copy") ) {
throw new IOException("File could not copy Successfully");
}
System.out.println("File copied successfully");
System.out.println("COPY Object Test [End]");
// Retrieve list of objects from the LeoFS
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName));
System.out.println("-----List objects----");
for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {
System.out.println(objectSummary.getKey() + " \t Size:" + objectSummary.getSize());
}
// DELETE an object from the LeoFS
System.out.println("DELETE Object Test [Start]");
System.out.println("-----List objects----");
for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {
s3.deleteObject(bucketName, objectSummary.getKey());
if ( doesFileExist(s3,bucketName,objectSummary.getKey()) ) {
throw new IOException("Object Not Deleted");
} else {
System.out.println(objectSummary.getKey() + " \t\t Deleted Successfully");
}
}
System.out.println("ALL File deleted Successfully");
System.out.println("DELETE Object Test [End]\n");
// GET-PUT ACL
System.out.println("Bucket ACL Test [Start]");
System.out.println("#####Default ACL#####");
AccessControlList acp = s3.getBucketAcl(bucketName);
List<String> list = new ArrayList<String>();
System.out.println("Owner ID : " + acp.getOwner());
for ( Grant grant : acp.getGrants() ) {
System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
list.add(grant.getPermission().toString());
}
if ( list.contains("FULL_CONTROL") ) {
System.out.println("Bucket permission is private");
} else {
throw new IOException("Bucket permission is not private");
}
System.out.println("\n#####:public_read ACL#####");
s3.setBucketAcl(bucketName, CannedAccessControlList.PublicRead);
acp = s3.getBucketAcl(bucketName);
list.clear();
System.out.println("Owner ID : " + acp.getOwner());
for ( Grant grant : acp.getGrants() ) {
System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
list.add(grant.getPermission().toString());
}
if ( list.contains("READ") && list.contains("READ_ACP") ) {
System.out.println("Bucket permission is public_read");
} else {
throw new IOException("Bucket permission is not public_read");
}
System.out.println("\n#####:public_read_write ACL#####");
s3.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite);
acp = s3.getBucketAcl(bucketName);
list.clear();
System.out.println("Owner ID : " + acp.getOwner());
for ( Grant grant : acp.getGrants() ) {
System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
list.add(grant.getPermission().toString());
}
if ( list.contains("READ") && list.contains("WRITE") && list.contains("READ_ACP") && list.contains("WRITE_ACP") ) {
System.out.println("Bucket permission is public_read_write");
} else {
throw new IOException("Bucket permission is not public_read_write");
}
System.out.println("\n#####:private ACL#####");
s3.setBucketAcl(bucketName, CannedAccessControlList.Private);
acp = s3.getBucketAcl(bucketName);
list.clear();
System.out.println("Owner ID : " + acp.getOwner());
for ( Grant grant : acp.getGrants() ) {
System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
list.add(grant.getPermission().toString());
}
if ( list.contains("FULL_CONTROL") ) {
System.out.println("Bucket permission is private");
} else {
throw new IOException("Bucket permission is not private");
}
System.out.println("Bucket ACL Test [End]\n");
// DELETE a bucket from the LeoFS
System.out.println("DELETE Bucket Test [Start]");
s3.deleteBucket(bucketName);
System.out.println("Bucket deleted Successfully");
System.out.println("DELETE Bucket Test [End]");
} catch ( AmazonServiceException ase ) {
System.out.println(ase.getMessage());
System.out.println(ase.getStatusCode());
} catch ( AmazonClientException ace ) {
System.out.println(ace.getMessage());
} catch ( InterruptedException ie ) {
System.out.println(ie.getMessage());
}
}
/**
* Creates a temporary file with text data to demonstrate uploading a file
* to LeoFS
*
* @return A newly created temporary file with text data.
*
* @throws IOException
*/
private static File createFile() throws IOException {
File file = File.createTempFile("leofs_test", ".txt");
file.deleteOnExit();
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write("Hello, world!\n");
writer.close();
return file;
}
/**
* Displays the contents of the specified input stream as text.
*
* @param input
* The input stream to display as text.
*
* @throws IOException
*/
private static void dumpInputStream(InputStream input,String fileName) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
File file=new File(fileName);
OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(file));
int read = -1;
while ( ( read = (byte) reader.read() ) != -1 ) {
writer.write(read);
}
writer.flush();
writer.close();
reader.close();
}
public static boolean doesFileExist( AmazonS3 s3, String bucketName, String key ) throws AmazonClientException, AmazonServiceException {
boolean isValidFile = true;
try {
ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, key);
} catch ( AmazonS3Exception s3e ) {
if ( s3e.getStatusCode() == 404 ) {
// i.e. 404: NoSuchKey - The specified key does not exist
isValidFile = false;
}
else {
throw s3e; // rethrow all S3 exceptions other than 404
}
}
catch ( Exception exception ) {
exception.printStackTrace();
isValidFile = false;
}
return isValidFile;
}
public static String MD5( String filePath )
{
StringBuffer sb = new StringBuffer();
try {
MessageDigest md = MessageDigest.getInstance("MD5");
FileInputStream fis = new FileInputStream(filePath);
byte[] dataBytes = new byte[1024];
int nread = 0;
while ( (nread = fis.read(dataBytes)) != -1 ) {
md.update(dataBytes, 0, nread);
};
byte[] mdbytes = md.digest();
//convert the byte to hex format method 1
for (int i = 0; i < mdbytes.length; i++) {
sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
}
System.out.println("Digest(in hex format):: " + sb.toString());
} catch ( java.security.NoSuchAlgorithmException e ) {
System.out.println(e.getMessage());
} catch ( Exception e ) {
System.out.println(e.getMessage());
}
return sb.toString();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment