LeoFS_Java_Client_Testing

LeoFS_Java_Client_Testing_Results

bash-4.1$ ant
Buildfile: build.xml

run:
     [java] Jul 18, 2014 10:15:47 AM com.amazonaws.http.HttpClientFactory createHttpClient
     [java] INFO: Configuring Proxy. Proxy Host: localhost Proxy Port: 8080
     [java] Bucket Creation Test [Start]
     [java] Bucket Created Successfully
     [java] -----List Buckets----
     [java] Bucket:test Fri Jul 18 10:15:47 JST 2014
     [java] Bucket Creation Test [End]
     [java] 
     [java] Object Upload Test [Start]
     [java] Successfully created text file
     [java] Uploading a new object to S3 from a file
     [java] 
     [java] Single Part File Uploaded Successfully
     [java] Transfer: Uploading to test/testFile        StateWaiting
     [java]   - Progress: 0Byte         0.0%
     [java] File Uploaded Successfully
     [java] Object Upload Test [End] 
     [java] 
     [java] HEAD Object Test [Start]
     [java] Single Part Metadata => Etag :86e04e4744a20348dea89fd3e5ed988d      ContentLength :33
     [java] Digest(in hex format):: 86e04e4744a20348dea89fd3e5ed988d
     [java] Single Part File MetaData Test passed Successfully
     [java] Multi Part Metadata => Etag :86e04e4744a20348dea89fd3e5ed988d       ContentLength :33
     [java] Digest(in hex format):: 86e04e4744a20348dea89fd3e5ed988d
     [java] Multi Part File MetaData Test passed Successfully
     [java] HEAD Object Test [End]
     [java] 
     [java] GET Object Test [Start]
     [java] GET Object Test [End]
     [java] 
     [java] COPY Object Test [Start]
     [java] File copied successfully
     [java] COPY Object Test [End]
     [java] -----List objects----
     [java] test-key      Size:14
     [java] testFile      Size:33
     [java] testFile.copy         Size:33
     [java] testFile.single       Size:33
     [java] DELETE Object Test [Start]
     [java] -----List objects----
     [java] test-key             Deleted Successfully
     [java] testFile             Deleted Successfully
     [java] testFile.copy                Deleted Successfully
     [java] testFile.single              Deleted Successfully
     [java] ALL File deleted Successfully
     [java] DELETE Object Test [End]
     [java] 
     [java] Bucket ACL Test [Start]
     [java] #####Default ACL#####
     [java] Owner ID : S3Owner [name=05236,id=05236]
     [java] Grantee : null       Permissions : FULL_CONTROL
     [java] Bucket permission is private
     [java] 
     [java] #####:public_read ACL#####
     [java] Owner ID : S3Owner [name=05236,id=05236]
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : READ_ACP
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : READ
     [java] Bucket permission is public_read
     [java] 
     [java] #####:public_read_write ACL#####
     [java] Owner ID : S3Owner [name=05236,id=05236]
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : WRITE
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : WRITE_ACP
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : READ_ACP
     [java] Grantee : GroupGrantee [http://acs.amazonaws.com/groups/global/AllUsers]     Permissions : READ
     [java] Bucket permission is public_read_write
     [java] 
     [java] #####:private ACL#####
     [java] Owner ID : S3Owner [name=05236,id=05236]
     [java] Grantee : null       Permissions : FULL_CONTROL
     [java] Bucket permission is private
     [java] Bucket ACL Test [End]
     [java] 
     [java] DELETE Bucket Test [Start]
     [java] Bucket deleted Successfully
     [java] DELETE Bucket Test [End]

BUILD SUCCESSFUL
Total time: 1 second

LeoFS_Java_Client_Testing_Script

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.List;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.Grant;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
public class LeoFSSample {
    public static void main(String[] args) throws IOException {
        /* ---------------------------------------------------------
         * You need to set 'Proxy host', 'Proxy port' and 'Protocol'
         * --------------------------------------------------------- */
        ClientConfiguration config = new ClientConfiguration();
        config.setProxyHost("localhost"); // LeoFS Gateway's Host
        config.setProxyPort(8080);        // LeoFS Gateway's Port
        config.withProtocol(Protocol.HTTP);
        final String accessKeyId = "05236";
        final String secretAccessKey = "802562235";
        AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
        AmazonS3 s3 = new AmazonS3Client(credentials, config);
        final String bucketName = "test";
        final String key = "test-key";
        final String fileName = "testFile";
        try {
            // Create a bucket
            System.out.println("Bucket Creation Test [Start]");
            s3.createBucket(bucketName);
            if ( s3.doesBucketExist(bucketName) ) {
                System.out.println("Bucket Created Successfully");
            } else {
                throw new IOException("Bucket Creation Failed");
            }

            // Retrieve list of buckets
            System.out.println("-----List Buckets----");
            for ( Bucket bucket : s3.listBuckets() ) {
                System.out.println("Bucket:" + bucket.getName() + "\t" + bucket.getCreationDate());
            }
            System.out.println("Bucket Creation Test [End]\n");

            System.out.println("Object Upload Test [Start]");

            // PUT an object into the LeoFS 
            s3.putObject(new PutObjectRequest(bucketName, key, createFile()));
            if ( !doesFileExist(s3, bucketName, key) ) {
                throw new IOException("Text File could not Created Successfully");
            } else {
                System.out.println("Successfully created text file");
            }

            // File Upload to LeoFS using single part upload method
            String filePath= "../temp_data/" + fileName;
            System.out.println("Uploading a new object to S3 from a file\n");
            File file = new File(filePath);
            s3.putObject(new PutObjectRequest(bucketName, file.getName()+".single", file));
            if ( !doesFileExist(s3, bucketName, fileName+".single") ) {
                throw new IOException("Single Part File could not Uploaded Successfully");
            } else {
                System.out.println("Single Part File Uploaded Successfully");
            }

            /* Multipart upload allows you to upload a single object as a set of parts.Each part is a contiguous
            portion of the object's data. You can upload these object parts independently and in any order. 
            If transmission of any part fails, you can retransmit that part without affecting other parts. 
            After all parts of your object are uploaded, LeoFS assembles these parts and creates the object. 
            In general, when your object size reaches 100 MB, you should consider using multipart uploads instead
            of uploading the object in a single operation. Advantages : Improved throughput, Quick recovery from 
            any network issues, Pause and resume object uploads Begin an upload before you know the final object size. */

            // File Upload to LeoFS using multipart upload method
            TransferManager tx = new TransferManager(s3);
            Upload upload = tx.upload(bucketName, file.getName(), file); 
            System.out.println("Transfer: " + upload.getDescription() + "\t" + "State" + upload.getState());

            // You can poll your transfer's status to check its progress
            while ( upload.isDone() == false ) {
                System.out.println(" - Progress: " + upload.getProgress().getBytesTransferred() 
                                      + "Byte \t" + upload.getProgress().getPercentTransferred() + "%" );
                Thread.sleep(100);
            } 
            upload.waitForCompletion();
            tx.shutdownNow(Boolean.FALSE.booleanValue());
            if ( !doesFileExist(s3, bucketName, fileName) ) {
                throw new IOException("Multi-part File could not Uploaded Successfully");
            } else {
                System.out.println("File Uploaded Successfully");
            }
            System.out.println("Object Upload Test [End] \n");

            /* Files in Amazon S3 & LeoFS are called "objects" and are stored in buckets. A specific object is
            referred to by its key (i.e., name) and holds data. Here, we create a new object with
            the key name, HEAD request is Metadata of that object. e.g. Size, etag, Content_type etc.
            For more information http://boto.readthedocs.org/en/latest/s3_tut.html#storing-data */ 

            // Head Object
            System.out.println("HEAD Object Test [Start]");
            ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, fileName+".single");
            System.out.println("Single Part Metadata => Etag :" + objectMetadata.getETag() + " \tContentLength :" + objectMetadata.getContentLength() );
            if ( file.length() == objectMetadata.getContentLength() && objectMetadata.getETag() == MD5(filePath) ) {
                throw new IOException("Sigle Part File Metadata could not match");
            } else {
                System.out.println("Single Part File MetaData Test passed Successfully");
            }

            objectMetadata = s3.getObjectMetadata(bucketName, fileName);
            System.out.println("Multi Part Metadata => Etag :" + objectMetadata.getETag() + " \tContentLength :" + objectMetadata.getContentLength());
            if ( file.length() == objectMetadata.getContentLength() && objectMetadata.getETag() == MD5(filePath) ) {
                throw new IOException("Multi Part File Metadata could not match");
            } else {
                System.out.println("Multi Part File MetaData Test passed Successfully");
            }

            System.out.println("HEAD Object Test [End]\n");
            
            // File Download from LeoFS
            System.out.println("GET Object Test [Start]");
            S3Object object = s3.getObject(new GetObjectRequest(bucketName, fileName));
            dumpInputStream(object.getObjectContent(),fileName+".copy");
            File newfile = new File(fileName+".copy");
            if ( file.length() != newfile.length() ) {
                throw new IOException("Downloaded File content-length is not equal");
            }
            System.out.println("GET Object Test [End]\n");

            // File copy bucket internally
            
            System.out.println("COPY Object Test [Start]");
            s3.copyObject( bucketName, file.getName(), bucketName, fileName+".copy");
            if (!doesFileExist(s3, bucketName, fileName+".copy") ) {
                throw new IOException("File could not copy Successfully");
            }
            System.out.println("File copied successfully");
            System.out.println("COPY Object Test [End]");

            // Retrieve list of objects from the LeoFS
            ObjectListing objectListing =
                s3.listObjects(new ListObjectsRequest().withBucketName(bucketName));
            System.out.println("-----List objects----");
            for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {
                System.out.println(objectSummary.getKey() + " \t Size:" + objectSummary.getSize());
            }

            // DELETE an object from the LeoFS
            System.out.println("DELETE Object Test [Start]");
            System.out.println("-----List objects----");
            for ( S3ObjectSummary objectSummary : objectListing.getObjectSummaries() ) {
                s3.deleteObject(bucketName, objectSummary.getKey());
                if ( doesFileExist(s3,bucketName,objectSummary.getKey()) ) {
                    throw new IOException("Object Not Deleted");
                } else {
                    System.out.println(objectSummary.getKey() + " \t\t Deleted Successfully");
                }
            }
            
            System.out.println("ALL File deleted Successfully");
            System.out.println("DELETE Object Test [End]\n");

            // GET-PUT ACL
            System.out.println("Bucket ACL Test [Start]");
            System.out.println("#####Default ACL#####");
            AccessControlList acp = s3.getBucketAcl(bucketName);
            List<String> list = new ArrayList<String>();
            System.out.println("Owner ID : " + acp.getOwner());
            for ( Grant grant : acp.getGrants() ) {
                System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
            list.add(grant.getPermission().toString());
            }
            if ( list.contains("FULL_CONTROL") ) {
                System.out.println("Bucket permission is private");
            } else {
                throw new IOException("Bucket permission is not private");
            }

            System.out.println("\n#####:public_read ACL#####");
            s3.setBucketAcl(bucketName, CannedAccessControlList.PublicRead);
            acp = s3.getBucketAcl(bucketName);
            list.clear();
            System.out.println("Owner ID : " + acp.getOwner());
            for ( Grant grant : acp.getGrants() ) {
                System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
                list.add(grant.getPermission().toString());
            }
            if ( list.contains("READ") && list.contains("READ_ACP") ) {
                System.out.println("Bucket permission is public_read");
            } else {
                throw new IOException("Bucket permission is not public_read");
            }

            System.out.println("\n#####:public_read_write ACL#####"); 
            s3.setBucketAcl(bucketName, CannedAccessControlList.PublicReadWrite);
            acp = s3.getBucketAcl(bucketName);
            list.clear();
            System.out.println("Owner ID : " + acp.getOwner());
            for ( Grant grant : acp.getGrants() ) {
                System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
                list.add(grant.getPermission().toString());
            } 
            if ( list.contains("READ") && list.contains("WRITE") && list.contains("READ_ACP") && list.contains("WRITE_ACP") ) {
                System.out.println("Bucket permission is public_read_write");
            } else {
                throw new IOException("Bucket permission is not public_read_write");
            }

            System.out.println("\n#####:private ACL#####"); 
            s3.setBucketAcl(bucketName, CannedAccessControlList.Private);
            acp = s3.getBucketAcl(bucketName);
            list.clear();
            System.out.println("Owner ID : " + acp.getOwner());
            for ( Grant grant : acp.getGrants() ) {
                System.out.println("Grantee : " + grant.getGrantee() + " \t Permissions : " + grant.getPermission());
                list.add(grant.getPermission().toString());
            } 
            if ( list.contains("FULL_CONTROL") ) {
                System.out.println("Bucket permission is private");
            } else {
                throw new IOException("Bucket permission is not private");
            }
            System.out.println("Bucket ACL Test [End]\n");

            // DELETE a bucket from the LeoFS
            System.out.println("DELETE Bucket Test [Start]");
            s3.deleteBucket(bucketName);
            System.out.println("Bucket deleted Successfully");
            System.out.println("DELETE Bucket Test [End]");
        } catch ( AmazonServiceException ase ) {
            System.out.println(ase.getMessage());
            System.out.println(ase.getStatusCode());
        } catch ( AmazonClientException ace ) {
            System.out.println(ace.getMessage());
        } catch ( InterruptedException ie ) {
            System.out.println(ie.getMessage());
        }
        
    }

    /**
     * Creates a temporary file with text data to demonstrate uploading a file
     * to LeoFS
     *
     * @return A newly created temporary file with text data.
     *
     * @throws IOException
     */

    private static File createFile() throws IOException {
        File file = File.createTempFile("leofs_test", ".txt");
        file.deleteOnExit();
        Writer writer = new OutputStreamWriter(new FileOutputStream(file));
        writer.write("Hello, world!\n");
        writer.close();
        return file;
    }

    /**
     * Displays the contents of the specified input stream as text.
     *
     * @param input
     * The input stream to display as text.
     *
     * @throws IOException
     */

    private static void dumpInputStream(InputStream input,String fileName) throws IOException {
        BufferedReader reader = new BufferedReader(new InputStreamReader(input));
        File file=new File(fileName);
        OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(file));
        int read = -1;
        while ( ( read = (byte) reader.read() ) != -1 ) {
            writer.write(read);
        }
        writer.flush();
        writer.close();
        reader.close();
    }
    public static boolean doesFileExist( AmazonS3 s3, String bucketName, String key ) throws AmazonClientException, AmazonServiceException {
        boolean isValidFile = true;
        try {
            ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, key);
        } catch ( AmazonS3Exception s3e ) {
            if ( s3e.getStatusCode() == 404 ) {
                // i.e. 404: NoSuchKey - The specified key does not exist
                isValidFile = false;
            }
            else {
                throw s3e; // rethrow all S3 exceptions other than 404 
            }
        }
        catch ( Exception exception ) {
            exception.printStackTrace();
            isValidFile = false;
        }
        return isValidFile;
    }

    public static String MD5( String filePath )
    {
        StringBuffer sb = new StringBuffer();
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            FileInputStream fis = new FileInputStream(filePath); 
            byte[] dataBytes = new byte[1024];

            int nread = 0; 
            while ( (nread = fis.read(dataBytes)) != -1 ) {
                md.update(dataBytes, 0, nread);
            };
            byte[] mdbytes = md.digest();

            //convert the byte to hex format method 1
            for (int i = 0; i < mdbytes.length; i++) {
                sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1));
            }

            System.out.println("Digest(in hex format):: " + sb.toString());
 
        } catch ( java.security.NoSuchAlgorithmException e ) {
            System.out.println(e.getMessage());
        } catch ( Exception e ) {
            System.out.println(e.getMessage());
        }
        return sb.toString();
    }
}