AWS S3Location

S3Location.java

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.common.collect.ImmutableList;
import com.google.common.io.InputSupplier;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.base.Throwables.propagate;
import S3Objects.s3ObjectListing;
import S3Objects.s3ObjectPrefixes;

public class S3Location
{
    private final String bucket;
    private final String key;

    public S3Location(String bucket, String key)
    {
        checkNotNull(bucket, "bucket");
        checkNotNull(key, "key");
        checkArgument(!key.startsWith("/"), "key must not start with a slash");
        checkArgument(!key.contains("//"), "key must not contain double slashes");
        this.bucket = bucket;
        this.key = key;
    }

    public String getBucket()
    {
        return bucket;
    }

    public String getKey()
    {
        return key;
    }

    public boolean isDirectory()
    {
        return key.isEmpty() || key.endsWith("/");
    }

    public String getName()
    {
        String key = removeTrailingSlash(this.key);
        return key.substring(key.lastIndexOf('/') + 1);
    }

    public S3Location getParent()
    {
        String key = removeTrailingSlash(this.key);
        key = key.substring(0, key.length() - getName().length());
        return key.isEmpty() ? null : new S3Location(bucket, key);
    }

    public S3Location appendDirectory(String name)
    {
        checkState(isDirectory(), "cannot append to a non-location directory");
        checkArgument(!name.contains("/"), "directory name contains slashes");
        return new S3Location(bucket, key + name + "/");
    }

    public S3Location appendObject(String name)
    {
        checkState(isDirectory(), "cannot append to a non-location directory");
        checkArgument(!name.contains("/"), "object name contains slashes");
        return new S3Location(bucket, key + "/" + name);
    }

    public URI toUri()
    {
        try {
            return new URI("s3", bucket, "/" + key, null);
        }
        catch (URISyntaxException e) {
            throw propagate(e);
        }
    }

    @Override
    public String toString()
    {
        return toUri().toString();
    }

    public List<S3Location> listDirectories(AmazonS3 client)
    {
        checkNotNull(client, "client");
        Iterable<String> prefixes = s3ObjectPrefixes(client, listObjectsRequest());

        ImmutableList.Builder<S3Location> list = ImmutableList.builder();
        for (String prefix : prefixes) {
            list.add(new S3Location(bucket, forceTrailingSlash(prefix)));
        }
        return list.build();
    }

    public List<S3ObjectSummary> listObjects(AmazonS3 client)
    {
        checkNotNull(client, "client");
        return ImmutableList.copyOf(s3ObjectListing(client, listObjectsRequest()));
    }

    public S3Object getObject(AmazonS3 client)
            throws IOException
    {
        checkNotNull(client, "client");
        checkState(!isDirectory(), "cannot download a directory");
        try {
            return client.getObject(bucket, key);
        }
        catch (AmazonClientException e) {
            throw new IOException(e);
        }
    }

    public InputSupplier<InputStream> getInputSupplier(final AmazonS3 client)
    {
        checkNotNull(client, "client");
        checkState(!isDirectory(), "cannot download a directory");
        return new InputSupplier<InputStream>()
        {
            @Override
            public InputStream getInput()
                    throws IOException
            {
                return getObject(client).getObjectContent();
            }
        };
    }

    public static S3Location createS3Location(String location)
    {
        URI uri = validS3Uri(URI.create(checkNotNull(location, "location")));
        String bucket = uri.getAuthority();
        String path = uri.getPath();
        if (path.startsWith("/")) {
            path = path.substring(1);
        }
        return new S3Location(bucket, path);
    }

    public static S3Location createS3DirectoryLocation(String location)
    {
        checkNotNull(location, "location");
        checkArgument(location.endsWith("/"), "directory locations must end with a slash");
        return createS3Location(location);
    }

    public static S3Location createS3ObjectLocation(String location)
    {
        checkNotNull(location, "location");
        checkArgument(!location.endsWith("/"), "object locations must not end with a slash");
        return createS3Location(location);
    }

    public static S3Location createS3Location(S3ObjectSummary summary)
    {
        checkNotNull(summary, "summary");
        return new S3Location(summary.getBucketName(), summary.getKey());
    }

    private ListObjectsRequest listObjectsRequest()
    {
        checkState(isDirectory(), "location must be a directory");

        return new ListObjectsRequest(bucket, key, null, "/", null);
    }

    private static String removeTrailingSlash(String key)
    {
        return key.endsWith("/") ? key.substring(0, key.length() - 1) : key;
    }

    private static String forceTrailingSlash(String key)
    {
        return key.endsWith("/") ? key : (key + "/");
    }

    private static URI validS3Uri(URI uri)
    {
        checkArgument("s3".equals(uri.getScheme()), "location is not a S3 uri: %s", uri);
        checkArgument(uri.isAbsolute(), "location is not an absolute uri: %s", uri);
        checkArgument(!isNullOrEmpty(uri.getAuthority()), "location does not contain a bucket: %s", uri);
        return uri;
    }
}

S3Objects.java

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.common.base.Function;
import com.google.common.collect.AbstractSequentialIterator;

import java.util.Iterator;

import static com.google.common.collect.Iterators.concat;
import static com.google.common.collect.Iterators.transform;

public final class S3Objects
{
    private S3Objects() {}

    public static Iterable<S3ObjectSummary> s3ObjectListing(final AmazonS3 client, final ListObjectsRequest request)
    {
        return new Iterable<S3ObjectSummary>()
        {
            @Override
            public Iterator<S3ObjectSummary> iterator()
            {
                return concat(transform(listObjects(client, request), objectSummaries()));
            }
        };
    }

    public static Iterable<String> s3ObjectPrefixes(final AmazonS3 client, final ListObjectsRequest request)
    {
        return new Iterable<String>()
        {
            @Override
            public Iterator<String> iterator()
            {
                return concat(transform(listObjects(client, request), commonPrefixes()));
            }
        };
    }

    private static Iterator<ObjectListing> listObjects(final AmazonS3 client, ListObjectsRequest request)
    {
        return new AbstractSequentialIterator<ObjectListing>(client.listObjects(request))
        {
            @Override
            protected ObjectListing computeNext(ObjectListing previous)
            {
                if (!previous.isTruncated()) {
                    return null;
                }
                return client.listNextBatchOfObjects(previous);
            }
        };
    }

    private static Function<ObjectListing, Iterator<S3ObjectSummary>> objectSummaries()
    {
        return new Function<ObjectListing, Iterator<S3ObjectSummary>>()
        {
            @Override
            public Iterator<S3ObjectSummary> apply(ObjectListing input)
            {
                return input.getObjectSummaries().iterator();
            }
        };
    }

    private static Function<ObjectListing, Iterator<String>> commonPrefixes()
    {
        return new Function<ObjectListing, Iterator<String>>()
        {
            @Override
            public Iterator<String> apply(ObjectListing input)
            {
                return input.getCommonPrefixes().iterator();
            }
        };
    }
}