Skip to content

Instantly share code, notes, and snippets.

@enif-lee
Created October 2, 2019 07:19
Show Gist options
  • Save enif-lee/d3cc5bd16efd70838cdef3180e438817 to your computer and use it in GitHub Desktop.
Save enif-lee/d3cc5bd16efd70838cdef3180e438817 to your computer and use it in GitHub Desktop.
Upload multipart to AWS S3 Object in .NET Core 3.0
public static class AwsS3Helper
{
public static async Task UploadObjectStreamAsync(
this IAmazonS3 s3Client,
Stream stream,
UploadStreamToS3Request request)
{
async IAsyncEnumerable<UploadPartResponse> UploadAsync(string uploadId)
{
await foreach (var (partStream, index, isLast) in SplitPartsFromStream(stream, request.ParSize))
{
var partRequest = new UploadPartRequest
{
BucketName = request.Bucket,
Key = request.Key,
UploadId = uploadId,
InputStream = partStream,
PartNumber = index,
IsLastPart = isLast
};
yield return await s3Client.UploadPartAsync(partRequest);
await partStream.DisposeAsync();
}
}
var initRequest = new InitiateMultipartUploadRequest
{
BucketName = request.Bucket,
Key = request.Key
};
if (!string.IsNullOrEmpty(request.DownloadFileName))
initRequest.Headers.ContentDisposition = $"attachment; filename={request.DownloadFileName}";
if (!string.IsNullOrEmpty(request.ContentType))
initRequest.Headers.ContentType = request.ContentType;
var initResponse = await s3Client.InitiateMultipartUploadAsync(initRequest);
if (request.ParSize < 1024 * 1024 * 5)
throw new ArgumentException($"{nameof(request.ParSize)} must be greater than 5MB");
var uploadAsync = await UploadAsync(initResponse.UploadId).ToListAsync();
var uploadRequest = new CompleteMultipartUploadRequest
{
BucketName = request.Bucket,
Key = request.Key,
UploadId = initResponse.UploadId
};
uploadRequest.AddPartETags(uploadAsync);
await s3Client.CompleteMultipartUploadAsync(uploadRequest);
}
private static async IAsyncEnumerable<(MemoryStream stream, int index, bool isLast)> SplitPartsFromStream(
Stream inputStream,
int partSize)
{
inputStream.Position = 0;
var index = 1;
for (var remainingBytes = inputStream.Length; remainingBytes > 0; remainingBytes -= partSize)
{
var isLast = remainingBytes < partSize;
var buffer = isLast
? new byte[remainingBytes]
: new byte[partSize];
await inputStream.ReadAsync(buffer, 0, buffer.Length);
yield return (new MemoryStream(buffer), index, isLast);
index++;
}
}
private static async Task<IList<T>> ToListAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{
var list = new List<T>();
await foreach (var item in asyncEnumerable) list.Add(item);
return list;
}
}
public class UploadStreamToS3Request
{
public string Bucket { get; set; }
public string Key { get; set; }
public int ParSize { get; set; } = 1024 * 1024 * 5;
public string DownloadFileName { get; set; }
public string ContentType { get; set; }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment