Created
October 2, 2019 07:19
-
-
Save enif-lee/d3cc5bd16efd70838cdef3180e438817 to your computer and use it in GitHub Desktop.
Upload multipart to AWS S3 Object in .NET Core 3.0
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static class AwsS3Helper | |
{ | |
public static async Task UploadObjectStreamAsync( | |
this IAmazonS3 s3Client, | |
Stream stream, | |
UploadStreamToS3Request request) | |
{ | |
async IAsyncEnumerable<UploadPartResponse> UploadAsync(string uploadId) | |
{ | |
await foreach (var (partStream, index, isLast) in SplitPartsFromStream(stream, request.ParSize)) | |
{ | |
var partRequest = new UploadPartRequest | |
{ | |
BucketName = request.Bucket, | |
Key = request.Key, | |
UploadId = uploadId, | |
InputStream = partStream, | |
PartNumber = index, | |
IsLastPart = isLast | |
}; | |
yield return await s3Client.UploadPartAsync(partRequest); | |
await partStream.DisposeAsync(); | |
} | |
} | |
var initRequest = new InitiateMultipartUploadRequest | |
{ | |
BucketName = request.Bucket, | |
Key = request.Key | |
}; | |
if (!string.IsNullOrEmpty(request.DownloadFileName)) | |
initRequest.Headers.ContentDisposition = $"attachment; filename={request.DownloadFileName}"; | |
if (!string.IsNullOrEmpty(request.ContentType)) | |
initRequest.Headers.ContentType = request.ContentType; | |
var initResponse = await s3Client.InitiateMultipartUploadAsync(initRequest); | |
if (request.ParSize < 1024 * 1024 * 5) | |
throw new ArgumentException($"{nameof(request.ParSize)} must be greater than 5MB"); | |
var uploadAsync = await UploadAsync(initResponse.UploadId).ToListAsync(); | |
var uploadRequest = new CompleteMultipartUploadRequest | |
{ | |
BucketName = request.Bucket, | |
Key = request.Key, | |
UploadId = initResponse.UploadId | |
}; | |
uploadRequest.AddPartETags(uploadAsync); | |
await s3Client.CompleteMultipartUploadAsync(uploadRequest); | |
} | |
private static async IAsyncEnumerable<(MemoryStream stream, int index, bool isLast)> SplitPartsFromStream( | |
Stream inputStream, | |
int partSize) | |
{ | |
inputStream.Position = 0; | |
var index = 1; | |
for (var remainingBytes = inputStream.Length; remainingBytes > 0; remainingBytes -= partSize) | |
{ | |
var isLast = remainingBytes < partSize; | |
var buffer = isLast | |
? new byte[remainingBytes] | |
: new byte[partSize]; | |
await inputStream.ReadAsync(buffer, 0, buffer.Length); | |
yield return (new MemoryStream(buffer), index, isLast); | |
index++; | |
} | |
} | |
private static async Task<IList<T>> ToListAsync<T>(this IAsyncEnumerable<T> asyncEnumerable) | |
{ | |
var list = new List<T>(); | |
await foreach (var item in asyncEnumerable) list.Add(item); | |
return list; | |
} | |
} | |
public class UploadStreamToS3Request | |
{ | |
public string Bucket { get; set; } | |
public string Key { get; set; } | |
public int ParSize { get; set; } = 1024 * 1024 * 5; | |
public string DownloadFileName { get; set; } | |
public string ContentType { get; set; } | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment