Skip to content

Instantly share code, notes, and snippets.

@Arp-G
Created June 18, 2022 08:38
Show Gist options
  • Save Arp-G/e808d47f80e49458548bd7b37ebdeeb7 to your computer and use it in GitHub Desktop.
Save Arp-G/e808d47f80e49458548bd7b37ebdeeb7 to your computer and use it in GitHub Desktop.
Multipart upload to S3
import fs from 'fs';
import { Buffer } from 'node:buffer';
import {
S3Client,
CreateMultipartUploadCommand,
UploadPartCommand,
CompleteMultipartUploadCommand
} from '@aws-sdk/client-s3';
// 100 MB chunk/part size
const CHUNK_SIZE = 1024 * 1024 * 100;
// Max retries when uploading parts
const MAX_RETRIES = 3;
const multipartS3Uploader = async (filePath, options) => {
const { region, contentType, key, bucket } = options;
// Get file size
const fileSize = fs.statSync(filePath).size;
// Calculate total parts
const totalParts = Math.ceil(fileSize / CHUNK_SIZE);
// Initialize the S3 client instance
const S3 = new S3Client({ region });
const uploadParams = { Bucket: bucket, Key: key, ContentType: contentType };
let PartNumber = 1;
const uploadPartResults = [];
// Send multipart upload request to S3, this returns a UploadId for use when uploading individual parts
// https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#createmultipartupload
let { UploadId } = await S3.send(new CreateMultipartUploadCommand(uploadParams));
console.log(`Initiate multipart upload, uploadId: ${UploadId}, totalParts: ${totalParts}, fileSize: ${fileSize}`);
// Read file parts and upload parts to s3, this promise resolves when all parts are uploaded successfully
await new Promise(resolve => {
fs.open(filePath, 'r', async (err, fileDescriptor) => {
if (err) throw err;
// Read and upload file parts until end of file
while (true) {
// Read next file chunk
const { buffer, bytesRead } = await readNextPart(fileDescriptor);
// When end-of-file is reached bytesRead is zero
if (bytesRead === 0) {
// Done reading file, close the file, resolve the promise and return
fs.close(fileDescriptor, (err) => { if (err) throw err; });
return resolve();
}
// Get data chunk/part
const data = bytesRead < CHUNK_SIZE ? buffer.slice(0, bytesRead) : buffer;
// Upload data chunk to S3
const response = await uploadPart(S3,
{ data, bucket, key, PartNumber, UploadId }
);
console.log(`Uploaded part ${PartNumber} of ${totalParts}`);
uploadPartResults.push({ PartNumber, ETag: response.ETag });
PartNumber++;
}
});
});
console.log(`Finish uploading all parts for multipart uploadId: ${UploadId}`);
// Completes a multipart upload by assembling previously uploaded parts.
// https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#completemultipartupload
let completeUploadResponse = await S3.send(new CompleteMultipartUploadCommand({
Bucket: bucket,
Key: key,
MultipartUpload: { Parts: uploadPartResults },
UploadId: UploadId
}));
console.log('Successfully completed multipart upload');
return completeUploadResponse;
};
const readNextPart = async (fileDescriptor) => await new Promise((resolve, reject) => {
// Allocate an empty buffer to save data chunk that is read
const buffer = Buffer.alloc(CHUNK_SIZE);
fs.read(
fileDescriptor,
buffer, // Buffer where data will be written
0, // Start Offset on buffer while writing data
CHUNK_SIZE, // Length of bytes to read
null, // Position in file(PartNumber * CHUNK_SIZE); if position is null data is read from the current file position, and the position is updated
(err, bytesRead) => { // Callback function
if (err) return reject(err);
resolve({ bytesRead, buffer });
});
});
// Upload a given part with retries
const uploadPart = async (S3, options, retry = 1) => {
const { data, bucket, key, PartNumber, UploadId } = options;
let response;
try {
// Upload part to S3
// https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/classes/s3.html#uploadpart
response = await S3.send(
new UploadPartCommand({
Body: data,
Bucket: bucket,
Key: key,
PartNumber,
UploadId
})
);
} catch {
console.log(`ATTEMPT-#${retry} Failed to upload part ${PartNumber} due to ${JSON.stringify(response)}`);
if (retry >= MAX_RETRIES)
throw (response);
else
return uploadPart(S3, options, retry + 1);
}
return response;
};
export default multipartS3Uploader;
// Example:
// await multipartS3Uploader(<file path>,
// {
// region: 'us-east-1',
// bucket: 'my-s3-bucket',
// key: <file path in bucket>
// }
// );
// Try running the script like:
// AWS_ACCESS_KEY_ID=xxxx AWS_SECRET_ACCESS_KEY=xxxx node index.js
@Belrestro
Copy link

Thanks for creating this bucket, new aws s3 SDK has shamefully small number of examples, this one has proven to be most useful.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment