Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Example AWS S3 Multipart Upload with aws-sdk for Node.js - Retries to upload failing parts
// Based on Glacier's example: http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/examples.html#Amazon_Glacier__Multi-part_Upload
var fs = require('fs');
var AWS = require('aws-sdk');
AWS.config.loadFromPath('./aws-config.json');
var s3 = new AWS.S3();
// File
var fileName = '5.pdf';
var filePath = './' + fileName;
var fileKey = fileName;
var buffer = fs.readFileSync('./' + filePath);
// S3 Upload options
var bucket = 'loctest';
// Upload
var startTime = new Date();
var partNum = 0;
var partSize = 1024 * 1024 * 5; // Minimum 5MB per chunk (except the last part) http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
var numPartsLeft = Math.ceil(buffer.length / partSize);
var maxUploadTries = 3;
var multiPartParams = {
Bucket: bucket,
Key: fileKey,
ContentType: 'application/pdf'
};
var multipartMap = {
Parts: []
};
function completeMultipartUpload(s3, doneParams) {
s3.completeMultipartUpload(doneParams, function(err, data) {
if (err) {
console.log("An error occurred while completing the multipart upload");
console.log(err);
} else {
var delta = (new Date() - startTime) / 1000;
console.log('Completed upload in', delta, 'seconds');
console.log('Final upload data:', data);
}
});
}
function uploadPart(s3, multipart, partParams, tryNum) {
var tryNum = tryNum || 1;
s3.uploadPart(partParams, function(multiErr, mData) {
if (multiErr){
console.log('multiErr, upload part error:', multiErr);
if (tryNum < maxUploadTries) {
console.log('Retrying upload of part: #', partParams.PartNumber)
uploadPart(s3, multipart, partParams, tryNum + 1);
} else {
console.log('Failed uploading part: #', partParams.PartNumber)
}
return;
}
multipartMap.Parts[this.request.params.PartNumber - 1] = {
ETag: mData.ETag,
PartNumber: Number(this.request.params.PartNumber)
};
console.log("Completed part", this.request.params.PartNumber);
console.log('mData', mData);
if (--numPartsLeft > 0) return; // complete only when all parts uploaded
var doneParams = {
Bucket: bucket,
Key: fileKey,
MultipartUpload: multipartMap,
UploadId: multipart.UploadId
};
console.log("Completing upload...");
completeMultipartUpload(s3, doneParams);
});
}
// Multipart
console.log("Creating multipart upload for:", fileKey);
s3.createMultipartUpload(multiPartParams, function(mpErr, multipart){
if (mpErr) { console.log('Error!', mpErr); return; }
console.log("Got upload ID", multipart.UploadId);
// Grab each partSize chunk and upload it as a part
for (var rangeStart = 0; rangeStart < buffer.length; rangeStart += partSize) {
partNum++;
var end = Math.min(rangeStart + partSize, buffer.length),
partParams = {
Body: buffer.slice(rangeStart, end),
Bucket: bucket,
Key: fileKey,
PartNumber: String(partNum),
UploadId: multipart.UploadId
};
// Send a single part
console.log('Uploading part: #', partParams.PartNumber, ', Range start:', rangeStart);
uploadPart(s3, multipart, partParams);
}
});
@danneu
Copy link

danneu commented Oct 31, 2017

AWS SDK implements arbitrary-size uploads for you: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property

@yashSrijan
Copy link

you have saved me a hell lot of time.. tysm !

@Gabbar3112
Copy link

not given proper how to run this code

@ony19161
Copy link

This script is for file size less than 2gb, for above 2gb file its always getting the following error
" multiErr, upload part error: Error: Timeout
at XMLHttpRequest. (aws-sdk-2.283.1.min.js:50) "

can you help please

@sharma-vishal-660
Copy link

this code is working fine for me in chrome but i see very slow performance on IE-11. can you please suggest me what could be going wrong. why the upload speed is slow on IE but not on other browsers.

@rishisharma08
Copy link

@yiweirong if still having that issue: you have to make sure that the s3.completeMultipartUpload function gets called after all the parts have been uploaded and that multipartMap.Parts is complete with information about all the parts.
i just had that issue and fixed it by doing that.

@engrpeters
Copy link

I should ask . Can't you call list parts , instead of manually saving the parts that have been uploaded .

@sboyina
Copy link

sboyina commented Aug 20, 2020

Thanks for the code.

@tania11
Copy link

tania11 commented Aug 25, 2020

Thank you for the code.Very helpfull

@nassiharel
Copy link

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

@marceloavf
Copy link

marceloavf commented Oct 19, 2020

@PierBover
Copy link

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

@nassiharel
Copy link

nassiharel commented Nov 3, 2020

Yes, you are right @marceloavf, @PierBover.
But this is a multi-part gist, so I just shared my part if someone still wants to use the multi-part API.
There is also an API for getting multi-parts.

@hightechtony
Copy link

@PierBover how would one check upload progress?

@hightechtony
Copy link

hightechtony commented Nov 6, 2020

@PierBover figured it out heres the snippet

   try {
   	await s3.upload(params, options)
    .on('httpUploadProgress', function(evt) {

     console.log("Uploaded :: " + parseInt((evt.loaded * 100) / evt.total)+'%');
   })
    .promise();
   	console.log('upload OK' + oldpath);

   } catch (error) {

   	console.log('upload ERROR' + error);
   }

@ferozsho
Copy link

@PierBover figured it out heres the snippet

   try {
   	await s3.upload(params, options)
    .on('httpUploadProgress', function(evt) {

     console.log("Uploaded :: " + parseInt((evt.loaded * 100) / evt.total)+'%');
   })
    .promise();
   	console.log('upload OK' + oldpath);

   } catch (error) {

   	console.log('upload ERROR' + error);
   }

hi, the "httpUploadProgress" showing after uploaded complete to s3 on uploading its not firing the progress event, please advice.

@nickwild-999
Copy link

nickwild-999 commented Jan 25, 2021

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

@marceloavf, @PierBover. Do you have the code for the full async function for this please?

@nickwild-999
Copy link

Thanks @marceloavf but this code is quite different from the code @PierBover posted and I am notusing Tyoescript. Do you have any code for the async function for this @PierBover?

@PierBover
Copy link

@nickwild-999 just put the example into an async function. My example is already using await.

@jotta008
Copy link

jotta008 commented Jan 31, 2021

@marceloavf is right. Using .upload() you get automatic multipart uploads with concurrency and retries.

const AWS = require('aws-sdk');
const mime = require('mime-types');
const endpoint = new AWS.Endpoint(S3_ENDPOINT);

const S3 = new AWS.S3({
	endpoint: endpoint,
	accessKeyId: S3_KEY,
	secretAccessKey: S3_SECRET,
	maxRetries: 10
});

const stream = fs.createReadStream(filePath);
const contentType = mime.lookup(filePath)

const params = {
	Bucket: BUCKET_NAME,
	Key: OBJECT_KEY,
	Body: stream,
	ACL: 'public-read',
	ContentType: contentType
};

const options = {
	partSize: 10 * 1024 * 1024,
        // how many concurrent uploads
	queueSize: 5
};

try {
	await S3.upload(params, options).promise();
	console.log('upload OK', filePath);
} catch (error) {
	console.log('upload ERROR', filePath, error);
}

Try in Node 14?

@mike1011
Copy link

mike1011 commented Feb 13, 2021

Has anybody been able to set Metadata fields with createMultipartUpload?

This might help.
const params = {
ACL: "public-read",
Key: "sample-videofile",
ContentType: video/mp4,
Body: file,
ContentLength: file.size,
Metadata: {
user_email: "sample@gmail.com",
user_id: "300",
},
};

@mike1011
Copy link

Upload with progress bar...this is a tested code.

S3.upload(params, options, function (err, data) {
          if (err) {
            reject("error");
          }
          alert("Successfully Uploaded!");
        }).on("httpUploadProgress", (progress) => {
          let uploaded = parseInt((progress.loaded * 100) / progress.total);
          this.setState({
            progress: uploaded,
            uploadText: "Uploading...",
            uploading: true,
          });
        });

@avi2021
Copy link

avi2021 commented Mar 25, 2021

data

How to call the above function for multiple large files

Example:

FileList - [file1, file2]

let PromiseArray = []

for(file in FileList){
PromiseArray.push(multiPart(file))
}
Promise.all(PromiseArray)
.then(result => {
//succese
})

when I can parallel the multiPart function its mismatch file 1 and file 2 data and overwrite one of the file for both

Any Solution ?

@swetarajbhar
Copy link

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

@swetarajbhar
Copy link

this._client.createMultipartUpload is available in which npm package ?
Please Reply
Does this code work ??

@soubhikchatterjee
Copy link

Just one question. Does this occupy the nodejs server's disk space during the upload process?

@soubhikchatterjee
Copy link

thanks for the input!

I created a similar script based on the async package including retries that might be a bit easier to use & understand for some ppl!

https://gist.github.com/magegu/ea94cca4a40a764af487

I am looking into your script, and wondering if there is a way i can resume broken uploads? For eg if my frontend app has uploaded PartNumber:2 and then encountered a broken internet connection, is it possible to resume the upload from where it got last uploaded i.e. partNumber2 instead of uploading from 0?

@Grendaizo90
Copy link

Grendaizo90 commented Oct 20, 2021

    /**
     * initiate a multipart upload and get an upload ID that must include in upload part request.
     * Each part must be at least 5 MB in size, except the last part.
     */
    async multiPart(options) {
        const { data, bucket, key } = options;
        const multiPartParams = {
            Bucket: bucket,
            Key: key
        };
        const multipart = await this._client.createMultipartUpload(multiPartParams).promise();
        const multipartMap = { Parts: [] };
        let partNum = 0;
        for (const d of data) {
            partNum += 1;
            const partParams = { ...multiPartParams, Body: d, PartNumber: String(partNum), UploadId: multipart.UploadId };
            const result = await this._client.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = { ETag: result.ETag, PartNumber: Number(partNum) };
        }
        const doneParams = { ...multiPartParams, MultipartUpload: multipartMap, UploadId: multipart.UploadId };
        const result = await this._client.completeMultipartUpload(doneParams).promise();
        return result;
    }

Hi! Your solution looks very good, but i don't understand what kind of data i should use? I mean your data looks like it is an iterable object

@thewebguy27
Copy link

@jotta008 Thank you !
Will this work with file size of 600 mb?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment