chunked-upload-example.js

'use strict';

// Require in some useful utilities and a request library
const request = require('request-promise');
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');

// Node.js streams don't provide strong guarantees about reading exactly the
// amount of data you request, so we might need to buffer data in some cases
const streamBuffer = [];

// Helper function for getting one chunk of data from the stream
// Since Node.js streams don't guarantee that they'll always give
// you the exact amount of data you asked for, there's some special
// logic for waiting for more data to be read from disk and buffering
// any extra data that comes back from the stream.
function getNextChunkFromStream(stream, chunkSize) {

    let chunk = stream.read(chunkSize);

    if (!chunk) {
        // The stream needs time to read more data from disk, wait for one turn
        // of the event loop and then call this function again to retry.
        return new Promise(resolve => setTimeout(resolve, 100))
            .then(() => getNextChunkFromStream(stream, chunkSize));
    }
    
    if (chunk.length > chunkSize) {

        // The stream is done reading and had extra data, so we need to
        // buffer the remainder of the file since chunks have a fixed size.
        for (let i = 0; i < chunk.length; i += chunkSize) {
            streamBuffer.push(chunk.slice(i, i + chunkSize));
        }
        chunk = streamBuffer.shift();
    }

    return Promise.resolve(chunk);
}

// ----------------------------------------------------------------------------
// Main parameters below.  Fill these in with your own values!
// ----------------------------------------------------------------------------
const ACCESS_TOKEN = 'INSERT_ACCESS_TOKEN_HERE';
const FILE_PATH = 'PATH_TO_FILE';

// Wrap the main logic in an async function so we can use await to make the code easier to follow
(async () => {

// A stream to read the file from disk
let fileStream = fs.createReadStream(FILE_PATH);

// Get the total size of the file in bytes
let fileSize = fs.statSync(FILE_PATH).size;

// A variable to keep track of our progress through the file
let position = 0;

// A place to store the chunk records for each upload; we'll need these
// to commit the upload session.  These must be in order!
let chunkRecords = [];

// We'll need to compute the hash of the entire file; we can do that as we
// go with a Node.js Hash object
let fileHash = crypto.createHash('sha1');

// ----------------------------------------------------------------------------
// Chunked upload logic starts here!
// ----------------------------------------------------------------------------

// Create the upload session
let uploadSessionInfo = await request({
    json: true,
    method: 'POST',
    uri: 'https://upload.box.com/api/2.0/files/upload_sessions',
    headers: {
        Authorization: `Bearer ${ACCESS_TOKEN}`
    },
    body: {
        folder_id: '0', // upload to All Files
        file_size: fileSize,
        file_name: path.basename(FILE_PATH)
    }
});

console.log('Created upload session');
console.log(uploadSessionInfo);

// The session information includes the URLs to use when performing later operations,
// so we can save those for when we use then later.
let uploadPartURL = uploadSessionInfo.session_endpoints.upload_part;
let commitURL = uploadSessionInfo.session_endpoints.commit;

// The upload session also gives the appropriate chunk size, so we'll save that for later use.
let chunkSize = uploadSessionInfo.part_size;

// We're reading from a stream, so we'll read off chunks of the correct
// size and then upload them one by one.  For simplicity, we'll do this
// serially instead of in parallel so the code is less confusing..
while (position < fileSize) {

    // Get the next chunk to be uploaded
    let chunk = await getNextChunkFromStream(fileStream, chunkSize);
    
    // Update the in-progress hash of the full file with the new chunk
    fileHash.update(chunk);

    // Calculate the hash of just this chunk, encoded in base64
    let chunkHash = crypto.createHash('sha1').update(chunk).digest('base64');

    // Upload this chunk
    let chunkRecord = await request({
        method: 'PUT',
        uri: uploadPartURL,
        headers: {
            Authorization: `Bearer ${ACCESS_TOKEN}`,
            'Content-Type': 'application/octet-stream',
            Digest: `SHA=${chunkHash}`,
            'Content-Range': `bytes ${position}-${position + chunk.length - 1}/${fileSize}`
        },
        body: chunk,
    });
    chunkRecord = JSON.parse(chunkRecord);


    console.log('Uploaded bytes', position, 'to', position + chunk.length, 'out of total', fileSize);
    console.log(chunkRecord);

    // Save the chunk record for committing the upload session later
    chunkRecords.push(chunkRecord.part);

    // Advance the tracking variable for the position in the file that we're at
    position += chunk.length;
}

// Commit the upload session once all chunks are uploaded
let uploadedFile = await request({
    json: true,
    method: 'POST',
    uri: commitURL,
    headers: {
        Authorization: `Bearer ${ACCESS_TOKEN}`,
        Digest: `SHA=${fileHash.digest('base64')}`
    },
    body: {
        parts: chunkRecords
    }
});

console.log('Successfully uploaded file!');
console.log(uploadedFile);

// End wrapper async function
})();