Skip to content

Instantly share code, notes, and snippets.

@ve3
Last active November 22, 2023 10:25
Show Gist options
  • Save ve3/9bc4ec3ee9bcce755c23bddbfce33306 to your computer and use it in GitHub Desktop.
Save ve3/9bc4ec3ee9bcce755c23bddbfce33306 to your computer and use it in GitHub Desktop.
Slice large file into chunks and upload using JavaScript.
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>XHR upload large file with slice</title>
<style>
#debug {
border: 3px dashed #ccc;
margin: 10px 0;
padding: 10px;
}
</style>
</head>
<body>
<p>XHR upload large file with slice. Upload multiple chunks per time.</p>
<form id="upload-form" method="post" enctype="multipart/form-data">
<input type="hidden" name="hidden-name" value="hidden-value">
<p>text: <input type="text" name="text"></p>
<p>file: <input id="file" type="file" name="file"></p>
<button type="submit">Submit</button>
<button type="reset">Reset</button>
<div id="debug"></div>
<p>
References:<br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise" target="_blank">https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FileReader" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FileReader</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/File" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/File</a><br>
</p>
</form>
<script src="xhr.js"></script>
<script type="application/javascript">
function uploadChunks(formData, file, numberOfChunks) {
function doUpload(i, allResolve, allReject) {
formData.delete('file');// delete previous for append new.
formData.append('file', chunkContentParts[i]);
return XHR('upload-single.php?chunkNumber=' + i, formData)
.then((responseObject) => {
const response = responseObject.response;
if (typeof(response.chunkNumber) === 'number' && totalFailure > 0) {
totalFailure = (totalFailure - 1);// decrease total failure.
} else if (typeof(response.chunkNumber) === 'undefined' || response.chunkNumber === null || response.chunkNumber === '') {
// if did not response chunk number that was uploaded
// mark as failure permanently and can't continue. you must have return `chunkNumber` from server before continue.
totalFailure = (totalFailure + 100000);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: red;"> &nbsp; &gt; The property chunkNumber must be returned from the server.</p>');
throw new Error('The property chunkNumber must be returned from the server.');
}
console.log('upload for chunk number ' + i + ' of ' + (numberOfChunks - 1) + ' success.', response);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p> &nbsp; &gt; Chunk number ' + response.chunkNumber + ' uploaded success!</p>');
if (response.chunkNumber) {
// if there is response chunk number.
// add success number to array.
ajaxSuccessChunks.push(response.chunkNumber);
delete chunkContentParts[i];
}
if (parseInt(ajaxSuccessChunks.length) === (parseInt(numberOfChunks) - 1)) {
// if finish upload all chunks.
console.log('all chunks uploaded completed.');
allResolve(responseObject);
}
return Promise.resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
totalFailure++;// increase total failure.
console.warn('connection error! Loop ' + i, responseObject);
if (totalFailure <= maxFailConnection) {
// if total failure does not reach limit.
// retry.
console.warn('retrying from total failure: ', totalFailure);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: red;"> &nbsp; &gt; Error in chunk number ' + response.chunkNumber + '!, retrying. (see console.)</p>');
doUpload(i, allResolve, allReject);
} else {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: red;"> &nbsp; &gt; Error in chunk number ' + response.chunkNumber + '! aborting.(see console.)</p>');
}
return Promise.reject(responseObject);
})
}// doUpload
let chunkStart = 0;
let chunkEnd = parseInt(chunkFileSize);
let totalFailure = 0;
let ajaxCons = [];
let ajaxSuccessChunks = [];
let chunkContentParts = {};
let promiseObject = new Promise((allResolve, allReject) => {
let loopPromise = Promise.resolve();
for (let i = 0; i < numberOfChunks; i++) {
loopPromise = loopPromise.then(() => {
if (ajaxCons.length < maxConcurrentConnection && totalFailure <= maxFailConnection) {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p>Uploading file to server for chunk number ' + i + ' of ' + (numberOfChunks - 1) + '</p>');
chunkContentParts[i] = file.slice(chunkStart, chunkEnd, file.type);
chunkStart = chunkEnd;
chunkEnd = (chunkStart + parseInt(chunkFileSize));
ajaxCons.push(
doUpload(i, allResolve, allReject)
);
if ((parseInt(ajaxCons.length)) === parseInt(maxConcurrentConnection)) {
// if number of concurrent connection reach maximum allowed.
// hold using Promise.
return new Promise((resolve, reject) => {
Promise.any(ajaxCons)
.then((responseObject) => {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p> &nbsp; <em>Removing finished connection count to allow make new connection.</em></p>');
ajaxCons.splice(0, 1);
resolve();
});
});
}
}// endif; concurrent connection not reach maximum.
});// loopPromise.then()
}// endfor;
});
return promiseObject;
}// uploadChunks
function mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks) {
let loopPromise = Promise.resolve();
// loop request for merge uploaded chunks.
// start from 1 because 0 already has been done by first request where there is only chunkNumber=-1 in GET parameter.
for (let i = 1; i < totalMergeLoop; i++) {
loopPromise = loopPromise.then(() => {
return new Promise((resolve, reject) => {
XHR('upload-single.php?chunkNumber=-1&startMergeOffset=' + startMergeOffset, formData)
.then((responseObject) => {
const response = responseObject.response;
console.log('merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1));
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p> &nbsp; &gt; Merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1) + '</p>');
startMergeOffset = response.mergedChunkNumberEnd
resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
reject(responseObject);
});
});// end return new Promise()
});// end loopPromise.then()
}// endfor;
return loopPromise;
}// mergeChunks
let debugElement = document.getElementById('debug');
const thisForm = document.getElementById('upload-form');
const inputFile = thisForm.querySelector('#file');
const chunkFileSize = 1000000;// 1,000,000 bytes = 1MB. Limit this too high may cause out of memory on server when merge process.
const maxConcurrentConnection = 3;// Limit too much concurrent connection may flood the request on server and can be blocked by firewall.
const maxFailConnection = 3;
thisForm.addEventListener('submit', (event) => {
event.preventDefault();
let formData = new FormData(thisForm);
formData.delete('file');// delete original input file.
if (!inputFile || !inputFile.files || inputFile.files.length <= 0) {
alert('Please select a file to upload.');
return ;
}
const file = inputFile.files[0];
const fileName = inputFile.files[0].name;
const numberOfChunks = Math.ceil(parseInt(file.size) / chunkFileSize);
// for debug
debugElement.innerHTML = '';
let debugMessage = '<p>File size: ' + file.size + ' bytes.<br>'
+ ' Chunk file size: ' + chunkFileSize + ' bytes.<br>'
+ ' Number of chunks: ' + numberOfChunks + ' (loop 0 - ' + (parseInt(numberOfChunks) - 1) + ').'
+ '</p><hr style="border: none; border-top: 1px dashed #ccc;">';
debugElement.insertAdjacentHTML('beforeend', debugMessage);
// upload chunks.
uploadChunks(formData, file, numberOfChunks)
// all chunks were uploaded successfully.
.then((responseObject) => {
// prepare for merge them.
formData.delete('file');
formData.append('file_name', file.name);
formData.append('file_size', file.size);
formData.append('file_mimetype', file.type);
formData.append('file_chunkcount', (parseInt(numberOfChunks) - 1));
let totalMergeLoop = 0;
let allSuccess = false;
let startMergeOffset = 0;
console.log('starting to merge chunks.');
XHR('upload-single.php?chunkNumber=-1', formData)
.then((responseObject) => {
const response = responseObject.response;
if (response.mergeSuccess === true) {
// if merged success.
allSuccess = true;
console.log('all chunks merged completed (in 1 request).');
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p><hr>');
} else {
if (response.totalMergeLoop && totalMergeLoop === 0) {
totalMergeLoop = parseInt(response.totalMergeLoop);
}
startMergeOffset = parseInt(response.mergedChunkNumberEnd);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<hr style="border: none; border-top: 1px dashed #ccc;">');
debugElement.insertAdjacentHTML('beforeend', '<p>Starting to merge uploaded temp files. Total loop: ' + totalMergeLoop + ', start next merge offset: ' + startMergeOffset + '</p>');
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.then((responseObject) => {
if (totalMergeLoop > 0 && allSuccess === false) {
mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks)
.then(() => {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p>');
console.log('all chunks were merged complete successfully.');
});
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.catch((responseObject) => {
console.warn(responseObject);
const response = responseObject.response;
if (response.error && response.error.message) {
alert(response.error.message);
}
return Promise.reject(responseObject);
}); // merge chunks first round finished.
}); // uploadChunks() promise finished.
});// form event listener submit.
</script>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>XHR upload large file with slice - with progress bar</title>
<style>
#debug {
border: 3px dashed #ccc;
margin: 10px 0;
padding: 10px;
}
.file-chunks-progress {
border: 1px solid #bbb;
display: inline-flex;
flex-direction: row;
flex-wrap: wrap;
}
.file-chunks-progress .each-chunk {
border: 1px solid #bbb;
height: 10px;
margin: -1px 0 0 -1px;
width: 10px;;
}
.file-chunks-progress .each-chunk.success {
background-color: lightgreen;
}
.file-chunks-progress .each-chunk.error {
background-color: rgb(223, 130, 130);
}
.file-chunks-progress .each-chunk.uploading {
background-color: #eee;
}
.text-fade {
color: #999;
}
.text-super-fade {
color: #ddd;
}
</style>
</head>
<body>
<p>XHR upload large file with slice. Upload multiple chunks per time.</p>
<p>With progress bar example.</p>
<form id="upload-form" method="post" enctype="multipart/form-data">
<input type="hidden" name="hidden-name" value="hidden-value">
<p>text: <input type="text" name="text"></p>
<p>file: <input id="file" type="file" name="file"> <progress id="file-progress" max="100"></progress></p>
<button type="submit">Submit</button>
<button type="reset">Reset</button>
<div id="debug"></div>
<p>
References:<br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise" target="_blank">https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FileReader" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FileReader</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/File" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/File</a><br>
</p>
</form>
<script src="xhr.js"></script>
<script type="application/javascript">
function mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks) {
let loopPromise = Promise.resolve();
// loop request for merge uploaded chunks.
// start from 1 because 0 already has been done by first request where there is only chunkNumber=-1 in GET parameter.
for (let i = 1; i < totalMergeLoop; i++) {
loopPromise = loopPromise.then(() => {
return new Promise((resolve, reject) => {
XHR('upload-single.php?chunkNumber=-1&startMergeOffset=' + startMergeOffset, formData)
.then((responseObject) => {
const response = responseObject.response;
console.log('merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1));
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p> &nbsp; &gt; Merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1) + '</p>');
startMergeOffset = response.mergedChunkNumberEnd
resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
reject(responseObject);
});
});// end return new Promise()
});// end loopPromise.then()
}// endfor;
return loopPromise;
}// mergeChunks
function renderChunkProgress(maxChunks) {
let debugElement = document.getElementById('debug');
let chunkProgressbarHtml = '<div class="file-chunks-progress">';
for (let i = 0; i < maxChunks; i++) {
chunkProgressbarHtml += '<div id="file-chunk-index-i-' + i + '" class="each-chunk" title="chunk number ' + i + '"></div>';
}
chunkProgressbarHtml += '</div>';
debugElement.insertAdjacentHTML('beforeend', chunkProgressbarHtml);
}// renderChunkProgress
function updateProgress() {
const inputFile = thisForm.querySelector('#file');
const file = inputFile.files[0];
let progressBarValue = progressBar.value;
if (typeof(progressBarValue) === 'undefined' || progressBarValue === null || progressBarValue === '') {
progressBarValue = 0;
} else {
progressBarValue = parseInt(progressBarValue);
}
const newProgressBarValue = (progressBarValue + 1);
progressBar.value = newProgressBarValue;
}// updateProgress
function uploadChunks(formData, file, numberOfChunks) {
function doUpload(i, allResolve, allReject) {
formData.delete('file');// delete previous for append new.
formData.append('file', chunkContentParts[i]);
let fileChunkProgressBlock = document.querySelector('#file-chunk-index-i-' + i);
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.add('uploading');
}
return XHR('upload-single.php?chunkNumber=' + i, formData)
.then((responseObject) => {
const response = responseObject.response;
if (typeof(response.chunkNumber) === 'number' && totalFailure > 0) {
totalFailure = (totalFailure - 1);// decrease total failure.
} else if (typeof(response.chunkNumber) === 'undefined' || response.chunkNumber === null || response.chunkNumber === '') {
// if did not response chunk number that was uploaded
// mark as failure permanently and can't continue. you must have return `chunkNumber` from server before continue.
totalFailure = (totalFailure + 100000);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: red;"> &nbsp; &gt; The property chunkNumber must be returned from the server.</p>');
throw new Error('The property chunkNumber must be returned from the server.');
}
updateProgress();
console.log('upload for chunk number ' + i + ' of ' + (numberOfChunks - 1) + ' success.', response);
// for debug
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.remove('uploading');
fileChunkProgressBlock.classList.add('success');
}
if (response.chunkNumber) {
// if there is response chunk number.
// add success number to array.
ajaxSuccessChunks.push(response.chunkNumber);
delete chunkContentParts[i];
}
if (parseInt(ajaxSuccessChunks.length) === (parseInt(numberOfChunks) - 1)) {
// if finish upload all chunks.
console.log('all chunks uploaded completed.');
allResolve(responseObject);
}
return Promise.resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
totalFailure++;// increase total failure.
console.warn('connection error! Loop ' + i, responseObject);
if (totalFailure <= maxFailConnection) {
// if total failure does not reach limit.
// retry.
console.warn('retrying from total failure: ', totalFailure);
doUpload(i, allResolve, allReject);
} else {
// for debug
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.remove('uploading');
fileChunkProgressBlock.classList.add('error');
}
}
return Promise.reject(responseObject);
});
}// doUpload
let chunkStart = 0;
let chunkEnd = parseInt(chunkFileSize);
let totalFailure = 0;
let ajaxCons = [];
let ajaxSuccessChunks = [];
let chunkContentParts = {};
let promiseObject = new Promise((allResolve, allReject) => {
let loopPromise = Promise.resolve();
for (let i = 0; i < numberOfChunks; i++) {
loopPromise = loopPromise.then(() => {
if (ajaxCons.length < maxConcurrentConnection && totalFailure <= maxFailConnection) {
chunkContentParts[i] = file.slice(chunkStart, chunkEnd, file.type);
chunkStart = chunkEnd;
chunkEnd = (chunkStart + parseInt(chunkFileSize));
ajaxCons.push(
doUpload(i, allResolve, allReject)
);
if ((parseInt(ajaxCons.length)) === parseInt(maxConcurrentConnection)) {
// if number of concurrent connection reach maximum allowed.
// hold using Promise.
return new Promise((resolve, reject) => {
Promise.any(ajaxCons)
.then((responseObject) => {
ajaxCons.splice(0, 1);
resolve();
});
});
}
}// endif; concurrent connection not reach maximum.
});// loopPromise.then()
}// endfor;
});
return promiseObject;
}// uploadChunks
let debugElement = document.getElementById('debug');
const thisForm = document.getElementById('upload-form');
const inputFile = thisForm.querySelector('#file');
const progressBar = thisForm.querySelector('#file-progress');
const chunkFileSize = 1000000;// 1,000,000 bytes = 1MB. Limit this too high may cause out of memory on server when merge process.
const maxConcurrentConnection = 3;// Limit too much concurrent connection may flood the request on server and can be blocked by firewall.
const maxFailConnection = 3;
thisForm.addEventListener('submit', (event) => {
event.preventDefault();
let formData = new FormData(thisForm);
formData.delete('file');// delete original input file.
if (!inputFile || !inputFile.files || inputFile.files.length <= 0) {
alert('Please select a file to upload.');
return ;
}
const file = inputFile.files[0];
const fileName = inputFile.files[0].name;
const numberOfChunks = Math.ceil(parseInt(file.size) / chunkFileSize);
// assign new `max` for progress bar.
progressBar.max = numberOfChunks;
progressBar.value = 0;// reset.
// for debug
debugElement.innerHTML = '';
let debugMessage = '<p>File size: ' + file.size + ' bytes.<br>'
+ ' Chunk file size: ' + chunkFileSize + ' bytes.<br>'
+ ' Number of chunks: ' + numberOfChunks + ' (loop 0 - ' + (parseInt(numberOfChunks) - 1) + ').<br>'
+ '</p><hr style="border: none; border-top: 1px dashed #ccc;">';
debugElement.insertAdjacentHTML('beforeend', debugMessage);
renderChunkProgress(numberOfChunks);
// upload chunks.
uploadChunks(formData, file, numberOfChunks)
// all chunks were uploaded successfully.
.then((responseObject) => {
// prepare for merge them.
formData.delete('file');
formData.append('file_name', file.name);
formData.append('file_size', file.size);
formData.append('file_mimetype', file.type);
formData.append('file_chunkcount', (parseInt(numberOfChunks) - 1));
let totalMergeLoop = 0;
let allSuccess = false;
let startMergeOffset = 0;
console.log('starting to merge chunks.');
XHR('upload-single.php?chunkNumber=-1', formData)
.then((responseObject) => {
const response = responseObject.response;
if (response.mergeSuccess === true) {
// if merged success.
allSuccess = true;
console.log('all chunks merged completed (in 1 request).');
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p><hr>');
} else {
if (response.totalMergeLoop && totalMergeLoop === 0) {
totalMergeLoop = parseInt(response.totalMergeLoop);
}
startMergeOffset = parseInt(response.mergedChunkNumberEnd);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<hr style="border: none; border-top: 1px dashed #ccc;">');
debugElement.insertAdjacentHTML('beforeend', '<p>Starting to merge uploaded temp files. Total loop: ' + totalMergeLoop + ', start next merge offset: ' + startMergeOffset + '</p>');
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.then((responseObject) => {
if (totalMergeLoop > 0 && allSuccess === false) {
mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks)
.then(() => {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p>');
console.log('all chunks were merged complete successfully.');
});
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.catch((responseObject) => {
console.warn(responseObject);
const response = responseObject.response;
if (response.error && response.error.message) {
alert(response.error.message);
}
return Promise.reject(responseObject);
}); // merge chunks first round finished.
}); // uploadChunks() promise finished.
});// form event listener submit.
</script>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>XHR upload large file with slice - with progress bar</title>
<style>
#debug {
border: 3px dashed #ccc;
margin: 10px 0;
padding: 10px;
}
.file-chunks-progress {
border: 1px solid #bbb;
display: inline-flex;
flex-direction: row;
flex-wrap: wrap;
}
.file-chunks-progress .each-chunk {
border: 1px solid #bbb;
height: 10px;
margin: -1px 0 0 -1px;
width: 10px;;
}
.file-chunks-progress .each-chunk.success {
background-color: lightgreen;
}
.file-chunks-progress .each-chunk.error {
background-color: rgb(223, 130, 130);
}
.file-chunks-progress .each-chunk.uploading {
background-color: #eee;
}
.text-fade {
color: #999;
}
.text-super-fade {
color: #ddd;
}
</style>
</head>
<body>
<p>XHR upload large file with slice. Upload single chunk per time.</p>
<p>With progress bar example.</p>
<form id="upload-form" method="post" enctype="multipart/form-data">
<input type="hidden" name="hidden-name" value="hidden-value">
<p>text: <input type="text" name="text"></p>
<p>file: <input id="file" type="file" name="file"> <progress id="file-progress" max="100"></progress></p>
<button type="submit">Submit</button>
<button type="reset">Reset</button>
<div id="debug"></div>
<p>
References:<br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FormData/FormData</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise" target="_blank">https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/XMLHttpRequest</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/FileReader" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/FileReader</a><br>
<a href="https://developer.mozilla.org/en-US/docs/Web/API/File" target="_blank">https://developer.mozilla.org/en-US/docs/Web/API/File</a><br>
</p>
</form>
<script src="xhr.js"></script>
<script type="application/javascript">
function mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks) {
let loopPromise = Promise.resolve();
// loop request for merge uploaded chunks.
// start from 1 because 0 already has been done by first request where there is only chunkNumber=-1 in GET parameter.
for (let i = 1; i < totalMergeLoop; i++) {
loopPromise = loopPromise.then(() => {
return new Promise((resolve, reject) => {
XHR('upload-single.php?chunkNumber=-1&startMergeOffset=' + startMergeOffset, formData)
.then((responseObject) => {
const response = responseObject.response;
console.log('merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1));
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p> &nbsp; &gt; Merged chunk ' + startMergeOffset + ' to ' + (parseInt(response.mergedChunkNumberEnd) - 1) + ' of ' + (numberOfChunks - 1) + '</p>');
startMergeOffset = response.mergedChunkNumberEnd
resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
reject(responseObject);
});
});// end return new Promise()
});// end loopPromise.then()
}// endfor;
return loopPromise;
}// mergeChunks
function renderChunkProgress(maxChunks) {
let debugElement = document.getElementById('debug');
let chunkProgressbarHtml = '<div class="file-chunks-progress">';
for (let i = 0; i < maxChunks; i++) {
chunkProgressbarHtml += '<div id="file-chunk-index-i-' + i + '" class="each-chunk" title="chunk number ' + i + '"></div>';
}
chunkProgressbarHtml += '</div>';
debugElement.insertAdjacentHTML('beforeend', chunkProgressbarHtml);
}// renderChunkProgress
function updateMainProgress() {
const inputFile = thisForm.querySelector('#file');
const file = inputFile.files[0];
let progressBarValue = progressBar.value;
if (typeof(progressBarValue) === 'undefined' || progressBarValue === null || progressBarValue === '') {
progressBarValue = 0;
} else {
progressBarValue = parseInt(progressBarValue);
}
const newProgressBarValue = (progressBarValue + 1);
progressBar.value = newProgressBarValue;
}// updateMainProgress
function updateProgress(event) {
if (event.lengthComputable) {
// if computable.
const items = document.querySelectorAll('.each-chunk-progress');
let progressBar = items[(items.length - 1)];
if (progressBar) {
const percent = Math.round((event.loaded / event.total) * 100);
progressBar.value = percent;
}
// for debug
let bytesLoaded = parseInt(document.querySelector('#xhr-byte-loaded').innerHTML);
if (isNaN(bytesLoaded)) {
bytesLoaded = 0;
}
let newBytesLoaded = (bytesLoaded + parseInt(event.loaded));
document.querySelector('#xhr-byte-loaded').innerHTML = newBytesLoaded;
}
}// updateProgress
function uploadChunks(formData, file, numberOfChunks) {
function doUpload(i, allResolve, allReject) {
formData.delete('file');// delete previous for append new.
formData.append('file', chunkContentParts[i]);
let fileChunkProgressBlock = document.querySelector('#file-chunk-index-i-' + i);
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.add('uploading');
}
// for debug
debugElement.insertAdjacentHTML('beforeend', '<div>Upload chunk number ' + i + ' <progress class="each-chunk-progress" value="0" max="100"></progress></div>');
return XHR('upload-single.php?chunkNumber=' + i, formData, updateProgress)
.then((responseObject) => {
const response = responseObject.response;
if (typeof(response.chunkNumber) === 'number' && totalFailure > 0) {
totalFailure = (totalFailure - 1);// decrease total failure.
} else if (typeof(response.chunkNumber) === 'undefined' || response.chunkNumber === null || response.chunkNumber === '') {
// if did not response chunk number that was uploaded
// mark as failure permanently and can't continue. you must have return `chunkNumber` from server before continue.
totalFailure = (totalFailure + 100000);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: red;"> &nbsp; &gt; The property chunkNumber must be returned from the server.</p>');
throw new Error('The property chunkNumber must be returned from the server.');
}
updateMainProgress();
console.log('upload for chunk number ' + i + ' of ' + (numberOfChunks - 1) + ' success.', response);
// for debug
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.remove('uploading');
fileChunkProgressBlock.classList.add('success');
}
if (response.chunkNumber) {
// if there is response chunk number.
// add success number to array.
ajaxSuccessChunks.push(response.chunkNumber);
delete chunkContentParts[i];
}
if (parseInt(ajaxSuccessChunks.length) === (parseInt(numberOfChunks) - 1)) {
// if finish upload all chunks.
console.log('all chunks uploaded completed.');
allResolve(responseObject);
}
return Promise.resolve(responseObject);
})
.catch((responseObject) => {
const response = responseObject.response;
totalFailure++;// increase total failure.
console.warn('connection error! Loop ' + i, responseObject);
if (totalFailure <= maxFailConnection) {
// if total failure does not reach limit.
// retry.
console.warn('retrying from total failure: ', totalFailure);
doUpload(i, allResolve, allReject);
} else {
// for debug
if (fileChunkProgressBlock) {
fileChunkProgressBlock.classList.remove('uploading');
fileChunkProgressBlock.classList.add('error');
}
}
return Promise.reject(responseObject);
});
}// doUpload
let chunkStart = 0;
let chunkEnd = parseInt(chunkFileSize);
let totalFailure = 0;
let ajaxSuccessChunks = [];
let chunkContentParts = {};
let promiseObject = new Promise((allResolve, allReject) => {
let loopPromise = Promise.resolve();
for (let i = 0; i < numberOfChunks; i++) {
loopPromise = loopPromise.then(() => {
if (totalFailure <= maxFailConnection) {
chunkContentParts[i] = file.slice(chunkStart, chunkEnd, file.type);
chunkStart = chunkEnd;
chunkEnd = (chunkStart + parseInt(chunkFileSize));
return doUpload(i, allResolve, allReject);
}// endif; concurrent connection not reach maximum.
});// loopPromise.then()
}// endfor;
});
return promiseObject;
}// uploadChunks
let debugElement = document.getElementById('debug');
const thisForm = document.getElementById('upload-form');
const inputFile = thisForm.querySelector('#file');
const progressBar = thisForm.querySelector('#file-progress');
const chunkFileSize = 1000000;// 1,000,000 bytes = 1MB. Limit this too high may cause out of memory on server when merge process.
const maxFailConnection = 3;
thisForm.addEventListener('submit', (event) => {
event.preventDefault();
let formData = new FormData(thisForm);
formData.delete('file');// delete original input file.
if (!inputFile || !inputFile.files || inputFile.files.length <= 0) {
alert('Please select a file to upload.');
return ;
}
const file = inputFile.files[0];
const fileName = inputFile.files[0].name;
const numberOfChunks = Math.ceil(parseInt(file.size) / chunkFileSize);
// assign new `max` for progress bar.
progressBar.max = numberOfChunks;
progressBar.value = 0;// reset.
// for debug
debugElement.innerHTML = '';
let debugMessage = '<p>File size: ' + file.size + ' bytes.<br>'
+ ' Chunk file size: ' + chunkFileSize + ' bytes.<br>'
+ ' Number of chunks: ' + numberOfChunks + ' (loop 0 - ' + (parseInt(numberOfChunks) - 1) + ').<br>'
+ ' <span class="text-super-fade">Bytes upload (via XHR.progress - <small>count from <a href="https://developer.mozilla.org/en-US/docs/Web/API/ProgressEvent/total" target="_blank">body message</a> not a file.</small>): <span id="xhr-byte-loaded"></span></span><br>'
+ '</p><hr style="border: none; border-top: 1px dashed #ccc;">';
debugElement.insertAdjacentHTML('beforeend', debugMessage);
renderChunkProgress(numberOfChunks);
// upload chunks.
uploadChunks(formData, file, numberOfChunks)
// all chunks were uploaded successfully.
.then((responseObject) => {
// prepare for merge them.
formData.delete('file');
formData.append('file_name', file.name);
formData.append('file_size', file.size);
formData.append('file_mimetype', file.type);
formData.append('file_chunkcount', (parseInt(numberOfChunks) - 1));
let totalMergeLoop = 0;
let allSuccess = false;
let startMergeOffset = 0;
console.log('starting to merge chunks.');
XHR('upload-single.php?chunkNumber=-1', formData)
.then((responseObject) => {
const response = responseObject.response;
if (response.mergeSuccess === true) {
// if merged success.
allSuccess = true;
console.log('all chunks merged completed (in 1 request).');
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p><hr>');
} else {
if (response.totalMergeLoop && totalMergeLoop === 0) {
totalMergeLoop = parseInt(response.totalMergeLoop);
}
startMergeOffset = parseInt(response.mergedChunkNumberEnd);
// for debug
debugElement.insertAdjacentHTML('beforeend', '<hr style="border: none; border-top: 1px dashed #ccc;">');
debugElement.insertAdjacentHTML('beforeend', '<p>Starting to merge uploaded temp files. Total loop: ' + totalMergeLoop + ', start next merge offset: ' + startMergeOffset + '</p>');
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.then((responseObject) => {
if (totalMergeLoop > 0 && allSuccess === false) {
mergeChunks(formData, totalMergeLoop, startMergeOffset, numberOfChunks)
.then(() => {
// for debug
debugElement.insertAdjacentHTML('beforeend', '<p style="color: green;">All chunks were uploaded and merged successfully.</p>');
console.log('all chunks were merged complete successfully.');
});
}
return Promise.resolve(responseObject);
}, (responseObject) => {
return Promise.reject(responseObject);
})
.catch((responseObject) => {
console.warn(responseObject);
const response = responseObject.response;
if (response.error && response.error.message) {
alert(response.error.message);
}
return Promise.reject(responseObject);
}); // merge chunks first round finished.
}); // uploadChunks() promise finished.
});// form event listener submit.
</script>
</body>
</html>
<?php
/**
* This file will be included under the condition that `chunkNumber` is `-1`.
*/
$file_chunkcount = (isset($_POST['file_chunkcount']) ? (int) $_POST['file_chunkcount'] : false);
$file_name = 'upload/' . ($_POST['file_name'] ?? date('Y-m-dHis') . '_' . uniqid() . '.unknown');
if ($file_chunkcount !== false && $file_chunkcount > 0) {
// if chunk count is more than one file.
// limit max merge tasks per loop to prevent maximum execution timeout error. this way don't have to modify php ini value.
// do not limit max merge tasks per loop too much because it maybe out of memory or timeout error.
$maxTaskPerLoop = 10;
$totalMergeLoop = ceil($file_chunkcount / $maxTaskPerLoop);
$startMergeOffset = (int) ($_GET['startMergeOffset'] ?? 0);
$endMergeOffset = ($startMergeOffset + ($maxTaskPerLoop - 1));
$output['totalMergeLoop'] = $totalMergeLoop;
if ($endMergeOffset > $file_chunkcount) {
$endMergeOffset = $file_chunkcount;
}
for ($i = $startMergeOffset; $i <= $endMergeOffset; $i++) {
$eachChunkFile = 'upload/' . session_id() . '_' . $i . '.tmp';
$handleRead = fopen($eachChunkFile, 'rb');
$eachChunkContents = fread($handleRead, filesize($eachChunkFile));
fclose($handleRead);
unset($handleRead);
if ($i === 0) {
$mode = 'wb';
} else {
$mode = 'ab';
}
$handleWrite = fopen($file_name, $mode);
$writeStatus = fwrite($handleWrite, $eachChunkContents);
fclose($handleWrite);
if ($writeStatus === false) {
$output['error']['message'] = 'Write file error! chunk file name: ' . $eachChunkFile
. 'target file name: ' . $file_name;
$hasError = true;
http_response_code(500);
break;
} else {
unlink($eachChunkFile);
}
unset($eachChunkContents, $eachChunkFile, $handleWrite, $mode, $writeStatus);
}// endfor;
$output['mergedChunkNumberStart'] = $startMergeOffset;
$output['mergedChunkNumberEnd'] = $i;// the last $i is already +1. if $i is loop 0 to 9, the last one is $i++ = 10.
if (($i - 1) === $file_chunkcount) {
// if the last $i - 1 equal to total chunks.
// mark as merge completed.
$output['mergeSuccess'] = true;
}
} elseif ($file_chunkcount === 0) {
// if only one file, just rename it.
rename(
'upload/' . session_id() . '_0.tmp',
$file_name
);
$output['mergeSuccess'] = true;
}
if (isset($output['mergeSuccess']) && $output['mergeSuccess'] === true) {
$fileSize = (int) ($_POST['file_size'] ?? 0);
if ($fileSize < (20 * 1024 * 1024) && is_file($file_name)) {
// if file size less than xx MB
// allow to calculate md5 and sha1. otherwise skip it or it will be execution timeout error.
$output['md5file'] = md5_file($file_name);
$output['sha1file'] = sha1_file($file_name);
}
// do the task after uploaded complete here.
// ...
}
<?php
if (
isset($testFailConnection) &&
$testFailConnection === true &&
($chunkNumber === 3 || $chunkNumber === 6)
) {
if (!isset($_SESSION['retryforchunk3'])) {
$_SESSION['retryforchunk3'] = 0;
}
if ($_SESSION['retryforchunk3'] < 1) {
$_SESSION['retryforchunk3'] = ($_SESSION['retryforchunk3'] + 1);
http_response_code(500);
header('Content-Type: application/json');
echo json_encode($output);
exit();
} else {
$_SESSION['retryforchunk3'] = 0;
}
}
<?php
/**
* Recursively delete directory and contents.
*
* @link https://stackoverflow.com/a/3338133/128761 Original source code.
* @param string $dir
* @return void
*/
function rrmdir($dir)
{
if (is_dir($dir)) {
$objects = scandir($dir);
foreach ($objects as $object) {
if ($object != "." && $object != "..") {
if (is_dir($dir . DIRECTORY_SEPARATOR . $object) && !is_link($dir . "/" . $object))
rrmdir($dir . DIRECTORY_SEPARATOR . $object);
else
unlink($dir . DIRECTORY_SEPARATOR . $object);
}
}
rmdir($dir);
}
}
if (is_dir('upload')) {
rrmdir('upload');
echo 'Upload folder was deleted successfully.<br>' . PHP_EOL;
}
<?php
session_start();
$output = [];
$chunkNumber = (int) ($_GET['chunkNumber'] ?? 0);
$output['chunkNumber'] = $chunkNumber;
// test upload failure due to server or connection errors.
$testFailConnection = false;
include '_includes_upload-single-test-connection-error.php';
if (
isset($_SERVER['REQUEST_METHOD']) &&
strtolower($_SERVER['REQUEST_METHOD']) === 'post'
) {
// if method post.
$output['post_data'] = $_POST;// for debug
if (
isset($_FILES['file']['name'])
) {
// if there is file upload.
if (!is_dir('upload')) {
mkdir('upload');
}
if ($chunkNumber !== -1 && $chunkNumber >= 0) {
// temp file name for create uploaded chunk and merge files later.
$tempFilename = 'upload/' . session_id() . '_' . $chunkNumber . '.tmp';
// move uploaded file. you can use any php class to handle this but target file must be temporary for merge/rename later.
move_uploaded_file($_FILES['file']['tmp_name'], $tempFilename);
$output['uploadedFile'] = $tempFilename;
$output['result'] = 'success';
// don't do anything here because it maybe not yet completed upload the whole file (just chunk or part of a file).
// to do anything after all chunks were uploaded and merged complete, please look at **includes/upload-single-merge-chunk.php** file.
}
}// endif; file upload
if ($chunkNumber === -1) {
// if chunk number is -1 means merge all files.
$_SESSION['retryforchunk3'] = 0;// for test upload failure due to server or connection errors.
require '_includes_upload-single-merge-chunks.php';
}// endifl chunk number -1 (merge)
}// endif method post.
header('Content-Type: application/json');
echo json_encode($output);
function XHR(url, formData, progressCallback) {
return new Promise((resolve, reject) => {
let XHR = new XMLHttpRequest();
XHR.addEventListener('abort', (event) => {
reject({'response': '', 'status': (event.currentTarget ? event.currentTarget.status : ''), 'event': event});
});
XHR.addEventListener('error', (event) => {
reject({'response': '', 'status': (event.currentTarget ? event.currentTarget.status : ''), 'event': event});
});
XHR.addEventListener('timeout', (event) => {
reject({'response': '', 'status': (event.currentTarget ? event.currentTarget.status : ''), 'event': event});
});
XHR.addEventListener('load', (event) => {
let response = XHR.response;
let headers = XHR.getAllResponseHeaders();
let headerMap = {};
if (headers) {
let headersArray = headers.trim().split(/[\r\n]+/);
headersArray.forEach(function (line) {
let parts = line.split(': ');
let header = parts.shift();
let value = parts.join(': ');
headerMap[header] = value;
});
}
if (event.currentTarget && event.currentTarget.status >= 200 && event.currentTarget.status < 300) {
resolve({'response': response, 'status': event.currentTarget.status, 'event': event, 'headers': headerMap});
} else {
reject({'response': response, 'status': event.currentTarget.status, 'event': event, 'headers': headerMap});
}
});
XHR.upload.addEventListener('progress', progressCallback);
XHR.open('POST', url);
XHR.responseType = 'json';
XHR.send(formData);
});// end new Promise
}// XHR
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment