Created
April 18, 2021 09:22
-
-
Save semlinker/837211c039e6311e1e7629e5ee5f0a42 to your computer and use it in GitHub Desktop.
JavaScript 中实现大文件并行下载
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<!DOCTYPE html> | |
<html lang="zh-cn"> | |
<head> | |
<meta charset="UTF-8" /> | |
<meta http-equiv="X-UA-Compatible" content="IE=edge" /> | |
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> | |
<title>多线程下载示例</title> | |
<script src="multi-thread-download.js"></script> | |
</head> | |
<body> | |
<p>文件地址:<input type="text" id="fileUrl" value="" /></p> | |
<div> | |
<h3>多线程下载</h3> | |
<button onclick="multiThreadedDownload()">多线程下载</button> | |
</div> | |
<script> | |
function multiThreadedDownload() { | |
const url = document.querySelector("#fileUrl").value; | |
if (!url || !/https?/.test(url)) return; | |
console.log("多线程下载开始: " + +new Date()); | |
download({ | |
url, | |
chunkSize: 0.1 * 1024 * 1024, | |
poolLimit: 6, | |
}).then((buffers) => { | |
console.log("多线程下载结束: " + +new Date()); | |
saveAs({ buffers, name: "我的压缩包", mime: "application/zip" }); | |
}); | |
} | |
</script> | |
</body> | |
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function concatenate(arrays) { | |
if (!arrays.length) return null; | |
let totalLength = arrays.reduce((acc, value) => acc + value.length, 0); | |
let result = new Uint8Array(totalLength); | |
let length = 0; | |
for (let array of arrays) { | |
result.set(array, length); | |
length += array.length; | |
} | |
return result; | |
} | |
function getContentLength(url) { | |
return new Promise((resolve, reject) => { | |
let xhr = new XMLHttpRequest(); | |
xhr.open("HEAD", url); | |
xhr.send(); | |
xhr.onload = function () { | |
resolve( | |
// xhr.getResponseHeader("Accept-Ranges") === "bytes" && | |
~~xhr.getResponseHeader("Content-Length") | |
); | |
}; | |
xhr.onerror = reject; | |
}); | |
} | |
function getBinaryContent(url, start, end, i) { | |
return new Promise((resolve, reject) => { | |
try { | |
let xhr = new XMLHttpRequest(); | |
xhr.open("GET", url, true); | |
xhr.setRequestHeader("range", `bytes=${start}-${end}`); // 请求头上设置范围请求信息 | |
xhr.responseType = "arraybuffer"; // 设置返回的类型为arraybuffer | |
xhr.onload = function () { | |
resolve({ | |
index: i, // 文件块的索引 | |
buffer: xhr.response, // 范围请求对应的数据 | |
}); | |
}; | |
xhr.send(); | |
} catch (err) { | |
reject(new Error(err)); | |
} | |
}); | |
} | |
function saveAs({ name, buffers, mime = "application/octet-stream" }) { | |
const blob = new Blob([buffers], { type: mime }); | |
const blobUrl = URL.createObjectURL(blob); | |
const a = document.createElement("a"); | |
a.download = name || Math.random(); | |
a.href = blobUrl; | |
a.click(); | |
URL.revokeObjectURL(blob); | |
} | |
async function asyncPool(poolLimit, array, iteratorFn) { | |
const ret = []; // 存储所有的异步任务 | |
const executing = []; // 存储正在执行的异步任务 | |
for (const item of array) { | |
// 调用iteratorFn函数创建异步任务 | |
const p = Promise.resolve().then(() => iteratorFn(item, array)); | |
ret.push(p); // 保存新的异步任务 | |
// 当poolLimit值小于或等于总任务个数时,进行并发控制 | |
if (poolLimit <= array.length) { | |
// 当任务完成后,从正在执行的任务数组中移除已完成的任务 | |
const e = p.then(() => executing.splice(executing.indexOf(e), 1)); | |
executing.push(e); // 保存正在执行的异步任务 | |
if (executing.length >= poolLimit) { | |
await Promise.race(executing); // 等待较快的任务执行完成 | |
} | |
} | |
} | |
return Promise.all(ret); | |
} | |
async function download({ url, chunkSize, poolLimit = 1 }) { | |
const contentLength = await getContentLength(url); | |
const chunks = | |
typeof chunkSize === "number" ? Math.ceil(contentLength / chunkSize) : 1; | |
const results = await asyncPool( | |
poolLimit, | |
[...new Array(chunks).keys()], | |
(i) => { | |
let start = i * chunkSize; | |
let end = i + 1 == chunks ? contentLength - 1 : (i + 1) * chunkSize - 1; | |
return getBinaryContent(url, start, end, i); | |
} | |
); | |
const sortedBuffers = results | |
.map((item) => new Uint8Array(item.buffer)); | |
return concatenate(sortedBuffers); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment