Skip to content

Instantly share code, notes, and snippets.

@semlinker
Created April 18, 2021 09:22
Show Gist options
  • Star 31 You must be signed in to star a gist
  • Fork 7 You must be signed in to fork a gist
  • Save semlinker/837211c039e6311e1e7629e5ee5f0a42 to your computer and use it in GitHub Desktop.
Save semlinker/837211c039e6311e1e7629e5ee5f0a42 to your computer and use it in GitHub Desktop.
JavaScript 中实现大文件并行下载
<!DOCTYPE html>
<html lang="zh-cn">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>多线程下载示例</title>
<script src="multi-thread-download.js"></script>
</head>
<body>
<p>文件地址:<input type="text" id="fileUrl" value="" /></p>
<div>
<h3>多线程下载</h3>
<button onclick="multiThreadedDownload()">多线程下载</button>
</div>
<script>
function multiThreadedDownload() {
const url = document.querySelector("#fileUrl").value;
if (!url || !/https?/.test(url)) return;
console.log("多线程下载开始: " + +new Date());
download({
url,
chunkSize: 0.1 * 1024 * 1024,
poolLimit: 6,
}).then((buffers) => {
console.log("多线程下载结束: " + +new Date());
saveAs({ buffers, name: "我的压缩包", mime: "application/zip" });
});
}
</script>
</body>
</html>
function concatenate(arrays) {
if (!arrays.length) return null;
let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
let result = new Uint8Array(totalLength);
let length = 0;
for (let array of arrays) {
result.set(array, length);
length += array.length;
}
return result;
}
function getContentLength(url) {
return new Promise((resolve, reject) => {
let xhr = new XMLHttpRequest();
xhr.open("HEAD", url);
xhr.send();
xhr.onload = function () {
resolve(
// xhr.getResponseHeader("Accept-Ranges") === "bytes" &&
~~xhr.getResponseHeader("Content-Length")
);
};
xhr.onerror = reject;
});
}
function getBinaryContent(url, start, end, i) {
return new Promise((resolve, reject) => {
try {
let xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
xhr.setRequestHeader("range", `bytes=${start}-${end}`); // 请求头上设置范围请求信息
xhr.responseType = "arraybuffer"; // 设置返回的类型为arraybuffer
xhr.onload = function () {
resolve({
index: i, // 文件块的索引
buffer: xhr.response, // 范围请求对应的数据
});
};
xhr.send();
} catch (err) {
reject(new Error(err));
}
});
}
function saveAs({ name, buffers, mime = "application/octet-stream" }) {
const blob = new Blob([buffers], { type: mime });
const blobUrl = URL.createObjectURL(blob);
const a = document.createElement("a");
a.download = name || Math.random();
a.href = blobUrl;
a.click();
URL.revokeObjectURL(blob);
}
async function asyncPool(poolLimit, array, iteratorFn) {
const ret = []; // 存储所有的异步任务
const executing = []; // 存储正在执行的异步任务
for (const item of array) {
// 调用iteratorFn函数创建异步任务
const p = Promise.resolve().then(() => iteratorFn(item, array));
ret.push(p); // 保存新的异步任务
// 当poolLimit值小于或等于总任务个数时,进行并发控制
if (poolLimit <= array.length) {
// 当任务完成后,从正在执行的任务数组中移除已完成的任务
const e = p.then(() => executing.splice(executing.indexOf(e), 1));
executing.push(e); // 保存正在执行的异步任务
if (executing.length >= poolLimit) {
await Promise.race(executing); // 等待较快的任务执行完成
}
}
}
return Promise.all(ret);
}
async function download({ url, chunkSize, poolLimit = 1 }) {
const contentLength = await getContentLength(url);
const chunks =
typeof chunkSize === "number" ? Math.ceil(contentLength / chunkSize) : 1;
const results = await asyncPool(
poolLimit,
[...new Array(chunks).keys()],
(i) => {
let start = i * chunkSize;
let end = i + 1 == chunks ? contentLength - 1 : (i + 1) * chunkSize - 1;
return getBinaryContent(url, start, end, i);
}
);
const sortedBuffers = results
.map((item) => new Uint8Array(item.buffer));
return concatenate(sortedBuffers);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment