Skip to content

Instantly share code, notes, and snippets.

@thewh1teagle
Created May 3, 2024 17:43
Show Gist options
  • Save thewh1teagle/3e9820854c2cfcc683bcdfff4cf04f79 to your computer and use it in GitHub Desktop.
Save thewh1teagle/3e9820854c2cfcc683bcdfff4cf04f79 to your computer and use it in GitHub Desktop.
Concurrent NodeJS downloader
import fs from 'fs'
async function getMeta(url) {
let resp = await fetch(url, { method: 'HEAD' });
const size = parseInt(resp.headers.get('Content-Length'));
const contentDisposition = resp.headers.get('Content-Disposition')
const filename = contentDisposition.match(/; filename=(.+)/)[1] ?? 'Unnamed'
return {size, filename};
}
async function createRanges(url, chunkSize) {
const meta = await getMeta(url);
const ranges = [];
for (let start = 0; start < meta.size; start += chunkSize) {
const end = Math.min(start + chunkSize - 1, meta.size - 1);
ranges.push([start, end]);
}
return { meta, ranges };
}
async function fetchRange(url, start, end) {
const headers = { 'Range': `bytes=${start}-${end}` };
const resp = await fetch(url, { headers });
if (!resp.ok) {
throw new Error(`${resp.status}: ${await resp.text()}`);
}
const data = await resp.arrayBuffer();
return { start, end, data };
}
function toMB(bytes) {
return bytes / 1024 / 1024
}
async function download(url, maxWorkers = 15, chunkSize = 1024 * 1024 * 2, filename = null) {
let { meta, ranges } = await createRanges(url, chunkSize);
if (filename != null) {
meta.filename = filename
}
const writer = fs.createWriteStream(meta.filename)
let read = 0;
let completedChunks = 0;
for (let i = 0; i < ranges.length; i += maxWorkers) {
const slice = ranges.slice(i, i + maxWorkers);
const promises = slice.map((chunk) => fetchRange(url, chunk[0], chunk[1]));
let startTime = Date.now();
const chunks = await Promise.all(promises);
const elapsedTime = (Date.now() - startTime) / 1000; // in seconds
chunks.sort((a, b) => a.start - b.start);
let phaseSize = 0
for (const chunk of chunks) {
phaseSize += chunk.data.byteLength
}
read += phaseSize
completedChunks += slice.length;
for (const chunk of chunks) {
writer.write(chunk.data)
}
const progress = (completedChunks / ranges.length) * 100;
const speedMbps = toMB(phaseSize) / elapsedTime;
console.log(`${toMB(read).toFixed(0)}MB | ${progress.toFixed(1)}% | ${speedMbps.toFixed(1)}mb/s`);
}
return {meta}
}
await download("https://github.com/thewh1teagle/vibe/releases/download/v1.0.4/vibe_1.0.4_amd64.deb")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment