Skip to content

Instantly share code, notes, and snippets.

@bittailor
Created October 8, 2018 10:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bittailor/20c9829900bdeb06e2dd628c578614cb to your computer and use it in GitHub Desktop.
Save bittailor/20c9829900bdeb06e2dd628c578614cb to your computer and use it in GitHub Desktop.
a simple and incomplete wget for node
const fs = require('fs');
const fetch = require('node-fetch');
const progress = require('progress-stream');
const crypto = require('crypto');
function wgetNext(url, progStrem, hash, tracking, retry) {
console.log(`bt-wgetNext length=${tracking.length} pipedLength=${tracking.pipedLength} retry=${retry}`);
const options = {
headers: {
Range: `bytes=${tracking.pipedLength}-${tracking.length}`
}
};
return fetch(url,options)
.then((res) => {
console.log('wgetNext fetch res ', res);
if (!res.ok) {
throw new fetch.FetchError(`request failed with ${res.status}`, 'request');
}
const srcStream = res.body;
srcStream.on('data', chunk => {
tracking.pipedLength = tracking.pipedLength + chunk.length;
hash.update(chunk);
});
srcStream.on('error', err => reject(err));
srcStream.pipe(progStrem,{ end: false });
srcStream.on('end', () => {
console.log(`srcStream ${retry} end length=${tracking.length} pipedLength=${tracking.pipedLength}`);
if(tracking.pipedLength >= tracking.length) {
progStrem.end();
return;
}
return wgetNext(url, progStrem, hash, tracking, retry + 1);
});
})
.catch((err) => {
console.log(`wgetNext catch`, err);
});
}
function wget(url, filename, onProgress) {
console.log(`bt-wget ${url} => ${filename}`);
return fetch(url)
.then((res) => {
if (!res.ok) {
throw new fetch.FetchError(`request failed with ${res.status}`, 'request');
}
const hash = crypto.createHash('sha256');
const length = res.headers.get('content-length');
const destStream = fs.createWriteStream(filename);
const srcStream = res.body;
const progStream = progress({time:100, length:length }, onProgress);
const tracking = {length, pipedLength:0 };
return new Promise((resolve, reject) => {
srcStream.on('data', chunk => {
tracking.pipedLength = tracking.pipedLength + chunk.length;
hash.update(chunk);
});
srcStream.on('error', err => reject(err));
destStream.on('error', err => reject(err));
destStream.on('finish', () => {
console.log(`destStream finish length=${tracking.length} pipedLength=${tracking.pipedLength}`);
resolve({ filename, hash: hash.digest('hex') })
});
srcStream.on('end', () => {
console.log(`srcStream end length=${tracking.length} pipedLength=${tracking.pipedLength}`);
if(tracking.pipedLength >= tracking.length) {
progStream.end();
return;
}
wgetNext(url, progStream, hash, tracking, 1)
.catch((err) => {
reject(err);
})
});
srcStream.pipe(progStream,{ end: false }).pipe(destStream);
})
});
}
module.exports = wget;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment