Created
October 8, 2018 10:01
-
-
Save bittailor/20c9829900bdeb06e2dd628c578614cb to your computer and use it in GitHub Desktop.
a simple and incomplete wget for node
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const fetch = require('node-fetch'); | |
const progress = require('progress-stream'); | |
const crypto = require('crypto'); | |
function wgetNext(url, progStrem, hash, tracking, retry) { | |
console.log(`bt-wgetNext length=${tracking.length} pipedLength=${tracking.pipedLength} retry=${retry}`); | |
const options = { | |
headers: { | |
Range: `bytes=${tracking.pipedLength}-${tracking.length}` | |
} | |
}; | |
return fetch(url,options) | |
.then((res) => { | |
console.log('wgetNext fetch res ', res); | |
if (!res.ok) { | |
throw new fetch.FetchError(`request failed with ${res.status}`, 'request'); | |
} | |
const srcStream = res.body; | |
srcStream.on('data', chunk => { | |
tracking.pipedLength = tracking.pipedLength + chunk.length; | |
hash.update(chunk); | |
}); | |
srcStream.on('error', err => reject(err)); | |
srcStream.pipe(progStrem,{ end: false }); | |
srcStream.on('end', () => { | |
console.log(`srcStream ${retry} end length=${tracking.length} pipedLength=${tracking.pipedLength}`); | |
if(tracking.pipedLength >= tracking.length) { | |
progStrem.end(); | |
return; | |
} | |
return wgetNext(url, progStrem, hash, tracking, retry + 1); | |
}); | |
}) | |
.catch((err) => { | |
console.log(`wgetNext catch`, err); | |
}); | |
} | |
function wget(url, filename, onProgress) { | |
console.log(`bt-wget ${url} => ${filename}`); | |
return fetch(url) | |
.then((res) => { | |
if (!res.ok) { | |
throw new fetch.FetchError(`request failed with ${res.status}`, 'request'); | |
} | |
const hash = crypto.createHash('sha256'); | |
const length = res.headers.get('content-length'); | |
const destStream = fs.createWriteStream(filename); | |
const srcStream = res.body; | |
const progStream = progress({time:100, length:length }, onProgress); | |
const tracking = {length, pipedLength:0 }; | |
return new Promise((resolve, reject) => { | |
srcStream.on('data', chunk => { | |
tracking.pipedLength = tracking.pipedLength + chunk.length; | |
hash.update(chunk); | |
}); | |
srcStream.on('error', err => reject(err)); | |
destStream.on('error', err => reject(err)); | |
destStream.on('finish', () => { | |
console.log(`destStream finish length=${tracking.length} pipedLength=${tracking.pipedLength}`); | |
resolve({ filename, hash: hash.digest('hex') }) | |
}); | |
srcStream.on('end', () => { | |
console.log(`srcStream end length=${tracking.length} pipedLength=${tracking.pipedLength}`); | |
if(tracking.pipedLength >= tracking.length) { | |
progStream.end(); | |
return; | |
} | |
wgetNext(url, progStream, hash, tracking, 1) | |
.catch((err) => { | |
reject(err); | |
}) | |
}); | |
srcStream.pipe(progStream,{ end: false }).pipe(destStream); | |
}) | |
}); | |
} | |
module.exports = wget; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment