Skip to content

Instantly share code, notes, and snippets.

@supernovaplus
Last active April 17, 2020 15:26
Show Gist options
  • Save supernovaplus/052b1d6341bf0cfa212a005b6bdbe60c to your computer and use it in GitHub Desktop.
Save supernovaplus/052b1d6341bf0cfa212a005b6bdbe60c to your computer and use it in GitHub Desktop.
downloading file with node js
const fs = require('fs')
const fetch = require("node-fetch")
const path = "C:\\DOWNLOADS\\";
const urlArray = [
//["url","fileName"]
//"url"
//["url","filename"],"url",["url","filename"]
"http://somesite.com/somevideo.mp4",
["http://somesite.com/somevideo2.mp4","renamedvideo.mp4"],
["http://google.com","google.html"],
];
//## Downloading the files
//## either one file at the time
// startDownloading(urlArray, 1);
//## or X files at a time
startDownloading(urlArray, 4);
async function startDownloading(array, count = 1){
const splitLinksArray = [];
array.forEach((list,i)=>{
const array_index = i % count;
if( !splitLinksArray[array_index] ) splitLinksArray[array_index] = [];
splitLinksArray[array_index].push(list)
})
splitLinksArray.forEach((list,i)=>{
downloadFromArray(list, i+1);
})
}
async function downloadFromArray(array, index){
for (let i = 0; i < array.length; i++) {
await downloadFile( array[i], `${index}/${(i+1)}` );
};
}
async function downloadFile(url, index){
return new Promise(async (resolve, reject) => {
let fileName = "";
if(typeof url === "string"){
fileName = url.match(/[^\/]+(?=\/$|$)/gi)[0];
if(!/.mp4|.mp3|.jpg|.png|.gif|.gifv/gi.test(fileName)) fileName = url.replace(/[&\/\\#,+()$~%.'":*?<>{}]/g,'_') + ".html";
}else{
fileName = url[1];
url = url[0];
}
if(fs.existsSync(path + fileName)) {
console.log(`#${index}: ${fileName} => File already exists, skipping...`);
resolve();
}
console.log(`#${index}: ${fileName} => DOWNLOADING...`);
fetch(url).then(res => {
const writeStream = fs.createWriteStream(path + fileName);
res.body.pipe(writeStream);
writeStream.on('close', () => {
console.log(`#${index}: ${fileName} => File downloaded.`);
resolve();
});
writeStream.on('error', (err)=> {
console.log(`#${index}: ${fileName} => File system error => ${err}`);
resolve();
});
}).catch(err=>{
console.log(`#${index}: ${fileName} => URL error => ${err}`);
resolve();
});
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment