Skip to content

Instantly share code, notes, and snippets.

@progress44
Created September 2, 2020 20:20
Show Gist options
  • Save progress44/48c10d8acf20bddb641b9f2fbe3c7c9c to your computer and use it in GitHub Desktop.
Save progress44/48c10d8acf20bddb641b9f2fbe3c7c9c to your computer and use it in GitHub Desktop.
Downloading files test script (plain js)
const url = require('url');
const path = require('path');
const https = require('https');
const fs = require('fs');
const dl = fs.existsSync('./downloaded.json') ? require('./downloaded.json') : [];
const dirs = p => fs.readdirSync(p).filter(f => fs.statSync(path.join(p, f)).isDirectory());
const files = p => fs.readdirSync(p).filter(f => !fs.statSync(path.join(p, f)).isDirectory());
const mediaDirs = dirs(".");
var totals = {};
var downloaded = dl;
if (dl.lengh == 0) {
downloaded = dirs(".")
.map(dir => files(`./${dir}`).map(file => `./${dir}/${file}`))
.reduce((acc, val) => acc.concat(val), []);
fs.writeFileSync("./downloaded.json", JSON.stringify(downloaded));
}
const download = async (url, dest, dir) => {
return new Promise((resolve, reject) => {
if (fs.existsSync(dest) && fs.statSync(dest).size > 0) {
totals[dir]--;
return reject("File exists");
}
let file = fs.createWriteStream(dest);
let request = https.get(url, (response) => {
response.pipe(file);
file.on('finish', () => {
totals[dir]--;
file.close(() => {
console.log(`Downloaded ${dest}`);
resolve();
});
});
});
request.on('error', (err) => {
fs.unlinkSync(dest)
console.log(`Deleted file ${dest}`);
reject(`${err.message} for ${dest}`);
});
});
};
const filter = dir => {
const json = require(`./${dir}/${dir}.json`)
.filter(_ => !!_)
.filter(item => {
let file = `./${dir}/${path.basename(url.parse(item).pathname)}`;
if (downloaded.indexOf(file) >= 0) {
return false;
}
return !(fs.existsSync(file) && fs.statSync(file).size > 0 );
});
totals[dir] = json.length;
console.log(`Total ${dir}: ${json.length}`);
return json;
}
const loop = async () => {
mediaDirs.forEach(dir => {
filter(dir)
.forEach(async item => {
try {
await download(
item,
`./${dir}/${path.basename(url.parse(item).pathname)}`,
dir
);
} catch (e) {
console.log(e);
}
});
});
};
loop();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment