Skip to content

Instantly share code, notes, and snippets.

@m7v
Created July 15, 2019 09:29
Show Gist options
  • Save m7v/3863913bbc4034f72689606ce6e5c6de to your computer and use it in GitHub Desktop.
Save m7v/3863913bbc4034f72689606ce6e5c6de to your computer and use it in GitHub Desktop.
Parse https://warhammerunderworlds.com for saving all cards from this game.
const https = require("https");
const axios = require("axios");
const fs = require("fs");
const os = require("os");
const cluster = require("cluster");
const STEP = 20;
const saveFailedImg = url => {
failedImg.push(url);
};
const downloadImage = (url, name) => {
const filePath = `${__dirname}/images/${name}.png`;
const file = fs.createWriteStream(filePath);
return new Promise(resolve => {
https.get(url, response => {
if (response.statusCode === 200) {
response.pipe(file).on("finish", () => {
console.log("File downloaded: ", filePath);
resolve();
});
} else {
saveFailedImg(filePath);
resolve();
}
});
});
};
const failedImg = [];
const url = `https://warhammerunderworlds.com/wp-json/wp/v2/cards/?ver=13&per_page=1000`;
(async () => {
if (cluster.isMaster) {
cluster.on("message", (worker, msg) => console.log(msg));
let livingChilds = 0;
const queue = await axios
.get(url)
.then(res =>
res.data.map(card => ({
id: card.acf.card_number,
url: encodeURI(card.acf.card_image.url),
name: card.slug
}))
)
.then(images => images.sort((prev, next) => prev.id - next.id))
.catch(e => console.log(e));
const interval = setInterval(() => {
if (queue.length && livingChilds <= 10) {
livingChilds++;
const value = queue.pop();
cluster.fork({ value: JSON.stringify(value) }).on("disconnect", () => {
livingChilds--;
});
}
if (!queue.length) {
clearInterval(interval);
}
}, 10);
} else {
const value = JSON.parse(process.env.value);
downloadImage(value.url, value.name)
.then(res => process.exit(0))
.catch(err => {
console.log(err);
exit(1);
});
}
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment