Created
July 15, 2019 09:29
-
-
Save m7v/3863913bbc4034f72689606ce6e5c6de to your computer and use it in GitHub Desktop.
Parse https://warhammerunderworlds.com for saving all cards from this game.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const https = require("https"); | |
const axios = require("axios"); | |
const fs = require("fs"); | |
const os = require("os"); | |
const cluster = require("cluster"); | |
const STEP = 20; | |
const saveFailedImg = url => { | |
failedImg.push(url); | |
}; | |
const downloadImage = (url, name) => { | |
const filePath = `${__dirname}/images/${name}.png`; | |
const file = fs.createWriteStream(filePath); | |
return new Promise(resolve => { | |
https.get(url, response => { | |
if (response.statusCode === 200) { | |
response.pipe(file).on("finish", () => { | |
console.log("File downloaded: ", filePath); | |
resolve(); | |
}); | |
} else { | |
saveFailedImg(filePath); | |
resolve(); | |
} | |
}); | |
}); | |
}; | |
const failedImg = []; | |
const url = `https://warhammerunderworlds.com/wp-json/wp/v2/cards/?ver=13&per_page=1000`; | |
(async () => { | |
if (cluster.isMaster) { | |
cluster.on("message", (worker, msg) => console.log(msg)); | |
let livingChilds = 0; | |
const queue = await axios | |
.get(url) | |
.then(res => | |
res.data.map(card => ({ | |
id: card.acf.card_number, | |
url: encodeURI(card.acf.card_image.url), | |
name: card.slug | |
})) | |
) | |
.then(images => images.sort((prev, next) => prev.id - next.id)) | |
.catch(e => console.log(e)); | |
const interval = setInterval(() => { | |
if (queue.length && livingChilds <= 10) { | |
livingChilds++; | |
const value = queue.pop(); | |
cluster.fork({ value: JSON.stringify(value) }).on("disconnect", () => { | |
livingChilds--; | |
}); | |
} | |
if (!queue.length) { | |
clearInterval(interval); | |
} | |
}, 10); | |
} else { | |
const value = JSON.parse(process.env.value); | |
downloadImage(value.url, value.name) | |
.then(res => process.exit(0)) | |
.catch(err => { | |
console.log(err); | |
exit(1); | |
}); | |
} | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment