Parse store.line for saving images.

parserStoreLine.js

const request = require("request");
const https = require("https");
const fs = require("fs");
const os = require("os");
const cluster = require("cluster");
const jsdom = require("jsdom");

const { JSDOM } = jsdom;

const regExpr = /(https:.*;compress=true)/g;

const failedImg = [];

const urls = [
    `https://store.line.me/stickershop/product/1050364`,
    `https://store.line.me/stickershop/product/1188140`,
    `https://store.line.me/stickershop/product/1169792`,
    `https://store.line.me/stickershop/product/1100085`,
    `https://store.line.me/stickershop/product/1078340`,
    `https://store.line.me/stickershop/product/1120211`,
    `https://store.line.me/stickershop/product/4287693`,
]

const STEP = 20;

const saveFailedImg = url => {
    failedImg.push(url);
};

const downloadImage = (url, name) => {
    const filePath = `${__dirname}/images/${name}.png`;
    const file = fs.createWriteStream(filePath);
    return new Promise(resolve => {
        https.get(url, response => {
            if (response.statusCode === 200) {
                response.pipe(file).on("finish", () => {
                    console.log("File downloaded: ", filePath);
                    resolve();
                });
            } else {
                saveFailedImg(filePath);
                resolve();
            }
        });
    });
};

const parseImages = (html, imgList) => {
    const dom = new JSDOM(html);

    const list = dom.window.document.querySelectorAll('.FnCustomBase');
    list.forEach((item) => {
        const url = item.outerHTML;
        const clearedUrl = url
            .split('<span class="mdCMN09Image FnCustomBase" style="background-image:url(')[1]
            .split(';compress=true);"></span>')[0];
        if (clearedUrl) {
            imgList.push(clearedUrl);
        } else {
            console.log('url', url);
        }
    });

    return imgList;
};

const extractImageUrl = (url) => {
    let imgList = [];
    return new Promise((fullfil) => {
        request(url, (error, _, body) => {
            if (error) {
                console.log(error);
                return;
            }
            const promises = [];

            imgList = parseImages(body, imgList);

            Promise
                .all(promises)
                .then(() => {
                    console.log(`Total items ${imgList.length}`);
                })
                .then(() => fullfil({
                    imgList,
                }));
        });
    });
}

const promises = urls.map(url => extractImageUrl(url));
Promise.all(promises).then((allUrlSets) => {
    const urls = allUrlSets.reduce((arr, urlSet) => arr.concat(urlSet.imgList), []);

    Promise
        .all(urls.map((url, ind) => {
            downloadImage(url, ind);
        }))
        .then(() => console.log('file downloaded', failedImg.length));
})