puppeteer_scrapping

var http = require('http');
const puppeteer = require("puppeteer");
var url = require('url');

const PORT = 3000;
let browser;
let pages = {} //store object if you want to store each page url

console.log(`Running at Port : ${PORT}`)

//create a server object:
http.createServer(async(req, res) => {
    if (req.method === "GET") {
        var url = req.url.replace("/", "");
        var output = await screenshot(url);
        res.writeHead(200, {"Content-Type": "text/html"});
        res.end(`URL : ${url}  File : ${output} `);
    }
}).listen(PORT); //the server object listens on port 8080


var start = async() => {
    browser = await puppeteer.launch({
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
        ]
    });
}

var screenshot = async(url = "") => {
    var hostName = getHostname(url);
    try {
        if (!browser) {
            await start();
        }
        console.log(Object.keys(pages))
        if(hostName) {
            console.log(`URL : ${url} HostName : ${hostName}`)
            let page;
            //reuse page
            if(pages[hostName]) {
                page = pages[hostName]
            } else {
                page = await browser.newPage();
            }
            await page.goto(url, { waitUntil: 'networkidle0', timeout: 120000});
            var filename = new Date().getTime() + ".png";
            await page.screenshot({path: filename});
            //await page.close();
            pages[hostName] = page;
            return filename;
        }
        return "";
    } catch (err) {
        console.log(err)
        browser.close();
        browser = null;
        await start()
    }
}

var getHostname = (url) => {
    try {
        const myURL = new URL(url);
        return myURL.hostname ? myURL.hostname : "";
    } catch (err) {

        return "";
    }
}