NodeJS Headless Chrome Crawler Traffic Interceptor

crawler.js

const HCCrawler = require('headless-chrome-crawler');
const url = require('url');
const args = process.argv.slice(2);

var requests = [];

if(args[0] === undefined || args[1] === undefined){
    console.log('node crawler.js <url> <depth>');
    process.exit();
}

(async () => {
    const crawler = await HCCrawler.launch({
        maxDepth: args[1],
        customCrawl: async (page, crawl) => {
            // You can access the page object before requests
            await page.setRequestInterception(true);
            page.on('request', request => {

                requests.push(request.url());
                // console.log(request.url());
                request.continue();

                // if (['image', 'stylesheet', 'media', 'font'].includes(request.resourceType())) {
                //  request.abort();
                // }else{
                //  request.continue();
                // }
            });
            return await crawl();
        },
        onSuccess: (result => {
            // console.log(result.options.url);
        }),
    });
    await crawler.queue({
        url: args[0],
        allowedDomains: [url.parse(args[0]).hostname],
    });
    await crawler.onIdle();
    await crawler.close();

    requests = [ ...new Set(requests) ]
    requests.forEach( function(value) {
        console.log(value);
    });
})();

Installation

yarn add headless-chrome-crawler

Usage

node crawler.js https://example.com/ 2