test.js

const puppeteer = require('puppeteer');

async function visit(breadcrumb, allowed, denied, visited, browser, url) {
    if (visited.has(url)) {
        console.log(`already visited ${url}`);
        return;
    }
    if (denied.some(regex => regex.test(url))) {
        console.log(`denied ${url}`);
        return;
    }
    if (!allowed.some(regex => regex.test(url))) {
        console.log(`not allowed ${url}`);
        return;
    }

    visited.add(url);
    let page = await browser.newPage();
    let response = await page.goto(url);

    console.log(`status ${response.status} for ${breadcrumb}`);

    if (!response.ok) {
        return await page.close();
    }

    if (!response.headers['content-type'].startsWith('text/html')) {
        return await page.close();
    }

    //await page.waitForNavigation({waitUntil: 'networkidle2'});
    //await page.waitForNavigation();
    //await page.waitForSelector('a[href]');
    await page.waitFor(10000);
    let links = await page.$$eval('a[href]', async links => links.map(link => link.href));
    await links.forEach(async link => {
        await page.waitFor(2000);
        await visit(`${breadcrumb} > ${link}`, allowed, denied, visited, browser, link);
    });
    await page.close();
}

async function authenticate(base, browser) {
    let page = await browser.newPage();
    await page.goto(base);
    await page.$('[name=_username]').then(el => el.type('admin'));
    await page.$('[name=_password]').then(el => el.type('admin'));
    await page.$('[name=_submit]').then(el => el.click());
}

(async (allowed, denied, base, visited, DEBUG) => {
    let browser = await puppeteer.launch({
        args: ['--no-sandbox', '--disable-setuid-sandbox'],
        headless: !DEBUG,
        slowMo: DEBUG ? 25: 0,
    });

    await authenticate(base, browser);
    await visit(base, allowed.map(regex => new RegExp(regex)), denied.map(regex => new RegExp(regex)), visited, browser, base);

    if (!DEBUG) {
        browser.close();
    }
})(
    ['^http://127\.0\.0\.1:8080'],
    ['user/logout$', '/_profiler'],
    process.argv[2],
    new Set(),
    process.env.DEBUG === '1'
);