Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
Puppeteer Crawler Example
var Walker = require('puppeteer-walker')
var fs = require('fs')
var walker = Walker()
fs.writeFile('output.csv', 'URL, Title, H2\r\n', 'utf8', function (err) {
console.log(`Header written`)
walker.on('end', () => console.log('finished walking'))
walker.on('error', (err) => console.log('error', err))
walker.on('page', async (page) => {
var url = await page.url()
var title = await page.title()
try {
var h2 = await page.evaluate(() => document.querySelector('h2').innerText);
fs.appendFile('output.csv', `${url}, ${title}, ${h2}\r\n`, 'utf8', function (err) {
`url: ${url}, title: ${title}, h2: ${h2}`)
} catch(e) {
fs.appendFile('output.csv', `${url}, ${title}, ERROR\r\n`, 'utf8', function (err) {
`url: ${url}, title: ${title}, h2: ERROR`)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment