Instantly share code, notes, and snippets.

Embed
What would you like to do?
Puppeteer Crawler Example
var Walker = require('puppeteer-walker')
var fs = require('fs')
var walker = Walker()
fs.writeFile('output.csv', 'URL, Title, H2\r\n', 'utf8', function (err) {
console.log(`Header written`)
})
walker.on('end', () => console.log('finished walking'))
walker.on('error', (err) => console.log('error', err))
walker.on('page', async (page) => {
var url = await page.url()
var title = await page.title()
try {
var h2 = await page.evaluate(() => document.querySelector('h2').innerText);
fs.appendFile('output.csv', `${url}, ${title}, ${h2}\r\n`, 'utf8', function (err) {
console.log(
`url: ${url}, title: ${title}, h2: ${h2}`)
})
} catch(e) {
fs.appendFile('output.csv', `${url}, ${title}, ERROR\r\n`, 'utf8', function (err) {
console.log(
`url: ${url}, title: ${title}, h2: ERROR`)
})
}
})
walker.walk('http://jsseo.expert/')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment