Skip to content

Instantly share code, notes, and snippets.

@r-shafi
Created December 28, 2022 06:01
Show Gist options
  • Save r-shafi/0dcd097a19ddea17ff7c4ba0eb0d816d to your computer and use it in GitHub Desktop.
Save r-shafi/0dcd097a19ddea17ff7c4ba0eb0d816d to your computer and use it in GitHub Desktop.
A working modal data scraper. This uses Puppeteer to scrape user data from a forum. Error handling included.
const puppeteer = require('puppeteer');
async function scrape() {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://forum.freecodecamp.org/');
const avatars = await page.$$('.topic-poster img.avatar');
const names = [];
for (const avatar of avatars) {
try {
await page.evaluate((element) => element.click(), avatar);
await page.waitForSelector('.user-card.show', { timeout: 5000 });
let userNameElement = null;
let fullNameElement = null;
try {
userNameElement = await page.waitForSelector(
'.user-card.show .name-username-wrapper',
{
timeout: 2000,
}
);
} catch (error) {
console.log(error.message);
}
try {
fullNameElement = await page.waitForSelector(
'.user-card.show .full-name',
{
timeout: 2000,
}
);
} catch (error) {
console.log(error.message);
}
let userName = null;
let fullName = null;
if (userNameElement) {
userName = await page.evaluate(
(element) => element.textContent.trim(),
userNameElement
);
}
if (fullNameElement) {
fullName = await page.evaluate(
(element) => element.textContent.trim(),
fullNameElement
);
}
names.push({
userName,
fullName,
});
await new Promise((resolve) => setTimeout(resolve, 500));
} catch (error) {
console.error(error.message);
continue;
}
}
await browser.close();
console.log(names);
}
scrape();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment