Super Web Scraper - Puppeteer Script

sample.js

// Available functions :
//
// page : https://pptr.dev/#?product=Puppeteer&version=v8.0.0&show=api-class-page
// Custom
// scrollToBottom 
// scrollToSelector

await scrollToBottom(page);
await page.waitForSelector('iframe#dsq-app2953')
await scrollToBottom(page);

const elementHandle = await page.$('div#foo iframe');
const frame = await elementHandle.contentFrame();

await page.evaluate(()=>{ 
    document.documentElement.innerHTML = frame
})

// if you need to wait for a fixed number of ms
await page.waitFor(3000);

// If the case is like a login form and you need to wait for next page after submitting form, try use this

await Promise.all([
  page.click('#login_button'),
  page.waitForNavigation({ waitUntil: 'networkidle0' })
]);

// But if you need to wait an ajax request to finish after clicking the button, maybe this can help. in my case it works fine

await Promise.all([
  page.click('#refresh_table'),
  page.waitForResponse(response => response.status() === 200)
]);

// There is some method for response for checking: response.url() response.status() response.headers(), etc

// Don't forget to return whatever you need 
return await page.content()