Stocktwits Message Scraper using Headless Browser

stocktwits-message-scraper.js

const puppeteer = require('puppeteer');

const debug = process.env.DEBUG == 'true';

const url = 'https://stocktwits.com/mrinvestorpro';

const main = async () => {
  const browser = await puppeteer.launch({ headless: !debug });
  try {
    console.log('Launching page');
    const page = await browser.newPage();
    await page.goto(url, {
      timeout: (60 * 1000),
      waitUntil: ['load', 'networkidle2']
    });

    console.log('Waiting for Real-Time button');
    await page.waitForXPath("//span[text()='Real-Time']", );

    console.log('Clicking Real-Time button');
    await (await page.$x("//span[text()='Real-Time']"))[0].click();

    while(true) {
      let result = await page.$eval("//span[text()='1 New Post']", (el) => {
        el.click();
      });

      let result = await page.$eval("div.infinite-scroll-component article", (el) => {
        let url = el.querySelector("a[href*='/mrinvestorpro/message']").href;
        let msg = el.querySelector('div > div:nth-of-type(2) > div:nth-of-type(2)').innerHTML;
        msg = msg.replace(/(<([^>]+)>)/gi, "");
        return [url, msg]
      });

      console.log(result);
      await page.waitFor(2000);
    }
  } finally {
    await browser.close();
  }
};

main();