parserDota.js

static fetchBuff163Dota() {
    return new Promise(async (resolve, reject) => {
      try {
        const browser = await puppeteer.launch();
        const page = await browser.newPage();
        await page.setViewport({
          width: 1920,
          height: 1080,
        });
        await page.goto(
          "https://buff.163.com/market/?game=dota2#tab=selling&page_num=1"
        );

        await page.waitForSelector(
          "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li:nth-child(13) > a"
        );
        await page.click(
          "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li:nth-child(13) > a"
        );

        await page.waitForSelector("#j_popup_login > a");
        await page.click("#j_popup_login > a");

        await page.waitForSelector(
          "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li.active > span"
        );

        let pagesToScrape = await page.evaluate(() => {
          return document.querySelector(
            "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li.active > span"
          ).innerText;
        });
        console.log(`total pages  -> ${pagesToScrape}`); // ---____________________________________---
        if (!pagesToScrape) {
          pagesToScrape = 1;
        }

        let cards = [];
        while (pagesToScrape >= 1) {
          await page.screenshot({
            path: `./controllers/data/screen${pagesToScrape}.png`,
          });
          await page.waitForSelector("#j_list_card > ul.card_dota2 > li");
          await page.waitForSelector(
            "#j_list_card > ul.card_dota2 > li > h3 > a"
          );
          await page.waitForSelector(
            "#j_list_card > ul.card_dota2 > li > p > strong"
          );
          await page.waitForSelector("#j_list_card > ul.card_dota2 > li > a");
          await page.waitForSelector(
            "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li:nth-child(1) > a"
          );

          const newCards = await page.evaluate(() => {
            let results = [];
            let items = document.querySelectorAll("#j_list_card > ul > li");
            items.forEach((item) => {
              results.push({
                name: item.querySelector("h3 > a").innerText,
                price: item
                  .querySelector("p > strong")
                  .innerText.replace(/[^0-9.]/g, "")
                  .trim(),
                link: `https://buff.163.com${item
                  .querySelector("a")
                  .getAttribute("href")}`,
              });
            });
            return results;
          });

          cards = [...cards, ...newCards];

          if (pagesToScrape > 1) {
            await page.click(
              "#j_market_card > div.pager.card-pager.light-theme.simple-pagination > ul > li:nth-child(1) > a"
            );
          }
          pagesToScrape--;
        }

        browser.close();
        return resolve(cards);
      } catch (e) {
        return reject(e);
      }
    });
  }