crawlee.js

const { PlaywrightCrawler } = require("crawlee");
const { firefoxUserPrefs } = require("./firefox-user-prefs"); // this just some firefox setting in this repo https://github.com/yokoffing/BetterFox
const constNike = require("./const");
const currentDir = __dirname;
const path = require("path");
const { firefox } = require("playwright");
const { expect } = require("playwright/test");
const imageFolder = path.join(currentDir, "images");
const SimpleNodeLogger = require("simple-node-logger");
const fs = require("fs");
opts = {
  logFilePath: constNike.logFilePath,
  timestampFormat: constNike.logTimeStampFormat,
};
const log = SimpleNodeLogger.createSimpleLogger(opts);
const crawler = new PlaywrightCrawler({
  preNavigationHooks: [
    async ({ page, request }) => {
      await page.route("**/*", (route) => {
        if (
          constNike.RESOURCE_EXCLUSTIONS.includes(
            route.request().resourceType()
          )
        ) {
          route.abort();
          return;
        }
        route.continue();
        return;
      });
    },
  ],
  requestHandlerTimeoutSecs: 1000,
  // will run with a visible browser window.
  headless: false,
  // Let's limit our crawls to make our tests shorter and safer.
  maxRequestsPerCrawl: 50,
  launchContext: {
    // Set the Firefox browser to be used by the crawler.
    // If launcher option is not specified here,
    // default Chromium browser will be used.
    launcher: firefox,
    launchOptions: {
      slowMo: 2000,
      firefoxUserPrefs: firefoxUserPrefs,
    },
  },
});