-
-
Save ConradWeiser/91118a7488d8323faa2a977b247abd98 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
```ts | |
import { Browser } from "puppeteer" | |
const puppeteer = require('puppeteer-extra') | |
const stealthPlugin = require('puppeteer-extra-plugin-stealth') | |
puppeteer.use(stealthPlugin()) | |
export class RequestSystem { | |
browser: Browser | undefined | |
constructor() { | |
this.initializeBrowser() | |
} | |
private async initializeBrowser() { | |
// Create the browser. Note that if this is the first time being launched, it will have to download a lot of files. It may take a moment to init | |
this.browser = await puppeteer.launch() | |
} | |
async request(url: string, pageTimeout: number = 1000, waitForSelector: string = ""): Promise<string | undefined> { | |
if(!this.browser) { | |
// Give the browser 5 seconds before making a request, maybe the browser is still initializing | |
await delay(5000) | |
if(!this.browser) { | |
console.log(`Attempted to make a request to ${url} before the browser was initialized, even after giving it a safety buffer`) | |
return undefined | |
} | |
} | |
if(process.env.debug_mode) { | |
console.log(`Making request to ${url}`) | |
} | |
const page = await this.browser.newPage() | |
await page.goto(url) | |
// If we're provided a selector to wait for, do such. Otherwise, wait for a vague timeout | |
if(waitForSelector != "") { | |
await page.waitForSelector(waitForSelector) | |
} | |
else { | |
await page.waitForTimeout(pageTimeout) | |
} | |
if(process.env.debug_mode) { | |
await page.screenshot({path: 'out.png'}) | |
} | |
// We now should be at the specified page! | |
return await page.content() | |
} | |
} | |
/** | |
* A simple delay function | |
* @param ms The amount of ms that this function should delay | |
* @returns | |
*/ | |
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) | |
``` | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment