Last active
December 2, 2017 17:16
-
-
Save erasmo-marin/e8a3c04d0769ee27fbc081fa405eb880 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import puppeteer from "puppeteer"; | |
async function crawler({ url }) { | |
const browser = await puppeteer.launch({ args: ["--no-sandbox"] }); | |
let page = null; | |
let html = false; | |
try { | |
page = await browser.newPage(); | |
//networkidle0: consider navigation to be finished when | |
//there are no more than 2 network connections for at least 500 ms. | |
//(https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagegobackoptions) | |
await page.goto(url, { waitUntil: "networkidle0" }); | |
html = await page.content(); | |
} catch (e) { | |
debug.warn(`Not able to fetch ${url}`); | |
} finally { | |
if (page) { | |
await page.close(); | |
} | |
return html; | |
} | |
} | |
export default crawler; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment