Skip to content

Instantly share code, notes, and snippets.

@J2D2Development
Last active September 23, 2017 23:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save J2D2Development/ba21ad34bbbd155909ab8aefcc72e774 to your computer and use it in GitHub Desktop.
Save J2D2Development/ba21ad34bbbd155909ab8aefcc72e774 to your computer and use it in GitHub Desktop.
Optional argument- alias if not there
//products is your array to crawl, if crawlEmitter is passed, it's used (for broadcasting via socket to frontend).
//If not, it's aliased to console.log so the code with the logic stays the same (everything calls "crawlEmitter.emit").
async function run(products, crawlEmitter) {
if(!crawlEmitter) {
crawlEmitter = {
emit: function() {
console.log.apply(null, Array.from(arguments));
}
}
}
async function crawlPage(url) {
const page = await browser.newPage();
const pageResult = await page.goto(url);
let resultObj = { success: pageResult.ok, url };
resultGroup.push(resultObj);
crawlEmitter.emit('crawled', resultObj);
return page.close();
}
//more logic...
}
run(myProducts); //cli version
run(myProducts, crawlEmitter); //dashboard-UI version
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment