Skip to content

Instantly share code, notes, and snippets.

@AJamesPhillips
Last active December 17, 2015 06:28
Show Gist options
  • Save AJamesPhillips/5565218 to your computer and use it in GitHub Desktop.
Save AJamesPhillips/5565218 to your computer and use it in GitHub Desktop.
fs = require('fs');
function scrape() {
var url = "some local url with asset served";
var date = new Date();
var stamp = date.getTime();
console.log("scraping url: " + url);
//Screenshot 1 - Routes and Traffic Time Table
var page = require('webpage').create();
page.viewportSize = {width: 400, height: 600};
page.zoomfactor=1.1;
page.open(url, function() {
window.setTimeout(function() {
page.render("./DataV1_" + stamp + ".png", {format:"png", quality: 100000});
page.close();
}, 5000);
});
var page1 = require('webpage').create();
page1.viewportSize = {width: 1920, height: 1080};
page1.zoomfactor=1.1;
page1.open(url, function() {
window.setTimeout(function() {
page1.render("./DataV2_" + stamp + ".png", {format:"png", quality: 100000});
page1.close();
}, 5000);
});
var page2 = require('webpage').create();
page2.open(url, function (status) {
if (status !== 'success') {
console.log('Unable to access network');
} else {
var ua = page2.evaluate(function () {
return document.getElementById('<>').innerText;
});
f=null;
f=fs.open("./DataV3_" + stamp + ".txt", "w");
f.write(ua);
f.close();
}
page2.close();
});
}
//scrape every 5 minutes
// setInterval(scrape, 5 * 60 * 1000);
//scrape every 20 seconds
setInterval(scrape, 20 * 1000);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment