Skip to content

Instantly share code, notes, and snippets.

@erickbrower
Last active August 25, 2016 02:28
Show Gist options
  • Save erickbrower/5060177 to your computer and use it in GitHub Desktop.
Save erickbrower/5060177 to your computer and use it in GitHub Desktop.
A quick attempt at scraping apartment data from Apartments.com with Node, PhantomJS, and jQuery. Currently borked.
var page = require('webpage').create(), stepIndex = 0, loadInProgress = false;
page.onLoadStarted = function() {
loadInProgress = true;
console.log('loading...');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('loading complete');
};
page.onConsoleMessage = function(message) {
console.log('>> ' + message);
};
var listingUrls = [], steps = [], listingSteps = [];
steps.push(function() {
page.open('http://apartments.com');
});
steps.push(function() {
page.includeJs('http://cdnjs.cloudflare.com/ajax/libs/jquery/1.8.3/jquery.min.js');
});
steps.push(function() {
page.evaluate(function() {
$('#search-query').val('30346');
$('#search-submit').click();
});
});
steps.push(function() {
listingUrls = page.evaluate(function() {
var links = [];
$('div.listings').find('div.listing').each(function() {
$(this).find('div.details a').each(function() {
links.push('http://apartments.com' + $(this).attr('href'));
});
});
var uniqueLinks = [];
$.each(links, function(i, el) {
if($.inArray(el, uniqueLinks) === -1) uniqueLinks.push(el);
});
return uniqueLinks;
});
});
steps.push(function() {
for(var i = 0; i < listingUrls.length; i++) {
steps.push(function() {
page.open(listingUrls[i], function() {
page.includeJs('http://cdnjs.cloudflare.com/ajax/libs/jquery/1.8.3/jquery.min.js');
});
});
steps.push(function() {
page.evaluate(function() {
console.log($('body').text());
});
});
}
});
interval = setInterval(function() {
if(!loadInProgress && typeof steps[stepIndex] == 'function') {
console.log('step ' + (stepIndex + 1));
steps[stepIndex]();
stepIndex++;
}
if(typeof steps[stepIndex] != 'function') {
console.log('test complete!');
phantom.exit();
}
}, 50);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment