public
Last active

Playing with Tower and Craigslist in JavaScript

  • Download Gist
craigslist-now.js
JavaScript
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
// http://progrium.com/localtunnel/
// gem install localtunnel
var craigslist = require('craigslist')
, twilio = require('twilio')
, TwilioClient = twilio.Client
, credentials = require('./credentials');
 
var twilioClient = new TwilioClient(credentials.twilio.key, credentials.twilio.secret, credentials.twilio.domain)
, phone = twilioClient.getPhoneNumber(credentials.twilio.phone)
, phoneNumber = credentials.twilio.testPhone;
 
require('Tower');
 
Tower.config.databases = {
mongodb: {
default: true
, development: {
name: "craigslist-now-development"
, port: 27017
, host: "127.0.0.1"
}
}
}
 
global.App = Tower.Application.create();
 
App.Listing.__name__ = 'Listing';
App.Listing.field('title');
App.Listing.field('description');
App.Listing.field('url');
App.Listing.field('publishedAt', {type: 'Date'});
App.Listing.field('email');
App.Listing.field('emailUrl');
App.Listing.field('cities', {type: ['String']});
App.Listing.field('price');
App.Listing.field('bedrooms');
App.Listing.field('footage');
App.Listing.field('postId');
 
App.initialize(function() {
var refreshInterval = 10 * 1000; // every 10 seconds
var lastRefreshedAt, refreshAfter, timeout, now;
 
function textMessage(message) {
console.log(message);
 
phone.sendSms(phoneNumber, message, null, function(sms) {
sms.on('processed', function(params, response) {
console.log('Text sent:');
console.log(message);
});
});
}
 
function refresh() {
if (timeout) return;
process.nextTick(function() {
now = new Date;
if (now >= refreshAfter) {
fetch();
} else {
timeout = setTimeout(function() {
clearTimeout(timeout);
timeout = null;
process.nextTick(fetch);
}, refreshAfter.getTime() - now.getTime() + 1);
}
});
}
 
function fetch() {
lastRefreshedAt = new Date;
refreshAfter = new Date(lastRefreshedAt.getTime() + refreshInterval);
 
// get the latest listing saved in the database
App.Listing.desc('publishedAt').first(function(error, latestListing) {
latestListing = latestListing ? latestListing.toJSON() : {};
 
var listUrl = 'http://sfbay.craigslist.org/pen/apa/';
 
// scrape all the ones after the last one we fetched
craigslist.getList(listUrl, latestListing, function(error, array) {
if (!array.length) {
console.log('No new listings.');
refresh();
}
 
function iterate(attributes, next) {
// grab the details of each one
craigslist.getListing(attributes, function(error, attributes) {
// save them to the database
console.log('saving...', attributes.url);
App.Listing.create(attributes, next);
});
}
 
Tower.async(array, iterate, function() {
// and send yourself a text so you can respond immediately.
var message = [
array.length + ' new listings!'
, 'visit: ' + listUrl
].join('\n');
 
textMessage(message);
 
refresh();
});
});
});
}
 
phone.setup(fetch);
});

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.