node stuff

app.js

var request = require('request');
var cheerio = require('cheerio');
var Mongo = require('mongolian')
var cron = require('cron').CronJob;
var server = new Mongo
var db = server.db('stockish');
var queued = db.collection('bloomberg_queued');
var links = [];
var process_queue = require('./process_bloomberg_queue');

var BloombergJob = new cron({
	cronTime: '0 */10 * * * *',
	onTick: function() {
		console.log('running cron for bloomberg.com');
		request('http://www.bloomberg.com/archive/news/', function(err, response, body) {
			var that = this;

			$ = cheerio.load(body);
			if(err) console.log(err);
			$('.stories li a').each(function(ele) {
				//Print all links on the page
				var link = $(this).attr('href');
				var match = link.match(/http\:\/\//);
				if(match) {
					links.push(link);
				} else {
					links.push('http://www.bloomberg.com'+link);
				}
			});
			$(links).each(function(i, ele) {
				queued.findOne({link:ele}, function(err, data) {
					if(!data) { queued.insert({link:ele}) }
				});
			});
			process_queue.processBloomberg()
		});
	},
	start: true,
	onComplete: process_queue.processBloomberg()
});