App.js,

app.js

/** REQUIREMENTS **/
var Step		= require('step')
,	Messenger  	= require('./lib/messenger')
,	messenger 	= new Messenger()
,	util		= require('util')
,	models		= require('./models')
,	queue		= require('./queue')
;

var total		= 0
,	complete	= 0
,	runs 		= 0
,	failed		= 1
,	time 		= 60000
;

function Walker() {
	Step(
		function load() {
			// load up all "languages", "categories" and "sources" from db, and pass to next function.
		},

		function compile(err, languages, categories, sources) {
			if(err) this(err, null);

			// function returnCompiled();
			// Do some stuff to compile the different variables into a useable datatype

			var output = new Array;
			var next = this;

			for(var i in sources) {
				(function(source) {
					var compiled = returnCompiled(source);
					
					var job = queue.add({ // proxy function to ease the adding of Jobs.
						worker: 'download',
						data: compiled,
						priority: 'high',
						attempts: 2
					});

					// Increment total.
					total++;

					job.on('complete', function() {
						job.remove();
						complete++;
					});

					job.on('failed', function() {
						complete++;
						failed++;
					});
				})(sources[i]);
			}
		}
	);
};

// INITIATE THE FIRST LOOP.
Walker();

// catch something from the worker, and create more jobs.
messenger.subscribe(function(data) {
	if(data.sender === 'worker:download') {
		models.Article.findOne({uid: data.uid}, function(err, doc) {
			if(!err) {
				var job = queue.add({
					worker: 'analyse',
					data: doc,
					priority: 'high',
					attempts: 4
				});

				job.on('complete', function() {
					job.remove();
				});
			}
		});
	}
});