Created
May 12, 2012 21:41
-
-
Save fabdrol/2669256 to your computer and use it in GitHub Desktop.
App.js,
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** REQUIREMENTS **/ | |
var Step = require('step') | |
, Messenger = require('./lib/messenger') | |
, messenger = new Messenger() | |
, util = require('util') | |
, models = require('./models') | |
, queue = require('./queue') | |
; | |
var total = 0 | |
, complete = 0 | |
, runs = 0 | |
, failed = 1 | |
, time = 60000 | |
; | |
function Walker() { | |
Step( | |
function load() { | |
// load up all "languages", "categories" and "sources" from db, and pass to next function. | |
}, | |
function compile(err, languages, categories, sources) { | |
if(err) this(err, null); | |
// function returnCompiled(); | |
// Do some stuff to compile the different variables into a useable datatype | |
var output = new Array; | |
var next = this; | |
for(var i in sources) { | |
(function(source) { | |
var compiled = returnCompiled(source); | |
var job = queue.add({ // proxy function to ease the adding of Jobs. | |
worker: 'download', | |
data: compiled, | |
priority: 'high', | |
attempts: 2 | |
}); | |
// Increment total. | |
total++; | |
job.on('complete', function() { | |
job.remove(); | |
complete++; | |
}); | |
job.on('failed', function() { | |
complete++; | |
failed++; | |
}); | |
})(sources[i]); | |
} | |
} | |
); | |
}; | |
// INITIATE THE FIRST LOOP. | |
Walker(); | |
// catch something from the worker, and create more jobs. | |
messenger.subscribe(function(data) { | |
if(data.sender === 'worker:download') { | |
models.Article.findOne({uid: data.uid}, function(err, doc) { | |
if(!err) { | |
var job = queue.add({ | |
worker: 'analyse', | |
data: doc, | |
priority: 'high', | |
attempts: 4 | |
}); | |
job.on('complete', function() { | |
job.remove(); | |
}); | |
} | |
}); | |
} | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment