Getting Kue working on Heroku

README.md

Workers can be activated with heroku scale:

heroku scale web=1 worker=1

Each process is $0.05/hr (or about $35/month).

You'll need a Redis instance. For RedisToGo you need a mini or higher to accept two connections AFAICS. Not sure what the upper limit is on workers yet.

heroku addons:add redistogo:mini   

A mini instance is just $5 a month and should be billed by the second. To remove, do:

heroku addons:remove redistogo   

app.js

var express = require('express')
  , kue = require('kue')
  , url = require('url')
  , redis = require('kue/node_modules/redis');

// make sure we use the Heroku Redis To Go URL
// (put REDISTOGO_URL=redis://localhost:6379 in .env for local testing)

kue.redis.createClient = function() {
    var redisUrl = url.parse(process.env.REDISTOGO_URL)
      , client = redis.createClient(redisUrl.port, redisUrl.hostname);
    if (redisUrl.auth) {
        client.auth(redisUrl.auth.split(":")[1]);
    }
    return client;
};

// then access the current Queue
var jobs = kue.createQueue()
  , app = express.createServer();

app.get('/', function(req, res) {
    
    var job = jobs.create('crawl', {
        url: 'http://example.com'
      , token: 'foo'
    });

    job.on('complete', function(){
        // avoid sending data after the response has been closed
        if (res.finished) {
            console.log("Job complete");
        } else {
            res.send("Job complete");
        }
    }).on('failed', function(){
        if (res.finished) {
            console.log("Job failed");
        } else {
            res.send("Job failed");
        }
    }).on('progress', function(progress){
        console.log('job #' + job.id + ' ' + progress + '% complete');
    });
    
    job.save();

    // timeout after 5s
    setTimeout(function() {
        res.send("OK (timed out)");
    }, 5000);
});

// wire up Kue (see /active for queue interface)
app.use(kue.app);

app.listen(process.env.PORT);
console.log("Express server listening on port %d in %s mode", app.address().port, app.settings.env);

consumer.js

var kue = require('kue')
  , url = require('url')
  , redis = require('kue/node_modules/redis');

// make sure we use the Heroku Redis To Go URL
// (put REDISTOGO_URL=redis://localhost:6379 in .env for local testing)

kue.redis.createClient = function() {
    var redisUrl = url.parse(process.env.REDISTOGO_URL)
      , client = redis.createClient(redisUrl.port, redisUrl.hostname);
    if (redisUrl.auth) {
        client.auth(redisUrl.auth.split(":")[1]);
    }
    return client;
};

var jobs = kue.createQueue();

// see https://github.com/learnBoost/kue/ for how to do more than one job at a time
jobs.process('crawl', function(job, done) {
  console.log(job.data);
  done();
});

package.json

{
    "name": "kue-test"
  , "version": "0.0.1"
  , "private": true
  , "dependencies": {
      "express": "~2.4.6"
    , "ejs": "~0.0.1"
    , "kue": "~0.3.1"
  }
}

Procfile

web: node app.js
worker: node consumer.js