Skip to content

Instantly share code, notes, and snippets.

@markni
Created June 22, 2018 02:34
Show Gist options
  • Save markni/6e10b4aacbb789cea2f4ec70f6ec9dca to your computer and use it in GitHub Desktop.
Save markni/6e10b4aacbb789cea2f4ec70f6ec9dca to your computer and use it in GitHub Desktop.
Processing speed of large batch of agenda.now() jobs drops dramatically as time goes by
const Agenda = require(`agenda`);
const mongoose = require(`mongoose`);
mongoose.connect(process.env.MONGO);
const agenda = new Agenda({
maxConcurrency: 1,
db: {
address: process.env.MONGO,
},
});
let lastJobCount;
// agenda.processEvery(`30 seconds`);
agenda.defaultLockLifetime(5e3); // 5 second lock
agenda.define(`TEST JOB`, { concurrency: 1 }, function(job, done) {
setTimeout(()=>{
done();
}, 500)
});
agenda.on(`ready`, () => {
console.log(`${new Date()} Agenda is ready`);
agenda.start();
setTimeout(()=>{
//
for (let i = 0; i < 25000; i++) {
agenda.now(`TEST JOB`, {id: i});
}
}, 10e3);
setInterval(run, 60e3);
});
function run(){
mongoose.connection.db
.collection(`agendaJobs`)
.count({lastFinishedAt: {$exists: false}})
.then(c => {
console.log(`${new Date()} ${c} unfinished jobs in DB.`);
if (lastJobCount && c) {
let perM = lastJobCount - c;
console.log(`${new Date()} processed job at ${perM} / min, ETA finish in ${c / perM} minutes`);
} else if (c === 0) {
console.log(`${new Date()} Jobs all done`);
}
lastJobCount = c;
});
}
mongoose.connection.on(`connected`, () => {
console.log(`${new Date()} Mongodb connected`);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment