Created
June 22, 2018 02:34
-
-
Save markni/6e10b4aacbb789cea2f4ec70f6ec9dca to your computer and use it in GitHub Desktop.
Processing speed of large batch of agenda.now() jobs drops dramatically as time goes by
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Agenda = require(`agenda`); | |
const mongoose = require(`mongoose`); | |
mongoose.connect(process.env.MONGO); | |
const agenda = new Agenda({ | |
maxConcurrency: 1, | |
db: { | |
address: process.env.MONGO, | |
}, | |
}); | |
let lastJobCount; | |
// agenda.processEvery(`30 seconds`); | |
agenda.defaultLockLifetime(5e3); // 5 second lock | |
agenda.define(`TEST JOB`, { concurrency: 1 }, function(job, done) { | |
setTimeout(()=>{ | |
done(); | |
}, 500) | |
}); | |
agenda.on(`ready`, () => { | |
console.log(`${new Date()} Agenda is ready`); | |
agenda.start(); | |
setTimeout(()=>{ | |
// | |
for (let i = 0; i < 25000; i++) { | |
agenda.now(`TEST JOB`, {id: i}); | |
} | |
}, 10e3); | |
setInterval(run, 60e3); | |
}); | |
function run(){ | |
mongoose.connection.db | |
.collection(`agendaJobs`) | |
.count({lastFinishedAt: {$exists: false}}) | |
.then(c => { | |
console.log(`${new Date()} ${c} unfinished jobs in DB.`); | |
if (lastJobCount && c) { | |
let perM = lastJobCount - c; | |
console.log(`${new Date()} processed job at ${perM} / min, ETA finish in ${c / perM} minutes`); | |
} else if (c === 0) { | |
console.log(`${new Date()} Jobs all done`); | |
} | |
lastJobCount = c; | |
}); | |
} | |
mongoose.connection.on(`connected`, () => { | |
console.log(`${new Date()} Mongodb connected`); | |
}); | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment