agenda.md

Agenda investigation spike

Attempt 1:

BEHAVIOUR: This enforces only one instance carrying out jobs at a time

const Agenda = require('agenda');

const { MONGO_CONNECTION_STRING: mongoConnectionString } = require('./config');

const agenda = new Agenda({
  db: { address: mongoConnectionString },
})

let COUNTER = 0;

agenda.define("some job", { lockLifetime: 32000 }, async (job) => {
  console.log(`Starting Job ${++COUNTER} Polling S3 at ${new Date().toTimeString()}`);
  await new Promise((res) => setTimeout(res, 30000));
  console.log(`Processed file - job ${COUNTER} completed successfully at ${new Date().toTimeString()}`);
});

(async function () {
  await agenda.start();
  await agenda.every("10 seconds", "some job");
})();

ATTEMPT 2:

BEHAVIOUR: This processes files concurrently across multiple instances. BUT, lacks any mechanism for preventing the same file from being processed concurrently.

const Agenda = require('agenda');
const { MONGO_CONNECTION_STRING: mongoConnectionString } = require('./config');

const agenda = new Agenda({
  db: { address: mongoConnectionString },
})

let COUNTER = 0;

agenda.define("some job", { lockLifetime: 32000 }, async (job) => {
  console.log(`Starting Job ${++COUNTER} Polling S3 at ${new Date().toTimeString()}`);
  await new Promise((res) => setTimeout(res, 30000));
  console.log(`Processed file - job ${COUNTER} completed successfully at ${new Date().toTimeString()}`);
});

(async function () {
  const job = agenda.create("some job");
  await agenda.start();
  await job.repeatEvery("30 seconds").save();
})();