concurrent processing with chunks while holding a checkpoint as a file

concurrent_processing_with_chunks.js

const R = require("ramda");

// checkpoint, only processed data
const processed = require("./updated.json");
// processed and unprocessed data together, whole data
const toBeProcessed = require("./to_be_updated.json");
// data which will be processed on this execution
const willBeProcessed = R.difference(toBeUpdated, updated);
const chunks = R.splitEvery(10, willBeUpdated);

// function that does something
async function doSomething(param) {}

// write progress to the checkpoint file
function syncProgress() {
  fs.writeFileSync("updated.json", Buffer.from(JSON.stringify(updated)));
}

// concurrent execution with chunks
(async function () {
  for (const arr of chunks) {
    await Promise.all(
      arr.map((param) =>
        doSomething(param).then(() => {
          updated.push(param);
        })
      )
    ).catch(syncProgress);
    syncProgress();
  }
})();