batch-process-with-generators

batch-process-with-generators.js

import { default as _ } from "lodash";

async function* processInBatches(data, batchSize) {
  const batches = _.chunk(data, batchSize);

  for (const batch of batches) {
    const result = await processBatch(batch);

    yield {
      batch,
      result,
    };
  }
}

// simulate e.g. network request
const processBatch = (batch) => new Promise((resolve) => {
  setTimeout(() => resolve({ ids: batch.map((i) => i.id) }), 2000);
});

async function main() {
  const allData = [
    { id: "1" },
    { id: "2" },
    { id: "3" },
    { id: "4" },
    { id: "5" },
    { id: "6" },
    { id: "7" },
    { id: "8" },
    { id: "9" },
  ];

  let processingState = {
    processedItems: [],
    unprocessedItems: allData.map((i) => i.id),
  };

  console.log("starting processing of %d items", allData.length);
  console.log("> processingState:", processingState);

  try {
    for await (const result of processInBatches(allData, 5)) {
      console.log("> batch process result:", result);

      processingState = {
        processedItems: [
          ...processingState.processedItems,
          ...result.batch.map((i) => i.id),
        ],
        unprocessedItems: processingState.unprocessedItems.filter(
          (i) => !result.batch.map((i) => i.id).includes(i)
        ),
      };
    }
  } catch (err) {
    console.error("error processing data:", err);
  } finally {
    console.log("done - processingState:", processingState);
  }
}

main();