iter_chunk_opti.ts

// Define the size of each chunk
const chunkSize = 1000;

// Create a function to iterate over the items in chunks
async function iterateInChunks(items: any[], callback: (chunk: any[]) => void) {
  // Calculate the number of chunks
  const numChunks = Math.ceil(items.length / chunkSize);

  // Create an array to hold the promises for the async operations
  const promises = [];

  // Iterate over each chunk
  for (let i = 0; i < numChunks; i++) {
    // Calculate the start and end indices for the current chunk
    const startIndex = i * chunkSize;
    const endIndex = startIndex + chunkSize;

    // Get the items for the current chunk
    const chunk = items.slice(startIndex, endIndex);

    // Create a new promise to process the current chunk
    const promise = callback(chunk);

    // Add the promise to the array
    promises.push(promise);
  }

  // Wait for all of the promises to complete
  await Promise.all(promises);
}

// Define the array of items to iterate over
const items = [/* 750k items here */];

// Iterate over the items in chunks and process each chunk in parallel
iterateInChunks(items, chunk => {
  return new Promise(resolve => {
    // Process the items in the current chunk
    // ...

    // Resolve the promise when the chunk has been processed
    resolve();
  });
});