Run with very large data without out of memory

main-alpha.js

let loopRunning = true;

// Function to simulate resource-intensive task
function doTask() {
  // Simulate a task that consumes memory
  const array = new Array(10000).fill(
    new Array(10000).fill(
      new Array(10000).fill(
        new Array(10000).fill(
          new Array(10000).fill(
            new Array(10000).fill(
              new Array(10000).fill(
                new Array(10000).fill(
                  new Array(10000).fill(
                    new Array(10000).fill(
                      new Array(10000).fill(
                        new Array(10000).fill(
                          new Array(10000).fill(
                            new Array(10000).fill(
                              new Array(10000).fill(
                                new Array(10000).fill(
                                  new Array(10000).fill('9')
                                )
                              )
                            )
                          )
                        )
                      )
                    )
                  )
                )
              )
            )
          )
        )
      )
    )
  )
  
  // Do something with the array (e.g., process data)
}

// Function to periodically check system resources and stop loop if needed
function checkAndStopLoop() {
  const memoryUsage = performance.memory.usedJSHeapSize;
  const totalMemory = performance.memory.totalJSHeapSize;

  // Calculate the percentage of memory used
  const memoryUsagePercentage = (memoryUsage / totalMemory) * 100;

  // Check if memory usage exceeds a certain threshold (e.g., 90%)
  if (memoryUsagePercentage > 90) {
    console.log('Memory usage exceeded threshold. Stopping loop.');
    loopRunning = false;
  }
}

// Main loop
function mainLoop() {
  while (loopRunning) {
    doTask(); // Perform the task

    // Check system resources periodically
    setTimeout(checkAndStopLoop, 1000);
  }
}

// Start the loop
mainLoop();

main.js

function processLargeArray(largeArray) {
  const chunkSize = 1000; // Adjust chunk size as needed
  let processedCount = 0;

  function processChunk() {
    const chunk = largeArray.slice(processedCount, processedCount + chunkSize);
    processedCount += chunkSize;

    // Perform operations on the chunk
    for (const item of chunk) {
      // Do something with the item
      console.log(item); // Example operation
    }

    if (processedCount < largeArray.length) {
      setTimeout(processChunk, 0); // Schedule next chunk asynchronously
    } else {
      console.log('Array processing complete!');
    }
  }

  processChunk(); // Start processing the array in chunks
}


// Test
processLargeArray(
  new Array(10000).fill(
    new Array(10000).fill(
      new Array(10000).fill(
        new Array(10000).fill(
          new Array(10000).fill(
            new Array(10000).fill(
              new Array(10000).fill(
                new Array(10000).fill(
                  new Array(10000).fill(
                    new Array(10000).fill(
                      new Array(10000).fill(
                        new Array(10000).fill(
                          new Array(10000).fill(
                            new Array(10000).fill(
                              new Array(10000).fill(
                                new Array(10000).fill(
                                  new Array(10000)
                                )
                              )
                            )
                          )
                        )
                      )
                    )
                  )
                )
              )
            )
          )
        )
      )
    )
  )
);