fetch_and_process.r

# parallel iteration pattern for processing chunks of data in parallel
fetch_and_process <- function(get_data_chunk, process_func, reduce_func) {
    n_workers <- future::nbrOfWorkers()
    results <- list()
    
    i <- 1
    out_of_data <- FALSE
    
    while(TRUE) {
        for (. in 1:n_workers) {
            
            chunk <- get_data_chunk()
            
            if (is.null(chunk)) {
                out_of_data <- TRUE
                break
            }
            
            results[[i]] <- future::future({ process_func(chunk) })
            i <- i + 1
        }
        
        lapply(results, value)
        
        if (out_of_data) {
            break
        }
    }
    
    # Extract the results from the futures
    results <- lapply(results, value)
    
    purrr::reduce(results, reduce_func)
}