Testing nested for loops with foreach

nested_for_loops.R

# The idea I'm trying to test is if I can nest a parallel
# for loop inside another parallel for loop.

# The following does open up 4 cores with the first loop
# and opens an additional 3 cores with the second. I see
# seven running at one time, the four are turned off then
# 3 are turned off.

# I'm not sure if this is truly 7 parallel cores or if it's
# 4 then 3 sequentially.

library(doParallel)

# Create two big datasets
dat1 <- data.frame(a = rnorm(1e7), b = sample(letters, 1e7, T))
dat2 <- data.frame(y = rnorm(1e7), z = sample(letters, 1e7, T))

# Create a list where the first three elements have a chunk
# of dataset 1 and the last element is all of dataset 2
N <- 3
chunks <- split(dat1, sample(1:N, nrow(dat1), replace=T))
chunks[[4]] <- dat2

cl <- makeCluster(4)
registerDoParallel(cl)


# The idea is that the outer for loop processes the
# four pieces of the list and the inner loop will
# process the fourth element of the list (dataset 2)
# using multiple cores

res <- foreach(i = 1:(4))%dopar%{
  # This line is just to do something that takes
  # time so I can see the cores open
  junk <- data.frame(a = rnorm(1e7), b = sample(letters, 1e7, T))
  
  dat <- chunks[[i]]
  if(i<=3){
    newdat <- head(dat)
  }else{
    # send this dataset off in a new parallel process
    newdat<- f()
  }
  
  return(newdat)
}


stopCluster(cl)


# Here's the function that is the inner for loop
f <- function(){
  
  library(doParallel)
  cl <- makeCluster(3)
  registerDoParallel(cl)
  
  # Here is the second parallel for loop
  inner_res <- foreach(i = 1:(N+1), .combine = 'c')%dopar%{
    
    # again, this code is just to make it take long
    # enough to see what's happening
    junk <- data.frame(a = rnorm(1e7), b = sample(letters, 1e7, T))
    return(1)
  }
  stopCluster(cl)
  return(inner_res)
}