Convert nested list to dataframe quickly

nested_list_to_dataframe.R

pacman::p_load(Hmisc, rrapply)

# loop variables
scenarios <- c("scenario1", "scenario2")
groups <- c("grp1", "grp2", "grp3", "grp4")
 
# create two lists, one for the plots and one for the statistics only
ALL_stats <- list()
ALL_plots <- list()
for(scenario in scenarios){
  for(group in groups) {
    # do something, this line below pseudo-simulates the ouptut from compute_LR_stats()
    results <- list(stats = data.frame(LB=rnorm(1, 0, 1), LB_GSD=rnorm(1, 0, 0.1), disp=rnorm(1, 1, 1)), plot = "ggplot")
    # you store the statistics and plots in the two lists
    ALL_stats[[scenario]][[group]] <- results$stats # store only the statistics
    ALL_plots[[scenario]][[group]] <- results$plot  # store only the plots
  }
}

# show the tree structure of the list with all the statistics saved
list.tree(ALL_stats, fill = "_", depth = 3)

# convert ALL_stats list to a data.frame and give colnames.
ALL_results <- rrapply(ALL_stats, how = "bind",  options = list(namecols = TRUE))
colnames(ALL_results)[c(1,2)] <- c("scenario", "group")
head(ALL_results)

# the above should still retain the stats in the last level of the 'ALL_stats' list in a list form.
# to unnest only the last level in a wide-dataframe format do:
pacman::p_load(tidyr)
ALL_results <- rrapply(all_stats, how = "bind", options = list(namecols = TRUE, coldepth = 3)) %>% 
  unnest_wider(., value, names_sep = "_") %>% # unnest wider on the value list
  unnest_wider(., SE, names_sep = "_")        # unnest wider on the SE list
colnames(ALL_results) <- c("scenario", "val_group",
                      paste0("value_", rownames(all_stats[[1]][[1]])),
                      paste0("SE_", rownames(all_stats[[1]][[1]]))
                      )
head(ALL_results)