t-tests make high density neuroscience proposterously error prone

nesting_sim.R

library(psych)
library(ggplot2)
library(plyr)
library(lme4)
library(lmerTest)

n_sims <- 1000

sims <- expand.grid(
  i_sim     =  seq(n_sims),
  n_cells   =  c(10, 25, 50, 100, 500, 1000,  5000),
  n_animals =  c(5)
  
)
sims$p_t <- NA
sims$p_ml <- NA

pb <- txtProgressBar(min = 0, max = nrow(sims), style = 3)


for (row in 1:nrow(sims)){
  
  data <- sim.multi(n.obs   = sims[row,'n_animals']*2, 
                    nvar    = 1, 
                    nfact   = 1,
                    ntrials = sims[row, 'n_cells'],
                    phi.i   = rep(.01, sims[row,'n_animals']*2),
                    plot    = FALSE)
  
  data$condition <- as.factor(c(rep(0, sims[row,'n_animals']*sims[row,'n_cells']),
                                rep(1, sims[row,'n_animals']*sims[row,'n_cells'])))
  
  # pooled t-test
  sims[row, 'p_t'] <- t.test(data[data$condition == 0,]$V1,
                             data[data$condition == 1,]$V1)$p.value
  
  # multilevel
  sims[row, 'p_ml'] <- summary(lmer(V1 ~ condition + (1 | id), data=data))$coefficients[2, 'Pr(>|t|)']
  
  setTxtProgressBar(pb, row)
}

sims_sum <- ddply(sims, .(n_animals, n_cells), summarize,
                  mean_pt  = mean(p_t),
                  mean_pml = mean(p_ml),
                  pt_fpr.  = sum(p_t <0.05)/length(p_t),
                  pml_fpr  = sum(p_ml<0.05)/length(p_ml))


g.fpr <- ggplot(sims_sum, aes(x=n_cells, group=as.factor(n_animals)))+
  geom_hline(yintercept=0.05, linetype="dotted")+
  geom_line(aes(y=mean_pt), color="black", size=1)+
  geom_line(aes(y=pt_fpr),  color="red",   size=1)+
  geom_line(aes(y=pml_fpr), color="blue",  size=1)+
  theme_minimal()+
  theme(panel.grid.minor.x = element_blank(),
        panel.grid.major.x = element_blank(),
        panel.grid.minor.y = element_blank())
ggsave('../figures/fpr.pdf',g.fpr , width=5, height=3, units="in")