GKPW.R

GKPW.R

setwd("C:/Users/Daniel/Downloads/Gilbert, King, Pettigrew, Wilson 2016 replication files/variability analysis replication files/data")
load("many labs replication cis.RData")
## Drop the top rows which are statistics from pooling together all the replications
res <- lapply(res, function(x) x[-c(1:2),])
res[[12]] <- res[[12]][-1,]
names(res[[16]])[3:5] <- names(res[[15]])[3:5]


## For each replicated study, get the number of the other replicated
## studies that were outside the CI
h<-res[[1]]
ii<-1
jj<-2
for(ii in 1:length(res)){#for all 16 studies
  h <- res[[ii]] #temp store single datframe with 36 replications of single study 
  for(jj in 1:nrow(h)){ #For all 36 studies
    h$outside[jj] <- sum(h$Lower.Conf.Limit.smd[jj] >= h$smd[-jj] | #sum the confidence intervals in all studies except current line that are larger than smd of current study
                           h$Upper.Conf.Limit.smd[jj] <= h$smd[-jj]) #or that is smaller than the current study. 
  } #This loop performs sum for all 16 studies. 
  h$pct.outside.95 <- h$outside / (nrow(h)-1)
  res[[ii]] <- h
}

pct.outside <- unlist(sapply(res, function(x) x$pct.outside.95))


mean(pct.outside)
## If we run the same experiment multiple times, we should expect that about 34.54% 
## of the time the replicated result will fall outside of the 95% CI of the "published" result
1-mean(pct.outside) ## 65.5% will be inside the 95% CI (what's reported in the paper)