In noisy studies, we tend to overestimate the effect size when we select on statistical significance

typeM_error.R

set.seed(12345)

mu <- 0.5
sigma <- 1
n <- 25
B <- 1000

results <- replicate(B, {
  data <- rnorm(n, mu, sigma)
  
  mu_hat <- mean(data)
  sigma_hat <- sd(data)/sqrt(n)
  pval <- 2*pnorm(abs(mu_hat)/sigma_hat,
                  lower.tail = FALSE)
  
  return(c(mu_hat, sigma_hat, pval))
})

library(tidyverse)

tibble(mu_hat = results[1, ],
       selection = FALSE) %>% 
  bind_rows(
    tibble(mu_hat = results[1, results[3,] < 0.05],
           selection = TRUE)
  ) %>% 
  ggplot(aes(mu_hat, fill = selection)) + 
  geom_density(alpha = 0.5) + 
  geom_vline(xintercept = mu,
             linetype = 'dashed') + 
  theme_minimal()

Note that as the signal to noise ratio increases, the bias decreases