illustrate that tests with small P-values always have high observed power (Fig from Andrew D. Althouse, Post Hoc Power: Not Empowering, Just Misleading, Journal of Surgical Research, 2020)

observed_power_simulation.R

# https://www.sciencedirect.com/science/article/pii/S0022480420305023?via%3Dihub

# illustrates that tests with small P-values always have high observed power (on the left side, where the P-value is close to 0 and observed power close to 1). 

x1 <- rnorm(30, mean = 2, sd = 2)
x2 <- rnorm(30, mean = 3, sd = 2)
tout <- t.test(x1, x2)
obs_eff <- diff(tout$estimate)
pout <- power.t.test(n = 30, delta = abs(obs_eff), sd = 2, sig.level = 0.05)
pout$power

obs_power <- function(n = 30){
  x1 <- rnorm(n, mean = 2, sd = 2)
  x2 <- rnorm(n, mean = 3, sd = 2)
  tout <- t.test(x1, x2)
  obs_eff <- diff(tout$estimate)
  pout <- power.t.test(n = n, delta = abs(obs_eff), 
                       sd = sd(c(x1, x2)), 
                       sig.level = 0.05)
  c(pvalue = tout$p.value, power = pout$power)
}

rout <- replicate(n = 10000, obs_power(n=30))
# rout[,1:5]

plot(rout['pvalue',], rout['power',], 
     xlab = 'P-Value', ylab = 'Observed Power')