A comparison of std.errors of the treatment effect from an additive DGP experiment where a pre-treatment variable is interacted maximally, only additive no interaction, or unadjusted for.

control-vs-interaction.R

library(tidyverse)
library(lme4)
library(broom)

N <- 1e4

generate_data <- function() {
  tibble(
    intercept = 1,
    pretreatment_var = case_when(rbinom(N, 1, p = 0.5) == 1 ~ 0.5, TRUE ~ -0.5),
    treatment_effect = case_when(rbinom(N, 1, p = 0.5) == 1 ~ 1, TRUE ~ 0),
    discrepancy = rnorm(N),
    y = intercept + pretreatment_var + treatment_effect + discrepancy
  )
}

extract_std_error <- . %>%
  tidy() %>%
  filter(term == "treatment_effect") %>%
  pull(std.error)

seq_len(1e2) %>%
  map_dbl(~ generate_data() %>%
            lm(y ~ treatment_effect*pretreatment_var, data = .) %>%
            car::deltaMethod("treatment_effect")  %>%
            tidy() %>%
            filter(column == "SE") %>% pull(mean)) %>%
  ecdf() %>%
  plot(col = "green", lwd = 3, xlim = c(0.019, 0.023),
       main = "std.error of treatment_effect")

seq_len(1e2) %>%
  map_dbl(~ generate_data() %>%
            lm(y ~ treatment_effect + pretreatment_var, data = .) %>%
            extract_std_error()) %>%
  ecdf() %>%
  lines(col = "black", lwd = 3)

seq_len(1e2) %>%
  map_dbl(~ generate_data() %>% lm(y ~ treatment_effect, data = .) %>% extract_std_error()) %>%
  ecdf() %>%
  lines(col = "purple", lwd = 3)

legend("topleft", legend = c("maximal", "additive", "unadjusted for"),
       col = c("green", "black", "purple"),
       lwd = 5)