Coefficients of highly correlated covariates -- Lasso vs Ridge

lasso_vs_ridge.R

# simulate highly correlated x1-x2
set.seed(8)
X = MASS::mvrnorm(
  n = 100
  , mu = c(0, 0)
  , Sigma = matrix(c(1, 0.99, 0.99, 1), nrow=2, byrow=T)
  , empirical = F
)
# simulate y
df = data.frame(
  y = rnorm(100, mean=X[,1] + X[,2], sd=1)
  ,x1 = X[,1]
  ,x2 = X[,2]
)
# corrs
cor(df)

# no reg + lasso + ridge
library(glmnet)
mod = lm(y ~ ., df)
mod_lasso = cv.glmnet(X, df$y, alpha=1) 
mod_ridge = cv.glmnet(X, df$y, alpha=0) 


coef(mod) # "random" sparsity
coef(mod_lasso, s="lambda.1se") # "random" sparsity
coef(mod_ridge, s="lambda.1se") # no sparsity: similar covariates get similar weights