Feature selection by Backward Elimination using both the p-value and the adjusted r-squared

backward_elimination2.py

import numpy as np
import statsmodels.formula.api as sm
def backward_elimination2(X, y, sl):
    """
    X: the data matrix with the independent variables (predictors)
    y: the matrix of the dependent variable (target)
    sl: statistical level, by default the user should add 0.05 (5%)
    """
    X = np.append(arr=np.ones((len(X),1)).astype(int), values=X, axis=1)
    
    while(True):
        regressor_OLS = sm.OLS(y,X).fit()
        ind = np.argmax(regressor_OLS.pvalues)
        max_pvalue = regressor_OLS.pvalues[ind]    
        
        if max_pvalue > sl:
            actual_adj_rsquared = regressor_OLS.rsquared_adj
            X_temp = np.delete(X, ind, axis=1)
            next_regressor_OLS = sm.OLS(y,X_temp).fit()
            next_adj_rsquared = next_regressor_OLS.rsquared_adj    
            if(actual_adj_rsquared > next_adj_rsquared):
                X = np.delete(X, 0, axis=1)
                print(regressor_OLS.summary())
                return X
            else:
                X = np.delete(X, ind, axis=1)
        else:
            print(regressor_OLS.summary())
            X = np.delete(X, 0, axis=1)
            return X