linear regression counter example

lr.py

import numpy as np
from sklearn.linear_model import LinearRegression

#create some random data.
x1 = np.random.randn(250,1)
x2 = 0.001*np.random.randn(250,1) - x1
x3 = np.random.randn(250,1)

# Y is a linear combination of the created data, with 
# weights (10,10,0.01)
Y = 10*x1 + 10*x2 + 0.1*x3 + 0.01*random.randn( 250, 1)

#data matrix
X = np.concatenate( [x1,x2,x3], axis=1 )


lr = LinearRegression()
lr.fit(X, Y)


print "Coefficients: ",
print lr.coef_
print "R-squared value:", lr.score(X,Y)

print
# if we remove the last coefficient, on justification
# that it is too "small", our results are:
lr.coef_[0,-1] = 0
print "Coefficients, after dropping the 'insignificant' variable x_3 ",
print lr.coef_
print "R-squared value:", lr.score(X,Y)

"""
#Output: 

Coefficients:  [[ 9.296  9.296  0.101]]
R-squared value: 0.99080

Coefficients, after dropping the 'insignificant' variable x_3
[[ 9.296  9.296  0.  ]]

R-squared value: 0.025059
"""