Ridge and Lasso: Python

* Bold letters means output or conclusion, # denotes comments 

# importing libraries 
from sklearn.linear_model import LinearRegression
import mglearn
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split

# loading dataset and fitting linear regression
X, y = mglearn.datasets.load_extended_boston()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
lr = LinearRegression().fit(X_train, y_train)

 # Printing Coefficients and Intercept
print("lr.coef_: %s" % lr.coef_)
print("lr.intercept_: %s" % lr.intercept_)

lr.coef_: [-4.12710947e+02 -5.22432068e+01 -1.31898815e+02 -1.20041365e+01 -1.55107129e+01 2.87163342e+01 5.47040992e+01 -4.95346659e+01 2.65823927e+01 3.70620316e+01 -1.18281674e+01 -1.80581965e+01 -1.95246830e+01 1.22025403e+01 2.98078144e+03 1.50084257e+03 1.14187325e+02 -1.69700520e+01 4.09613691e+01 -2.42636646e+01 5.76157466e+01 1.27812142e+03 -2.23986944e+03 2.22825472e+02 -2.18201083e+00 4.29960320e+01 -1.33981515e+01 -1.93893485e+01 -2.57541277e+00 -8.10130128e+01 9.66019367e+00 4.91423718e+00 -8.12114800e-01 -7.64694179e+00 3.37837099e+01 -1.14464390e+01 6.85083979e+01 -1.73753604e+01 4.28128204e+01 1.13988209e+00 -7.72696840e-01 5.68255921e+01 1.42875996e+01 5.39551110e+01 -3.21709644e+01 1.92709675e+01 -1.38852338e+01 6.06343266e+01 -1.23153942e+01 -1.20041365e+01 -1.77243899e+01 -3.39868183e+01 7.08999816e+00 -9.22538241e+00 1.71980268e+01 -1.27718431e+01 -1.19727581e+01 5.73871915e+01 -1.75331865e+01 4.10103194e+00 2.93666477e+01 -1.76611772e+01 7.84049424e+01 -3.19098015e+01 4.81752461e+01 -3.95344813e+01 5.22959055e+00 2.19982410e+01 2.56483934e+01 -4.99982035e+01 2.91457545e+01 8.94267456e+00 -7.16599297e+01 -2.28147862e+01 8.40660981e+00 -5.37905422e+00 1.20137322e+00 -5.20877186e+00 4.11452351e+01 -3.78250760e+01 -2.67163851e+00 -2.55217108e+01 -3.33982030e+01 4.62272693e+01 -2.41509169e+01 -1.77532970e+01 -1.39723701e+01 -2.35522208e+01 3.68353800e+01 -9.46890859e+01 1.44302810e+02 -1.51158659e+01 -1.49513436e+01 -2.87729579e+01 -3.17673192e+01 2.49551594e+01 -1.84384534e+01 3.65073948e+00 1.73101122e+00 3.53617137e+01 1.19553429e+01 6.77025947e-01 2.73452009e+00 3.03720012e+01] lr.intercept_: 30.934563673637694


# printing performance
print("training set score: %f" % lr.score(X_train, y_train))
print("test set score: %f" % lr.score(X_test, y_test))

training set score: 0.952052 test set score: 0.607472

# Ridge
# regularization parameter alpha
from sklearn.linear_model import Ridge
# alpha = 1.0  default parameter
ridge = Ridge().fit(X_train, y_train)
print("training set score: %f" % ridge.score(X_train, y_train))
print("test set score: %f" % ridge.score(X_test, y_test))

training set score: 0.885797 test set score: 0.752768

# alpha = 10 ridge10 = Ridge(alpha=10).fit(X_train, y_train) print("training set score: %f" % ridge10.score(X_train, y_train)) print("test set score: %f" % ridge10.score(X_test, y_test))

training set score: 0.788279 test set score: 0.635941

# alpha = 0.1 ridge01 = Ridge(alpha=0.1).fit(X_train, y_train) print("training set score: %f" % ridge01.score(X_train, y_train)) print("test set score: %f" % ridge01.score(X_test, y_test))

training set score: 0.928227 test set score: 0.772207




# Lasso
# alpha = 1.0

from sklearn.linear_model import Lasso
lasso = Lasso().fit(X_train, y_train)
print("training set score: %f" % lasso.score(X_train, y_train))
print("test set score: %f" % lasso.score(X_test, y_test))

print("number of features used: %d" % np.sum(lasso.coef_ != 0))

training set score: 0.293238 test set score: 0.209375 number of features used: 4

# aplha = 0.01 lasso001 = Lasso(alpha=0.01).fit(X_train, y_train) print("training set score: %f" % lasso001.score(X_train, y_train)) print("test set score: %f" % lasso001.score(X_test, y_test)) print("number of features used: %d" % np.sum(lasso001.coef_ != 0))

training set score: 0.896112 test set score: 0.767800 number of features used: 32

lasso00001 = Lasso(alpha=0.0001).fit(X_train, y_train) print("training set score: %f" % lasso00001.score(X_train, y_train)) print("test set score: %f" % lasso00001.score(X_test, y_test)) print("number of features used: %d" % np.sum(lasso00001.coef_ != 0))

training set score: 0.942093 test set score: 0.697654 number of features used: 100

So we can say Ridge regression is usually the first choice between these two models. However, if you have a large number of features and expect only a few of them to be important, might be a better choice. Similarly, if you would like to have a Lasso model that is easy to interpret, it will provide a model that is easier to under Lasso stand, as it will select only a subset of the input features.