Gradient descent + Intro to scikit-learn
import numpy as np
import matplotlib.pyplot as plt
beta0true = 1
beta1true = 1
x = np.linspace(0,1,10)
t = beta0true + beta1true *x
tnoisy = t + np.random.normal(0,.1,len(x))
plt.scatter(x, tnoisy, c='r')
plt.plot(x, t)
plt.show()
# sum_i (t^i - (beta0 + beta1 x_1^i))^2
eta = .001
maxIter = 200
beta0 = np.random.normal(0,.1,1)
beta1 = np.random.normal(0,.1,1)
beta = np.squeeze(np.asarray([beta0, beta1]))
Xtildemat = np.hstack((np.ones((len(x), 1)), x.reshape(-1,1)))
for iteration in np.arange(maxIter):
beta0 = beta0 - \
(2/len(x))*eta * np.sum((tnoisy.reshape(-1,1) - np.matmul(Xtildemat, beta))*(-1))
beta1 = beta1 - \
(2/len(x))*eta* np.sum(np.multiply((tnoisy.reshape(-1,1) - np.matmul(Xtildemat, beta)), -x.reshape(-1,1)))
beta = np.squeeze(np.asarray([beta0, beta1]))
beta_learned = np.squeeze(np.asarray([beta0, beta1]))
xprediction = np.linspace(0,1,100)
XpredictionMatrix = np.hstack((np.ones((len(xprediction), 1)), \
xprediction.reshape(-1,1)))
prediction = np.matmul(XpredictionMatrix, beta_learned)
plt.scatter(x, tnoisy, c='r')
plt.plot(xprediction, prediction, c='g')
plt.plot(x, t)
plt.show()
from sklearn.linear_model import LinearRegression
reg = LinearRegression()
reg.fit(x.reshape(-1,1), tnoisy)
x_prediction = np.linspace(0,1,100)
predictions = reg.predict(x_prediction.reshape(-1,1))
plt.scatter(x, tnoisy, c='r')
plt.plot(x_prediction, predictions, c='g')
plt.show()
III.a Adding polynomial features and higher dimensional space
x = np.linspace(-1,1,10)
beta0true = .1
beta1true = .1
beta2true = 1
t = beta0true + beta1true *x + beta2true * (x**2)
tnoisy = t + np.random.normal(0,.1,len(x))
plt.scatter(x, tnoisy, c='r')
plt.show()
fig = plt.figure()
ax = fig.add_subplot(projection='3d')
ax.scatter(x, x**2, tnoisy, c='r')
<mpl_toolkits.mplot3d.art3d.Path3DCollection at 0x7f989ede9100>
III.b Learning the linear model (in the higher dimensional space)
xprediction = np.linspace(-1,1,50)
x2prediction = np.linspace(-1,1,50)
xx, xx2 = np.meshgrid(xprediction, x2prediction)
predictionMat = np.hstack((np.ones((len(xx.flatten()),1)),\
xx.flatten().reshape(-1,1), \
xx2.flatten().reshape(-1,1)))
predicted_target = reg.predict(predictionMat)
fig = plt.figure()
ax = fig.add_subplot(projection='3d')
ax.plot_surface(xx, xx2, predicted_target.reshape(np.shape(xx)), alpha=0.2)
ax.scatter(x, x**2, tnoisy, c='r')
plt.show()
XtildeMat = np.hstack((np.ones((len(x),1)), \
x.reshape(-1,1), (x**2).reshape(-1,1)))
print(XtildeMat)
[[ 1. -1. 1. ] [ 1. -0.77777778 0.60493827] [ 1. -0.55555556 0.30864198] [ 1. -0.33333333 0.11111111] [ 1. -0.11111111 0.01234568] [ 1. 0.11111111 0.01234568] [ 1. 0.33333333 0.11111111] [ 1. 0.55555556 0.30864198] [ 1. 0.77777778 0.60493827] [ 1. 1. 1. ]]
III.c Bringing the prediction in the orinal space
from sklearn.linear_model import LinearRegression
reg = LinearRegression()
reg.fit(XtildeMat, tnoisy)
x_prediction = np.linspace(-1,1,100)
predictionMat = np.hstack((np.ones((len(x_prediction),1)), \
x_prediction.reshape(-1,1),\
(x_prediction**2).reshape(-1,1)))
predictions = reg.predict(predictionMat)
plt.scatter(x, tnoisy, c='r')
plt.plot(x_prediction, predictions, c='g')
plt.show()
x = np.linspace(-1,1,10)
beta0true = .1
beta1true = .1
beta2true = 1
t = beta0true + beta1true *x + beta2true * (x**2)
tnoisy = t + np.random.normal(0,.2,len(x))
plt.scatter(x, tnoisy, c='r')
plt.show()
IV.a Generating the high dimensional model
from sklearn.preprocessing import PolynomialFeatures
# Generating high dimensional (high complexity) model
poly = PolynomialFeatures(50)
XtildeMat = poly.fit_transform(x.reshape(-1,1))
reg = LinearRegression()
reg.fit(XtildeMat, tnoisy)
x_prediction = np.linspace(-1,1,100)
predictions = reg.predict(poly.fit_transform(x_prediction.reshape(-1,1)))
plt.plot(x_prediction,predictions)
plt.scatter(x, tnoisy, c='r')
plt.show()
IV.b Regularizing with ridge/Lasso
from sklearn.linear_model import Ridge
clf = Ridge(alpha=.1)
clf.fit(XtildeMat, tnoisy)
x_prediction = np.linspace(-1,1,100)
predictions = clf.predict(poly.fit_transform(x_prediction.reshape(-1,1)))
plt.plot(x_prediction,predictions)
plt.scatter(x, tnoisy, c='r')
plt.show()