In [1]:
import warnings
warnings.filterwarnings('ignore')
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import watermark
%load_ext watermark
%matplotlib inline
In [2]:
%watermark -i -n -v -m -g -iv
Python implementation: CPython Python version : 3.10.9 IPython version : 8.10.0 Compiler : Clang 14.0.6 OS : Darwin Release : 22.5.0 Machine : x86_64 Processor : i386 CPU cores : 16 Architecture: 64bit Git hash: 03a6cf9c5faed1d6e551b54357ff497cfa569fb9 watermark : 2.4.2 numpy : 1.23.5 matplotlib: 3.7.0
In [3]:
plt.style.use('d4sci.mplstyle')
Load dataset¶
In [4]:
data = np.array(np.loadtxt("data/Anscombe1.dat"))
In [5]:
print(data)
[[10. 8.04] [ 8. 6.95] [13. 7.58] [ 9. 8.81] [11. 8.33] [14. 9.96] [ 6. 7.24] [ 4. 4.26] [12. 10.84] [ 7. 4.82] [ 5. 5.68]]
In [6]:
X = data[:, 0].reshape(-1, 1)
y = data[:, 1].reshape(-1, 1)
In [7]:
X
Out[7]:
array([[10.],
[ 8.],
[13.],
[ 9.],
[11.],
[14.],
[ 6.],
[ 4.],
[12.],
[ 7.],
[ 5.]])
In [8]:
y
Out[8]:
array([[ 8.04],
[ 6.95],
[ 7.58],
[ 8.81],
[ 8.33],
[ 9.96],
[ 7.24],
[ 4.26],
[10.84],
[ 4.82],
[ 5.68]])
In [9]:
plt.plot(X, y, '.', markersize=30, label='points')
plt.xlabel('X')
plt.ylabel('y')
plt.legend()
Out[9]:
<matplotlib.legend.Legend at 0x7f7a61853cd0>
Get matrix dimensions and add the bias column
In [10]:
M, N = X.shape
X = np.concatenate((np.ones((M, 1)), X), axis=1) # Add x0 for the bias/intercept
In [11]:
print(X)
[[ 1. 10.] [ 1. 8.] [ 1. 13.] [ 1. 9.] [ 1. 11.] [ 1. 14.] [ 1. 6.] [ 1. 4.] [ 1. 12.] [ 1. 7.] [ 1. 5.]]
Set the training parameters and initialize the weight matrix
In [12]:
alpha = 0.01 # Step size
epsilon = 0.12 # Maximum initial value
# Initial random weights chosen uniformly between +/- epsilon
weights = 2*np.random.rand(N+1, 1)*epsilon - epsilon
count = 0
Training Procedure¶
In [13]:
oldJ = 0
err = 1
Js = []
while err > 1e-6:
Hs = np.dot(X, weights)
deltas = alpha/M*np.dot(X.T, (Hs-y))
count += 1
weights -= deltas
J = np.sum(np.power(Hs-y, 2.))/(2*M)
Js.append(J)
err = np.abs(oldJ-J)
oldJ = J
if count % 100 == 0:
print(count, J, err, weights.flatten())
print(count, J, err, weights.flatten())
100 1.016062079312162 0.0008512756515690612 [0.33714331 0.76377468] 200 0.93964903716122 0.0006846917399059294 [0.61186691 0.73657169] 300 0.8781890639016857 0.0005507061993756945 [0.85824858 0.71217509] 400 0.8287560324880023 0.00044293993976429746 [1.07921223 0.69029538] 500 0.7889964220432137 0.00035626217838202745 [1.27738012 0.67067289] 600 0.7570172660515031 0.00028654616202139493 [1.45510397 0.65307476] 700 0.7312960276766381 0.000230472691044592 [1.6144929 0.63729215] 800 0.7106081113442585 0.00018537209133373533 [1.75743843 0.62313775] 900 0.6939685593935335 0.0001490971103323968 [1.88563696 0.6104436 ] 1000 0.6805851579589711 0.0001199206857389612 [2.00060986 0.59905904] 1100 0.6698207194398905 9.645371956512605e-05 [2.10372155 0.58884897] 1200 0.6611627458150501 7.757894278814792e-05 [2.1961957 0.57969222] 1300 0.6541990284641138 6.239772184157832e-05 [2.27912973 0.57148013] 1400 0.6485980227963886 5.018727442207549e-05 [2.35350785 0.56411525] 1500 0.6440930633172868 4.036625760617252e-05 [2.42021274 0.55751016] 1600 0.6404696676072462 3.2467089952525896e-05 [2.480036 0.55158649] 1700 0.6375553247519008 2.6113689812090612e-05 [2.53368759 0.54627394] 1800 0.635211282047714 2.1003569971256297e-05 [2.58180419 0.54150946] 1900 0.6333259389457611 1.6893436152520813e-05 [2.62495684 0.5372365 ] 2000 0.6318095337174205 1.3587603698961281e-05 [2.66365763 0.53340437] 2100 0.630589869872598 1.0928680975075267e-05 [2.69836584 0.52996758] 2200 0.6296088788640166 8.790075903153394e-06 [2.72949338 0.52688534] 2300 0.6288198554269419 7.069968879114263e-06 [2.75740964 0.52412109] 2400 0.6281852339280631 5.686465111121031e-06 [2.7824459 0.52164201] 2500 0.6276747998548587 4.573695587373194e-06 [2.8048993 0.51941869] 2600 0.6272642512939481 3.6786810283517823e-06 [2.82503629 0.51742473] 2700 0.6269340419099861 2.9588095340482568e-06 [2.84309584 0.51563648] 2800 0.6266684503390045 2.379807814256729e-06 [2.85929227 0.51403272] 2900 0.6264548316897401 1.914109430956046e-06 [2.87381779 0.51259441] 3000 0.6262830155167157 1.53954234927145e-06 [2.88684479 0.51130448] 3100 0.6261448216023993 1.2382733231008913e-06 [2.89852785 0.51014763] 3199 0.6260346664535367 9.981300822525796e-07 [2.9089064 0.50911996]
In [14]:
fig, ax = plt.subplots(1)
ax.semilogy(Js)
ax.set_xlabel('Epoch')
ax.set_ylabel('Error')
Out[14]:
Text(0, 0.5, 'Error')
In [15]:
weights.flatten()
Out[15]:
array([2.9089064 , 0.50911996])
In [16]:
plt.plot(X.T[1], y, '.', markersize=30, label='points')
plt.plot(X.T[1], np.dot(X, weights.flatten()), '-',\
label='y = %2.2f + %2.2f x' % tuple(weights.flatten()))
plt.xlabel('X')
plt.ylabel('y')
plt.legend()
Out[16]:
<matplotlib.legend.Legend at 0x7f7a61965c00>
