%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
plt.xkcd()
None
# learning rate hyperparameter (step size scaling)
ALPHA = 1e-4
# weight decay hyperparameter (regularization)
LAMBD = 1e-3
# threshold for convergence test
DIFF_THRESHOLD = 1e-5
def sigmoid(z):
return 1. / (1. + np.exp(-z))
def threshold(a):
''' Threshold given value to return 1 if greater than or equal to 0.5,
or 0 if otherwise.
'''
if a >= 0.5:
return 1
else:
return 0
h = np.vectorize(threshold, otypes=[np.dtype('int8')])
def cost(theta, X, y):
theta = np.matrix(theta)
m = X.shape[0]
z = np.dot(X, theta.T).sum(axis=1)
a = sigmoid(z)
y_hat = h(a)
cost_simple = np.add(np.multiply(y, np.log1p(y_hat)),
np.multiply((1-y), np.log1p(1-y_hat)))
reg = (LAMBD/2) * np.square(theta[:, 1:X.shape[1]]).sum()
cost = (-1./m) * (cost_simple.sum() + reg)
return cost
def gradient(theta, X, y):
theta = np.matrix(theta)
m = X.shape[0]
z = np.dot(X, theta.T).sum(axis=1)
a = sigmoid(z)
y_hat = h(a)
err = y_hat - y
derivs = np.multiply(err, X).sum(axis=0).A1 / m
derivs[1:X.shape[1]] += (LAMBD/m) * theta[:, 1:X.shape[1]].sum()
return derivs
def gradient_descent(theta, X, y):
iters = 1
m = X.shape[0]
cost_history = []
theta_history = [ np.ones(X.shape[1]) ]
while True:
theta -= ALPHA * gradient(theta, X, y)
# convergence test
last_theta = theta_history[-1]
diffs = np.sum(np.abs(np.subtract(last_theta, theta)) < DIFF_THRESHOLD)
if diffs == X.shape[1]:
print('... convergence at {} iterations'.format(iters))
break
else:
theta_history.append(np.copy(theta))
cost_history.append(cost(theta, X, y))
iters += 1
return theta, cost_history, theta_history
iris = load_iris()
np.random.seed(888)
idx_setosa = np.where(iris.target==0)[0]
idx_versicolor = np.where(iris.target==1)[0]
idx_virginica = np.where(iris.target==2)[0]
train_setosa = np.random.choice(idx_setosa, size=40, replace=False)
train_versicolor = np.random.choice(idx_versicolor, size=40, replace=False)
train_virginica = np.random.choice(idx_virginica, size=40, replace=False)
training_classes = [train_setosa, train_versicolor, train_virginica]
idx_train = np.concatenate(training_classes)
np.random.shuffle(idx_train)
idx_test = np.array([i for i in range(iris.data.shape[0]) if i not in idx_train])
np.random.shuffle(idx_test)
X = iris.data[idx_train, :]
X = np.insert(X, 0, np.ones(idx_train.size), axis=1)
classes = ['Iris-setosa', 'Iris-versicolor', 'Iris-virginica']
classifiers = []
for idx,cls in enumerate(classes):
print('\nTraining classifier for {}'.format(cls))
iters = 1
y = np.array([e in training_classes[idx] for e in idx_train], dtype=np.int8)
y = y.reshape((idx_train.size,1))
theta = np.random.normal(scale=1e-3, size=X.shape[1])
final_theta, cost_history, theta_history = gradient_descent(theta, X, y)
classifiers.append(final_theta)
classifiers = np.vstack(classifiers)
Training classifier for Iris-setosa ... convergence at 4 iterations Training classifier for Iris-versicolor ... convergence at 1625 iterations Training classifier for Iris-virginica ... convergence at 139 iterations
X_test = iris.data[idx_test, :]
X_test = np.insert(X_test, 0, np.ones(idx_test.size), axis=1)
Y = iris.target[idx_test]
correct = 0
for i,x in enumerate(X_test):
z = np.dot(classifiers, x.T)
g = sigmoid(z)
y_hat = g.argmax()
y = iris.target[idx_test[i]]
if y == y_hat:
correct += 1
print('> test {0:2d}: classifier predicted {1}, actual class is {2}'.format(i+1,
classes[y_hat],
classes[y]))
print('\n... Overall classification accuracy with gradient descent: {:.3f}'.format(correct/X_test.shape[0]))
> test 1: classifier predicted Iris-virginica, actual class is Iris-virginica > test 2: classifier predicted Iris-setosa, actual class is Iris-setosa > test 3: classifier predicted Iris-virginica, actual class is Iris-virginica > test 4: classifier predicted Iris-virginica, actual class is Iris-virginica > test 5: classifier predicted Iris-setosa, actual class is Iris-setosa > test 6: classifier predicted Iris-setosa, actual class is Iris-setosa > test 7: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 8: classifier predicted Iris-setosa, actual class is Iris-setosa > test 9: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 10: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 11: classifier predicted Iris-virginica, actual class is Iris-virginica > test 12: classifier predicted Iris-virginica, actual class is Iris-versicolor > test 13: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 14: classifier predicted Iris-virginica, actual class is Iris-virginica > test 15: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 16: classifier predicted Iris-setosa, actual class is Iris-setosa > test 17: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 18: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 19: classifier predicted Iris-setosa, actual class is Iris-setosa > test 20: classifier predicted Iris-setosa, actual class is Iris-setosa > test 21: classifier predicted Iris-setosa, actual class is Iris-setosa > test 22: classifier predicted Iris-virginica, actual class is Iris-virginica > test 23: classifier predicted Iris-virginica, actual class is Iris-virginica > test 24: classifier predicted Iris-versicolor, actual class is Iris-virginica > test 25: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 26: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 27: classifier predicted Iris-virginica, actual class is Iris-virginica > test 28: classifier predicted Iris-setosa, actual class is Iris-setosa > test 29: classifier predicted Iris-virginica, actual class is Iris-virginica > test 30: classifier predicted Iris-setosa, actual class is Iris-setosa ... Overall classification accuracy with gradient descent: 0.933
from scipy.optimize import minimize
classes = ['Iris-setosa', 'Iris-versicolor', 'Iris-virginica']
classifiers2 = []
for idx,cls in enumerate(classes):
print('\nTraining classifier for {}'.format(cls))
y = np.array([e in training_classes[idx] for e in idx_train], dtype=np.int8)
y = y.reshape((idx_train.size,1))
res = minimize(cost,
np.random.normal(scale=1e-3, size=X.shape[1]),
args=(X,y),
method='TNC',
jac=gradient,
options={'maxiter': 1000, 'disp': True})
print("success?: {}\n {}".format(res.success, res.message))
classifiers2.append(res.x)
classifiers2 = np.vstack(classifiers2)
correct = 0
for i,x in enumerate(X_test):
z = np.dot(classifiers2, x.T)
g = sigmoid(z)
y_hat = g.argmax()
y = iris.target[idx_test[i]]
if y == y_hat:
correct += 1
print('> test {:2d}: classifier predicted {}, actual class is {}'.format(i+1, classes[y_hat], classes[y]))
print('\n... Overall classification accuracy with scipy.optimize.minimize: {:.3f}'.format(correct/X_test.shape[0]))
Training classifier for Iris-setosa success?: True Local minimum reached (|pg| ~= 0) Training classifier for Iris-versicolor success?: True Converged (|f_n-f_(n-1)| ~= 0) Training classifier for Iris-virginica success?: True Converged (|f_n-f_(n-1)| ~= 0) > test 1: classifier predicted Iris-virginica, actual class is Iris-virginica > test 2: classifier predicted Iris-setosa, actual class is Iris-setosa > test 3: classifier predicted Iris-virginica, actual class is Iris-virginica > test 4: classifier predicted Iris-virginica, actual class is Iris-virginica > test 5: classifier predicted Iris-setosa, actual class is Iris-setosa > test 6: classifier predicted Iris-setosa, actual class is Iris-setosa > test 7: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 8: classifier predicted Iris-setosa, actual class is Iris-setosa > test 9: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 10: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 11: classifier predicted Iris-virginica, actual class is Iris-virginica > test 12: classifier predicted Iris-virginica, actual class is Iris-versicolor > test 13: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 14: classifier predicted Iris-virginica, actual class is Iris-virginica > test 15: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 16: classifier predicted Iris-setosa, actual class is Iris-setosa > test 17: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 18: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 19: classifier predicted Iris-setosa, actual class is Iris-setosa > test 20: classifier predicted Iris-setosa, actual class is Iris-setosa > test 21: classifier predicted Iris-setosa, actual class is Iris-setosa > test 22: classifier predicted Iris-virginica, actual class is Iris-virginica > test 23: classifier predicted Iris-virginica, actual class is Iris-virginica > test 24: classifier predicted Iris-virginica, actual class is Iris-virginica > test 25: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 26: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 27: classifier predicted Iris-virginica, actual class is Iris-virginica > test 28: classifier predicted Iris-setosa, actual class is Iris-setosa > test 29: classifier predicted Iris-virginica, actual class is Iris-virginica > test 30: classifier predicted Iris-setosa, actual class is Iris-setosa ... Overall classification accuracy with scipy.optimize.minimize: 0.967
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(solver='newton-cg',
multi_class='multinomial',
verbose=1,
max_iter=5000)
model.fit(X, iris.target[idx_train])
Y_hat = model.predict(X_test)
correct = 0
for i,y_hat in enumerate(Y_hat):
y = Y[i]
if y_hat == y:
correct += 1
print('> test {:2d}: classifier predicted {}, actual class is {}' \
.format(i+1, classes[y_hat], classes[y]))
print('\n... Overall classification accuracy with scipy.optimize.minimize: {:.3f}' \
.format(correct/X_test.shape[0]))
> test 1: classifier predicted Iris-virginica, actual class is Iris-virginica > test 2: classifier predicted Iris-setosa, actual class is Iris-setosa > test 3: classifier predicted Iris-virginica, actual class is Iris-virginica > test 4: classifier predicted Iris-virginica, actual class is Iris-virginica > test 5: classifier predicted Iris-setosa, actual class is Iris-setosa > test 6: classifier predicted Iris-setosa, actual class is Iris-setosa > test 7: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 8: classifier predicted Iris-setosa, actual class is Iris-setosa > test 9: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 10: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 11: classifier predicted Iris-virginica, actual class is Iris-virginica > test 12: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 13: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 14: classifier predicted Iris-virginica, actual class is Iris-virginica > test 15: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 16: classifier predicted Iris-setosa, actual class is Iris-setosa > test 17: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 18: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 19: classifier predicted Iris-setosa, actual class is Iris-setosa > test 20: classifier predicted Iris-setosa, actual class is Iris-setosa > test 21: classifier predicted Iris-setosa, actual class is Iris-setosa > test 22: classifier predicted Iris-virginica, actual class is Iris-virginica > test 23: classifier predicted Iris-virginica, actual class is Iris-virginica > test 24: classifier predicted Iris-virginica, actual class is Iris-virginica > test 25: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 26: classifier predicted Iris-versicolor, actual class is Iris-versicolor > test 27: classifier predicted Iris-virginica, actual class is Iris-virginica > test 28: classifier predicted Iris-setosa, actual class is Iris-setosa > test 29: classifier predicted Iris-virginica, actual class is Iris-virginica > test 30: classifier predicted Iris-setosa, actual class is Iris-setosa ... Overall classification accuracy with scipy.optimize.minimize: 1.000
[Parallel(n_jobs=1)]: Done 1 out of 1 | elapsed: 0.0s finished