Computing the posterior probability of $X$ being from class $c$ using Bayes rule $$ P(y_c|X)= \frac{P(X|y_c)P(y_c)}{P(X)}$$ From estimated prior probability of class $$P(y_c)$$ and probabity of $X$ $$P(X|y_c)$$
Assumption for this classifier is that data is gaussian.
Implimentation of NaiveBayes classifier is done following above equations only. Source code is *probabilistic* library.
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from spkit.ml import NaiveBayes
data = datasets.load_iris()
X = data.data
y = data.target
Xt,Xs,yt,ys = train_test_split(X,y,test_size=0.3)
print(Xt.shape,yt.shape,Xs.shape,ys.shape)
(105, 4) (105,) (45, 4) (45,)
clf = NaiveBayes()
clf.fit(Xt,yt)
ytp = clf.predict(Xt)
ysp = clf.predict(Xs)
print('Training Accuracy : ',np.mean(ytp==yt))
print('Testing Accuracy : ',np.mean(ysp==ys))
Training Accuracy : 0.9714285714285714 Testing Accuracy : 0.9333333333333333
ytpr = clf.predict_prob(Xt)
yspr = clf.predict_prob(Xs)
ytpr[0]
array([2.22677564e-09, 6.88449314e-02, 9.31155066e-01])
clf.predict(Xs[0]), clf.predict_prob(Xs[0])
(array([1]), array([[9.86455769e-10, 9.99992322e-01, 7.67736032e-06]]))
clf.parameters
{0: {'mu': array([4.98666667, 3.41666667, 1.47333333, 0.26666667]), 'sig': array([0.07982222, 0.15072222, 0.03195556, 0.01488889]), 'prior': 0.2857142857142857}, 1: {'mu': array([5.87894737, 2.79210526, 4.27894737, 1.35 ]), 'sig': array([0.27271468, 0.09335873, 0.24218837, 0.03986842]), 'prior': 0.3619047619047619}, 2: {'mu': array([6.64594595, 2.99189189, 5.59189189, 2.08108108]), 'sig': array([0.35707816, 0.10236669, 0.30560993, 0.06585829]), 'prior': 0.3523809523809524}}
Setting the names of classes and features (Optional)
clf.set_class_labels(data['target_names'])
clf.set_feature_names(data['feature_names'])
fig = plt.figure(figsize=(12,10))
clf.VizPx()
data = datasets.load_breast_cancer()
X = data.data
y = data.target
Xt,Xs,yt,ys = train_test_split(X,y,test_size=0.3)
print(Xt.shape,yt.shape,Xs.shape,ys.shape)
(398, 30) (398,) (171, 30) (171,)
clf = NaiveBayes()
clf.fit(Xt,yt)
ytp = clf.predict(Xt)
ysp = clf.predict(Xs)
print('Training Accuracy : ',np.mean(ytp==yt))
print('Testing Accuracy : ',np.mean(ysp==ys))
Training Accuracy : 0.9321608040201005 Testing Accuracy : 0.9649122807017544
clf.parameters[0] # class 0
{'mu': array([1.72674497e+01, 2.17200000e+01, 1.13917315e+02, 9.53271141e+02, 1.02753624e-01, 1.41000872e-01, 1.55203691e-01, 8.57557047e-02, 1.92042282e-01, 6.25029530e-02, 5.90004698e-01, 1.21125772e+00, 4.18012081e+00, 6.86748993e+01, 6.93873154e-03, 3.11968859e-02, 4.08352349e-02, 1.48303356e-02, 2.00738859e-02, 3.99146980e-03, 2.08057718e+01, 2.94116779e+01, 1.39071611e+02, 1.36938725e+03, 1.45458322e-01, 3.62481275e-01, 4.39459597e-01, 1.79128926e-01, 3.22568456e-01, 9.06604698e-02]), 'sig': array([9.19824316e+00, 1.27991597e+01, 4.27412041e+02, 1.14720796e+05, 1.60285507e-04, 2.79468421e-03, 5.60475988e-03, 1.16365586e-03, 7.95894218e-04, 5.22823946e-05, 1.08994705e-01, 2.27143052e-01, 5.94456248e+00, 2.98445515e+03, 9.94951468e-06, 2.93011232e-04, 4.58808910e-04, 2.71969209e-05, 7.30270203e-05, 3.48874832e-06, 1.55560727e+01, 2.85407039e+01, 7.26467479e+02, 2.70232448e+05, 4.68596654e-04, 2.51556031e-02, 3.22737993e-02, 2.34531931e-03, 5.68726441e-03, 3.92649178e-04]), 'prior': 0.3743718592964824}
clf.set_class_labels(data['target_names'])
#clf.set_feature_names(data['feature_names'])
fig = plt.figure(figsize=(12,10))
clf.VizPx(nfeatures=range(16))
fig = plt.figure(figsize=(12,10))
clf.VizPx(nfeatures=range(16,30))
data = datasets.load_digits()
X = data.data
y = data.target
# Avoiding features with zero variance (contant value)
#X = X[:,X.var(0)>0]
Xt,Xs,yt,ys = train_test_split(X,y,test_size=0.3)
print(Xt.shape,yt.shape,Xs.shape,ys.shape)
(1257, 64) (1257,) (540, 64) (540,)
clf = NaiveBayes()
clf.fit(Xt,yt)
ytp = clf.predict(Xt)
ysp = clf.predict(Xs)
print('Training Accuracy : ',np.mean(ytp==yt))
print('Testing Accuracy : ',np.mean(ysp==ys))
Training Accuracy : 0.7883850437549722 Testing Accuracy : 0.7722222222222223
clf.predict(Xs[0]), clf.predict_prob(Xs[0])
(array([5]), array([[4.28401634e-27, 4.28401634e-27, 4.28401634e-27, 4.28401634e-27, 4.28401634e-27, 1.00000000e+00, 4.28401634e-27, 4.28401634e-27, 1.78497054e-19, 4.28401634e-27]]))
plt.imshow(Xs[0].reshape([8,8]),cmap='gray')
plt.axis('off')
plt.show()
print('Prediction',clf.predict(Xs[0]))
Prediction [5]
fig = plt.figure(figsize=(12,10))
clf.VizPx(nfeatures=range(5,19))