Dataset¶
In [1]:
!wget https://data.mendeley.com/datasets/rscbjbr9sj/2/files/41d542e7-7f91-47f6-9ff2-dd8e5a5a7861/ChestXRay2017.zip
--2019-10-08 17:15:35-- https://data.mendeley.com/datasets/rscbjbr9sj/2/files/41d542e7-7f91-47f6-9ff2-dd8e5a5a7861/ChestXRay2017.zip Resolving data.mendeley.com (data.mendeley.com)... 104.18.24.209, 104.18.25.209, 2606:4700::6812:19d1, ... Connecting to data.mendeley.com (data.mendeley.com)|104.18.24.209|:443... connected. HTTP request sent, awaiting response... 302 Found Location: https://com-mendeley-internal.s3.amazonaws.com/platform/rdm/production/83172f6a-e8a9-404d-ad2e-8f144a17fdf2?response-content-disposition=inline%3B%20filename%3D%22ChestXRay2017.zip%22%3B%20filename%2A%3DUTF-8%27%27ChestXRay2017.zip&response-content-type=application%2Fzip&AWSAccessKeyId=AKIAI6DZXOGICLKVGYEA&Expires=1570558536&Signature=IRZLqiabdRH5qhL%2FYegG3BB1l0k%3D [following] --2019-10-08 17:15:36-- https://com-mendeley-internal.s3.amazonaws.com/platform/rdm/production/83172f6a-e8a9-404d-ad2e-8f144a17fdf2?response-content-disposition=inline%3B%20filename%3D%22ChestXRay2017.zip%22%3B%20filename%2A%3DUTF-8%27%27ChestXRay2017.zip&response-content-type=application%2Fzip&AWSAccessKeyId=AKIAI6DZXOGICLKVGYEA&Expires=1570558536&Signature=IRZLqiabdRH5qhL%2FYegG3BB1l0k%3D Resolving com-mendeley-internal.s3.amazonaws.com (com-mendeley-internal.s3.amazonaws.com)... 52.218.56.138 Connecting to com-mendeley-internal.s3.amazonaws.com (com-mendeley-internal.s3.amazonaws.com)|52.218.56.138|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 1235512464 (1.2G) [application/zip] Saving to: ‘ChestXRay2017.zip’ ChestXRay2017.zip 100%[===================>] 1.15G 31.3MB/s in 39s 2019-10-08 17:16:16 (30.6 MB/s) - ‘ChestXRay2017.zip’ saved [1235512464/1235512464]
In [0]:
!unzip -q ChestXRay2017.zip
TensorFlow¶
In [3]:
try:
%tensorflow_version 2.x
except Exception:
pass
TensorFlow 2.x selected.
In [0]:
import tensorflow as tf
In [5]:
tf.__version__
Out[5]:
'2.0.0-rc2'
In [0]:
from tensorflow.keras.optimizers import SGD, Adam
from tensorflow.keras import layers
from tensorflow.python.keras.utils.data_utils import Sequence
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import PIL
import cv2
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import confusion_matrix
import itertools
import os
Generators¶
In [0]:
class_1_len = len(os.listdir("chest_xray/train/NORMAL"))
class_2_len = len(os.listdir("chest_xray/train/PNEUMONIA"))
In [8]:
class_1_len, class_2_len
Out[8]:
(1349, 3884)
In [9]:
1 / class_1_len, 1 / class_2_len
Out[9]:
(0.0007412898443291327, 0.00025746652935118434)
In [0]:
batch_size = 32
img_size = 256
input_img_size = (256, 256, 1)
num_classes = 2
In [11]:
train_generator = ImageDataGenerator(
preprocessing_function=preprocess_input,
rotation_range=20,
zoom_range=0.05,
width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.05,
horizontal_flip=True,
fill_mode="nearest").flow_from_directory(
"chest_xray/train",
target_size=(img_size, img_size),
color_mode="grayscale",
shuffle=True,
batch_size=batch_size)
val_generator = ImageDataGenerator(
preprocessing_function=preprocess_input).flow_from_directory(
"chest_xray/test",
target_size=(img_size, img_size),
color_mode="grayscale",
shuffle=False,
batch_size=batch_size)
Found 5232 images belonging to 2 classes. Found 624 images belonging to 2 classes.
In [0]:
training_folder_size = 5232
validation_folder_size = 624
train_steps = training_folder_size // batch_size
val_steps = validation_folder_size // batch_size
In [13]:
train_steps, val_steps
Out[13]:
(163, 19)
Model¶
In [0]:
def create_vgg_model():
input = layers.Input(input_img_size)
model = layers.Conv2D(64, 3, padding='same', activation='relu')(input)
model = layers.Conv2D(64, 3, padding='same', activation='relu')(model)
model = layers.MaxPool2D(2, strides=2, padding='same')(model)
model = layers.Conv2D(128, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(128, 3, padding='same', activation='relu')(model)
model = layers.MaxPool2D(2, strides=2, padding='same')(model)
model = layers.Conv2D(256, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(256, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(256, 3, padding='same', activation='relu')(model)
model = layers.MaxPool2D(2, strides=2, padding='same')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.MaxPool2D(2, strides=2, padding='same')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.Conv2D(512, 3, padding='same', activation='relu')(model)
model = layers.GlobalAveragePooling2D()(model)
model = layers.Dense(num_classes, activation="softmax", kernel_initializer='uniform')(model)
model = Model(inputs=input, outputs=model)
return model
Find Learning Rate¶
In [0]:
class LearningRateFinder:
def __init__(self, model):
self.model = model
self.steps_per_epoch = train_steps
self.epochs = 5
self.start_lr = 1e-10
self.end_lr = 1e+1
self.learning_rates_tested = []
self.losses = []
self.batch_number = 0
self.average_loss = 0
self.best_loss = 1e9
self.beta = 0.98
self.batch_updates = self.epochs * self.steps_per_epoch
self.lr_multiplier = (self.end_lr / self.start_lr) ** (1.0 / self.batch_updates)
self.stop_factor = 4
def on_batch_end(self, batch, logs):
current_lr = tf.keras.backend.get_value(self.model.optimizer.lr)
self.learning_rates_tested.append(current_lr)
loss_value = logs["loss"]
self.batch_number += 1
self.average_loss = (self.beta * self.average_loss) + ((1 - self.beta) * loss_value)
smooth_loss = self.average_loss / (1 - (self.beta ** self.batch_number))
self.losses.append(smooth_loss)
max_loss_allowed = self.stop_factor * self.best_loss
if self.batch_number > 1 and smooth_loss > max_loss_allowed:
self.model.stop_training = True
return
if self.batch_number == 1 or smooth_loss < self.best_loss:
self.best_loss = smooth_loss
new_lr = current_lr * self.lr_multiplier
tf.keras.backend.set_value(self.model.optimizer.lr, new_lr)
def plot_model_results(self):
selected_learning_rates = self.learning_rates_tested[10:-1]
selected_losses = self.losses[10:-1]
plt.plot(selected_learning_rates, selected_losses)
plt.xscale("log")
plt.xlabel("Learning Rate")
plt.ylabel("Loss")
plt.title("learning rate finder")
def find_lr(self):
callback = tf.keras.callbacks.LambdaCallback(on_batch_end=lambda batch, logs:
self.on_batch_end(batch, logs))
optimizer = SGD(lr=self.start_lr, momentum=0.9)
self.model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"])
self.model.fit_generator(train_generator,
steps_per_epoch=self.steps_per_epoch,
epochs=self.epochs,
verbose=1,
callbacks=[callback])
self.plot_model_results()
In [0]:
test_model = None
test_model = create_vgg_model()
lr_finder = LearningRateFinder(test_model)
In [0]:
lr_finder.find_lr()
Epoch 1/5 WARNING:tensorflow:From /tensorflow-2.0.0-rc2/python3.6/tensorflow_core/python/ops/math_grad.py:1394: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.where in 2.0, which has the same broadcast rule as np.where 163/163 [==============================] - 235s 1s/step - loss: 0.6933 - accuracy: 0.2581 Epoch 2/5 163/163 [==============================] - 220s 1s/step - loss: 0.6932 - accuracy: 0.2567 Epoch 3/5 163/163 [==============================] - 220s 1s/step - loss: 0.6906 - accuracy: 0.7219 Epoch 4/5 163/163 [==============================] - 220s 1s/step - loss: 0.6100 - accuracy: 0.7437 Epoch 5/5 163/163 [==============================] - 219s 1s/step - loss: 0.6476 - accuracy: 0.7254
Cycle Learning Rate¶
In [0]:
epochs = 32
min_learning_rate = 1e-3
max_learning_rate = 1e-1
In [0]:
class CycleLearningRate(tf.keras.callbacks.Callback):
def __init__(self, model, policy="triangular"):
super(CycleLearningRate, self).__init__()
self.model = model
self.training_iterations = 0
self.cycle_iterations = 0
self.history = {}
self.step_size = 4 * train_steps
if policy == "triangular":
self.policy_fn = lambda x: 1.
else:
self.policy_fn = lambda x: 1 / (2. ** (x - 1))
def compute_learning_rate(self):
cycle = np.floor(1 + self.cycle_iterations / (2 * self.step_size))
x = np.abs(self.cycle_iterations / self.step_size - 2 * cycle + 1)
return min_learning_rate + (max_learning_rate - min_learning_rate) * np.maximum(0, (1 - x)) * self.policy_fn(cycle)
def on_train_begin(self, logs={}):
if self.cycle_iterations == 0:
tf.keras.backend.set_value(self.model.optimizer.lr, min_learning_rate)
else:
tf.keras.backend.set_value(self.model.optimizer.lr, self.compute_learning_rate())
def on_batch_end(self, batch, logs=None):
logs = logs or {}
self.training_iterations += 1
self.cycle_iterations += 1
self.history.setdefault('lr', []).append(tf.keras.backend.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', []).append(self.training_iterations)
for k, v in logs.items():
self.history.setdefault(k, []).append(v)
tf.keras.backend.set_value(self.model.optimizer.lr, self.compute_learning_rate())
CheckPoints¶
In [0]:
weights_name = "epoch={epoch:02d}|accuracy={acc:.4f}|val_accuracy={val_acc:.4f}.h5"
checkpoint = ModelCheckpoint(weights_name, monitor="val_acc", verbose=1, save_best_only=True,
save_weights_only=True, mode="max", save_freq='epoch')
Focal Loss¶
In [0]:
def categorical_focal_loss(alpha=0.25, gamma=2.0):
def focal_loss(y_true, y_pred):
crossentropy_loss = -y_true * tf.keras.backend.log(y_pred)
focal_loss = alpha * tf.keras.backend.pow(1 - y_pred, gamma) * crossentropy_loss
return tf.keras.backend.sum(focal_loss, axis=1)
return focal_loss
Training¶
In [0]:
model = None
normal_model = create_vgg_model()
cycle_lr = CycleLearningRate(normal_model, policy="triangular")
In [0]:
optimizer = SGD(lr=min_learning_rate, momentum=0.9)
normal_model.compile(loss=categorical_focal_loss(), optimizer=optimizer, metrics=['acc'])
In [0]:
trained_model = normal_model.fit_generator(train_generator,
epochs=epochs,
steps_per_epoch=train_steps,
callbacks=[checkpoint, cycle_lr],
class_weight={0:1.0, 1:0.33},
validation_data=val_generator,
validation_steps=val_steps,
verbose=1)
Epoch 1/32 162/163 [============================>.] - ETA: 1s - loss: 0.0218 - acc: 0.2976 Epoch 00001: val_acc improved from -inf to 0.38487, saving model to epoch=01|accuracy=0.2973|val_accuracy=0.3849.h5 163/163 [==============================] - 268s 2s/step - loss: 0.0218 - acc: 0.2973 - val_loss: 0.0435 - val_acc: 0.3849 Epoch 2/32 162/163 [============================>.] - ETA: 1s - loss: 0.0217 - acc: 0.3384 Epoch 00002: val_acc improved from 0.38487 to 0.61513, saving model to epoch=02|accuracy=0.3400|val_accuracy=0.6151.h5 163/163 [==============================] - 250s 2s/step - loss: 0.0217 - acc: 0.3400 - val_loss: 0.0430 - val_acc: 0.6151 Epoch 3/32 162/163 [============================>.] - ETA: 1s - loss: 0.0218 - acc: 0.3996 Epoch 00003: val_acc did not improve from 0.61513 163/163 [==============================] - 250s 2s/step - loss: 0.0218 - acc: 0.3981 - val_loss: 0.0437 - val_acc: 0.3849 Epoch 4/32 162/163 [============================>.] - ETA: 1s - loss: 0.0217 - acc: 0.4189 Epoch 00004: val_acc did not improve from 0.61513 163/163 [==============================] - 250s 2s/step - loss: 0.0217 - acc: 0.4177 - val_loss: 0.0441 - val_acc: 0.3849 Epoch 5/32 162/163 [============================>.] - ETA: 1s - loss: 0.0211 - acc: 0.5530 Epoch 00005: val_acc improved from 0.61513 to 0.65461, saving model to epoch=05|accuracy=0.5540|val_accuracy=0.6546.h5 163/163 [==============================] - 250s 2s/step - loss: 0.0211 - acc: 0.5540 - val_loss: 0.0403 - val_acc: 0.6546 Epoch 6/32 162/163 [============================>.] - ETA: 1s - loss: 0.0202 - acc: 0.5962 Epoch 00006: val_acc did not improve from 0.65461 163/163 [==============================] - 251s 2s/step - loss: 0.0202 - acc: 0.5948 - val_loss: 0.0410 - val_acc: 0.6332 Epoch 7/32 162/163 [============================>.] - ETA: 1s - loss: 0.0164 - acc: 0.7370 Epoch 00007: val_acc improved from 0.65461 to 0.81579, saving model to epoch=07|accuracy=0.7377|val_accuracy=0.8158.h5 163/163 [==============================] - 250s 2s/step - loss: 0.0163 - acc: 0.7377 - val_loss: 0.0280 - val_acc: 0.8158 Epoch 8/32 162/163 [============================>.] - ETA: 1s - loss: 0.0117 - acc: 0.8398 Epoch 00008: val_acc improved from 0.81579 to 0.82566, saving model to epoch=08|accuracy=0.8398|val_accuracy=0.8257.h5 163/163 [==============================] - 250s 2s/step - loss: 0.0117 - acc: 0.8398 - val_loss: 0.0258 - val_acc: 0.8257 Epoch 9/32 162/163 [============================>.] - ETA: 1s - loss: 0.0111 - acc: 0.8445 Epoch 00009: val_acc did not improve from 0.82566 163/163 [==============================] - 251s 2s/step - loss: 0.0111 - acc: 0.8451 - val_loss: 0.0299 - val_acc: 0.8092 Epoch 10/32 162/163 [============================>.] - ETA: 1s - loss: 0.0113 - acc: 0.8507 Epoch 00010: val_acc did not improve from 0.82566 163/163 [==============================] - 249s 2s/step - loss: 0.0112 - acc: 0.8509 - val_loss: 0.0319 - val_acc: 0.7928 Epoch 11/32 162/163 [============================>.] - ETA: 1s - loss: 0.0106 - acc: 0.8586 Epoch 00011: val_acc did not improve from 0.82566 163/163 [==============================] - 250s 2s/step - loss: 0.0106 - acc: 0.8587 - val_loss: 0.0294 - val_acc: 0.8141 Epoch 12/32 162/163 [============================>.] - ETA: 1s - loss: 0.0104 - acc: 0.8630 Epoch 00012: val_acc did not improve from 0.82566 163/163 [==============================] - 250s 2s/step - loss: 0.0104 - acc: 0.8620 - val_loss: 0.0296 - val_acc: 0.8109 Epoch 13/32 162/163 [============================>.] - ETA: 1s - loss: 0.0089 - acc: 0.8872 Epoch 00013: val_acc improved from 0.82566 to 0.86513, saving model to epoch=13|accuracy=0.8870|val_accuracy=0.8651.h5 163/163 [==============================] - 249s 2s/step - loss: 0.0089 - acc: 0.8870 - val_loss: 0.0241 - val_acc: 0.8651 Epoch 14/32 162/163 [============================>.] - ETA: 1s - loss: 0.0079 - acc: 0.9038 Epoch 00014: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0079 - acc: 0.9042 - val_loss: 0.0322 - val_acc: 0.8141 Epoch 15/32 162/163 [============================>.] - ETA: 1s - loss: 0.0074 - acc: 0.9094 Epoch 00015: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0074 - acc: 0.9092 - val_loss: 0.0257 - val_acc: 0.8421 Epoch 16/32 162/163 [============================>.] - ETA: 1s - loss: 0.0064 - acc: 0.9199 Epoch 00016: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0064 - acc: 0.9204 - val_loss: 0.0301 - val_acc: 0.8289 Epoch 17/32 162/163 [============================>.] - ETA: 1s - loss: 0.0059 - acc: 0.9265 Epoch 00017: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0059 - acc: 0.9265 - val_loss: 0.0264 - val_acc: 0.8388 Epoch 18/32 162/163 [============================>.] - ETA: 1s - loss: 0.0063 - acc: 0.9222 Epoch 00018: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0063 - acc: 0.9219 - val_loss: 0.0339 - val_acc: 0.8158 Epoch 19/32 162/163 [============================>.] - ETA: 1s - loss: 0.0075 - acc: 0.9100 Epoch 00019: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0074 - acc: 0.9100 - val_loss: 0.0372 - val_acc: 0.7845 Epoch 20/32 162/163 [============================>.] - ETA: 1s - loss: 0.0073 - acc: 0.9084 Epoch 00020: val_acc did not improve from 0.86513 163/163 [==============================] - 250s 2s/step - loss: 0.0073 - acc: 0.9089 - val_loss: 0.0492 - val_acc: 0.7007 Epoch 21/32 162/163 [============================>.] - ETA: 1s - loss: 0.0069 - acc: 0.9156 Epoch 00021: val_acc did not improve from 0.86513 163/163 [==============================] - 248s 2s/step - loss: 0.0069 - acc: 0.9153 - val_loss: 0.0271 - val_acc: 0.8470 Epoch 22/32 162/163 [============================>.] - ETA: 1s - loss: 0.0062 - acc: 0.9236 Epoch 00022: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0062 - acc: 0.9231 - val_loss: 0.0238 - val_acc: 0.8553 Epoch 23/32 162/163 [============================>.] - ETA: 1s - loss: 0.0069 - acc: 0.9172 Epoch 00023: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0069 - acc: 0.9177 - val_loss: 0.0274 - val_acc: 0.8454 Epoch 24/32 162/163 [============================>.] - ETA: 1s - loss: 0.0053 - acc: 0.9367 Epoch 00024: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0053 - acc: 0.9371 - val_loss: 0.0294 - val_acc: 0.8306 Epoch 25/32 162/163 [============================>.] - ETA: 1s - loss: 0.0052 - acc: 0.9363 Epoch 00025: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0052 - acc: 0.9365 - val_loss: 0.0260 - val_acc: 0.8421 Epoch 26/32 162/163 [============================>.] - ETA: 1s - loss: 0.0058 - acc: 0.9263 Epoch 00026: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0058 - acc: 0.9262 - val_loss: 0.0300 - val_acc: 0.8141 Epoch 27/32 162/163 [============================>.] - ETA: 1s - loss: 0.0066 - acc: 0.9212 Epoch 00027: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0066 - acc: 0.9210 - val_loss: 0.0249 - val_acc: 0.8421 Epoch 28/32 162/163 [============================>.] - ETA: 1s - loss: 0.0066 - acc: 0.9178 Epoch 00028: val_acc did not improve from 0.86513 163/163 [==============================] - 250s 2s/step - loss: 0.0066 - acc: 0.9175 - val_loss: 0.0254 - val_acc: 0.8322 Epoch 29/32 162/163 [============================>.] - ETA: 1s - loss: 0.0062 - acc: 0.9238 Epoch 00029: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0062 - acc: 0.9240 - val_loss: 0.0331 - val_acc: 0.8141 Epoch 30/32 162/163 [============================>.] - ETA: 1s - loss: 0.0060 - acc: 0.9284 Epoch 00030: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0059 - acc: 0.9288 - val_loss: 0.0296 - val_acc: 0.8191 Epoch 31/32 162/163 [============================>.] - ETA: 1s - loss: 0.0050 - acc: 0.9402 Epoch 00031: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0051 - acc: 0.9394 - val_loss: 0.0342 - val_acc: 0.8109 Epoch 32/32 162/163 [============================>.] - ETA: 1s - loss: 0.0048 - acc: 0.9389 Epoch 00032: val_acc did not improve from 0.86513 163/163 [==============================] - 249s 2s/step - loss: 0.0049 - acc: 0.9387 - val_loss: 0.0286 - val_acc: 0.8306
Results¶
In [0]:
def plot_validation_training(metric, trained_model):
validation_metric = trained_model.history[f'val_{metric}']
training_metric = trained_model.history[metric]
epochs = range(len(training_metric))
plt.plot(epochs, training_metric, 'b', label=f'Training {metric}')
plt.plot(epochs, validation_metric, 'r', label=f'Validation {metric}')
plt.ylim(bottom=0)
plt.xlabel('Epochs ', fontsize=16)
plt.ylabel(metric, fontsize=16)
loc = 'upper right' if metric == "loss" else 'lower right'
plt.legend(loc=loc)
plt.title(f'Training and validation {metric}', fontsize=20)
plt.show()
In [0]:
plot_validation_training("loss", trained_model)
In [0]:
plot_validation_training("acc", trained_model)
In [0]:
iterations = np.arange(0, len(cycle_lr.history["lr"]))
plt.figure()
plt.plot(iterations, cycle_lr.history["lr"])
plt.title("Cyclical Learning Rate (CLR)")
plt.xlabel("Training Iterations")
plt.ylabel("Learning Rate")
plt.show()
Confusion Matrix¶
In [0]:
normal_model.load_weights("/content/epoch=13_accuracy=0.8870_val_accuracy=0.8651.h5")
In [0]:
def plot_confusion_matrix(cm, classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.tight_layout()
In [0]:
val_generator.reset()
In [22]:
val_generator.class_indices
Out[22]:
{'NORMAL': 0, 'PNEUMONIA': 1}
In [0]:
def test_model(model, generator):
predicted_classes = model.predict_generator(generator, verbose=1)
max_class_predicted = np.argmax(predicted_classes, axis=1)
classes_names = ['NORMAL', 'PNEUMONIA']
labels = val_generator.classes
cm = confusion_matrix(labels, max_class_predicted, labels=range(num_classes))
plot_confusion_matrix(cm, classes_names)
sensitivity = cm[0, 0] / (cm[0, 0] + cm[0, 1])
print(f'sensitivity: {sensitivity}')
specificity = cm[1, 1] / (cm[1, 1] + cm[1, 0])
print(f'specifity: {specificity}')
incorrect_labels = np.where(max_class_predicted != labels)[0]
return max_class_predicted, incorrect_labels
In [0]:
val_class_predicted, val_incorrect_labels = test_model(normal_model, val_generator)
20/20 [==============================] - 10s 481ms/step sensitivity: 0.8333333333333334 specifity: 0.8846153846153846
In [0]:
from sklearn.metrics import classification_report
In [0]:
labels = val_generator.classes
classes_names = ['NORMAL', 'PNEUMONIA']
In [0]:
report = classification_report(val_class_predicted, labels, target_names=classes_names)
print(report)
precision recall f1-score support
NORMAL 0.83 0.81 0.82 240
PNEUMONIA 0.88 0.90 0.89 384
accuracy 0.87 624
macro avg 0.86 0.86 0.86 624
weighted avg 0.86 0.87 0.87 624
Check predictions¶
In [0]:
normal_path = "chest_xray/test/NORMAL/"
pneumonia_path = "chest_xray/test/PNEUMONIA/"
In [0]:
validation_images_paths = [normal_path + img_path for img_path in os.listdir(normal_path)] + [pneumonia_path + img_path for img_path in os.listdir(pneumonia_path)]
In [0]:
print("Found %d incorrect labels" % len(val_incorrect_labels))
In [0]:
def load_image(img_path):
img = PIL.Image.open(img_path)
img = img.resize((128, 128))
img = np.asarray(img)
return img
In [0]:
def check_predictions(from_num, to_num):
plt.figure(figsize=(15, 15))
for index, incorrect_label in enumerate(val_incorrect_labels[from_num:to_num]):
plt.subplot(3, 3, index + 1)
img = load_image(validation_images_paths[incorrect_label])
plt.imshow(img, cmap='gray')
plt.title("Predicted {}, Class {}".format(val_class_predicted[incorrect_label], val_generator.classes[incorrect_label]))
plt.tight_layout()
In [0]:
check_predictions(0, 9)