%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.vision import *
path = untar_data(URLs.MNIST)
path.ls()
[PosixPath('/home/ubuntu/.fastai/data/mnist_png/testing'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training')]
# PIL convert mode "L" means translating a color image to black and white
# Reference: https://pillow.readthedocs.io/en/3.1.x/handbook/concepts.html#concept-modes
il = ImageItemList.from_folder(path, convert_mode='L')
il.items[0]
PosixPath('/home/ubuntu/.fastai/data/mnist_png/testing/1/4010.png')
defaults.cmap = 'binary'
il
ImageItemList (70000 items) [Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28)]... Path: /home/ubuntu/.fastai/data/mnist_png
il[0].show()
sd = il.split_by_folder(train='training', valid='testing')
# Sanity check
type(sd)
fastai.data_block.ItemLists
sd
ItemLists; Train: ImageItemList (60000 items) [Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28)]... Path: /home/ubuntu/.fastai/data/mnist_png; Valid: ImageItemList (10000 items) [Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28)]... Path: /home/ubuntu/.fastai/data/mnist_png; Test: None
(path / 'training').ls()
[PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/1'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/8'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/9'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/6'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/7'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/2'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/0'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/3'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/4'), PosixPath('/home/ubuntu/.fastai/data/mnist_png/training/5')]
ll = sd.label_from_folder()
ll
LabelLists; Train: LabelList y: CategoryList (60000 items) [Category 1, Category 1, Category 1, Category 1, Category 1]... Path: /home/ubuntu/.fastai/data/mnist_png x: ImageItemList (60000 items) [Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28)]... Path: /home/ubuntu/.fastai/data/mnist_png; Valid: LabelList y: CategoryList (10000 items) [Category 1, Category 1, Category 1, Category 1, Category 1]... Path: /home/ubuntu/.fastai/data/mnist_png x: ImageItemList (10000 items) [Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28), Image (1, 28, 28)]... Path: /home/ubuntu/.fastai/data/mnist_png; Test: None
# Sanity check
type(ll.train), type(ll.train[0])
(fastai.data_block.LabelList, tuple)
x, y = ll.train[0]
# Sanity check
type(x), type(y)
(fastai.vision.image.Image, fastai.core.Category)
x.show()
print(y, x.shape)
1 torch.Size([1, 28, 28])
# first list is for train set, second empty list is for test set
# What does the star operator mean? unpack the list
tfms = ([*rand_pad(padding=3, size=28, mode='zeros')], []) # pad and crop. crop size 28px, pad mode 'zeros' (default relection)
ll = ll.transform(tfms)
bs = 128
What is DataBunch? From lesson 1:
a general fastai concept for your data, and from there, there are subclasses for particular applications like ImageDataBunch
.
dataset (i.e. self.train) -> dataloader (self.train_dl, self.train_ds)
# not using imagenet_stats because not using pretrained model
data = ll.databunch(bs=bs).normalize()
doc(ll.databunch)
# Sanity check
type(data)
fastai.vision.data.ImageDataBunch
x, y = data.train_ds[0]
x.show()
print(y)
1
# Sanity check
type(data.train_ds[0][0])
fastai.vision.image.Image
def _plot(i, j, ax): data.train_ds[0][0].show(ax, cmap='gray')
plot_multi(_plot, 3, 3, figsize=(8,8))
doc(data.one_batch)
# Get one batch from the data loader of `ds_type`. Optionally `detach` and `denorm`. Returns a Collection[Tensor]
# detach means move tensor from GPU to CPU.
xb, yb = data.one_batch()
xb.shape, yb.shape
(torch.Size([128, 1, 28, 28]), torch.Size([128]))
data.show_batch(rows=3, figsize=(5,5))
# ni is num input
# nf is num filter
def conv(ni, nf): return nn.Conv2d(ni, nf, kernel_size=3, stride=2, padding=1)
# PyTorch sequential container
model = nn.Sequential(
conv(1, 8), # 14
nn.BatchNorm2d(8),
nn.ReLU(),
conv(8, 16), # 7
nn.BatchNorm2d(16),
nn.ReLU(),
conv(16, 32), # 4
nn.BatchNorm2d(32),
nn.ReLU(),
conv(32, 16), # 2
nn.BatchNorm2d(16),
nn.ReLU(),
conv(16, 10), # 1
nn.BatchNorm2d(10),
Flatten() # remove (1,1) grid. 1 is unit axis
)
learn = Learner(data, model, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
# Sanity check
type(learn)
fastai.basic_train.Learner
print(learn.summary())
====================================================================== Layer (type) Output Shape Param # Trainable ====================================================================== Conv2d [128, 8, 14, 14] 80 True ______________________________________________________________________ BatchNorm2d [128, 8, 14, 14] 16 True ______________________________________________________________________ ReLU [128, 8, 14, 14] 0 False ______________________________________________________________________ Conv2d [128, 16, 7, 7] 1168 True ______________________________________________________________________ BatchNorm2d [128, 16, 7, 7] 32 True ______________________________________________________________________ ReLU [128, 16, 7, 7] 0 False ______________________________________________________________________ Conv2d [128, 32, 4, 4] 4640 True ______________________________________________________________________ BatchNorm2d [128, 32, 4, 4] 64 True ______________________________________________________________________ ReLU [128, 32, 4, 4] 0 False ______________________________________________________________________ Conv2d [128, 16, 2, 2] 4624 True ______________________________________________________________________ BatchNorm2d [128, 16, 2, 2] 32 True ______________________________________________________________________ ReLU [128, 16, 2, 2] 0 False ______________________________________________________________________ Conv2d [128, 10, 1, 1] 1450 True ______________________________________________________________________ BatchNorm2d [128, 10, 1, 1] 20 True ______________________________________________________________________ Lambda [128, 10] 0 False ______________________________________________________________________ Total params: 12126 Total trainable params: 12126 Total non-trainable params: 0
xb = xb.cuda()
model(xb).shape
torch.Size([128, 10])
learn.lr_find(end_lr=100)
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
learn.fit_one_cycle(3, max_lr=0.1)
epoch | train_loss | valid_loss | accuracy |
---|---|---|---|
1 | 0.222742 | 0.537284 | 0.842800 |
2 | 0.136951 | 0.081584 | 0.975300 |
3 | 0.075597 | 0.041812 | 0.986900 |
def conv2(ni, nf): return conv_layer(ni, nf, stride=2)
model = nn.Sequential(
conv(1, 8), # 14
conv(8, 16), # 7
conv(16, 32), # 4
conv(32, 16), # 2
conv(16, 10), # 1
Flatten() # remove (1,1) grid
)
learn = Learner(data, model, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.fit_one_cycle(10, max_lr=0.1)
epoch | train_loss | valid_loss | accuracy |
---|---|---|---|
1 | 1.493621 | 1.083352 | 0.668000 |
2 | 1.679995 | 0.914248 | 0.715200 |
3 | 13845.543945 | 637.649170 | 0.097400 |
4 | 3.488301 | 1.606864 | 0.502200 |
5 | 1.451810 | 0.885681 | 0.717800 |
6 | 1.329285 | 0.821450 | 0.767700 |
7 | 1.520882 | 1.571306 | 0.486200 |
8 | 1.294403 | 0.784840 | 0.787100 |
9 | 1.193693 | 0.783441 | 0.774000 |
10 | 1.110655 | 0.729030 | 0.797700 |
class ResBlock(nn.Module):
def __init__(self, nf):
super().__init__()
self.conv1 = conv_layer(nf, nf)
self.conv2 = conv_layer(nf, nf)
def forward(self, x): return x + self.conv2(self.conv1(x))
help(res_block)
Help on function res_block in module fastai.layers: res_block(nf, dense:bool=False, norm_type:Union[fastai.layers.NormType, NoneType]=<NormType.Batch: 1>, bottle:bool=False, **kwargs) Resnet block of `nf` features.
model = nn.Sequential(
conv2(1, 8),
res_block(8),
conv2(8, 16),
res_block(16),
conv2(16, 32),
res_block(32),
conv2(32, 16),
res_block(16),
conv2(16, 10),
Flatten()
)
def conv_and_res(ni, nf): return nn.Sequential(conv2(ni, nf), res_block(nf))
model = nn.Sequential(
conv_and_res(1, 8),
conv_and_res(8, 16),
conv_and_res(16, 32),
conv_and_res(32, 16),
conv2(16, 10),
Flatten()
)
learn = Learner(data, model, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)
learn.lr_find(end_lr=100)
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
learn.recorder.plot()
learn.fit_one_cycle(12, max_lr=0.05)
epoch | train_loss | valid_loss | accuracy |
---|---|---|---|
1 | 0.264350 | 0.197537 | 0.953900 |
2 | 0.141571 | 0.100534 | 0.969200 |
3 | 0.110794 | 0.221931 | 0.936400 |
4 | 0.095346 | 0.106152 | 0.968800 |
5 | 0.072221 | 0.039760 | 0.987500 |
6 | 0.063750 | 0.058403 | 0.981500 |
7 | 0.052838 | 0.044521 | 0.987300 |
8 | 0.042057 | 0.026808 | 0.990600 |
9 | 0.037357 | 0.034371 | 0.990200 |
10 | 0.028304 | 0.018587 | 0.994500 |
11 | 0.019751 | 0.016416 | 0.994300 |
12 | 0.019853 | 0.015870 | 0.994800 |
print(learn.summary())
====================================================================== Layer (type) Output Shape Param # Trainable ====================================================================== Conv2d [128, 8, 14, 14] 72 True ______________________________________________________________________ ReLU [128, 8, 14, 14] 0 False ______________________________________________________________________ BatchNorm2d [128, 8, 14, 14] 16 True ______________________________________________________________________ Conv2d [128, 8, 14, 14] 576 True ______________________________________________________________________ ReLU [128, 8, 14, 14] 0 False ______________________________________________________________________ BatchNorm2d [128, 8, 14, 14] 16 True ______________________________________________________________________ Conv2d [128, 8, 14, 14] 576 True ______________________________________________________________________ ReLU [128, 8, 14, 14] 0 False ______________________________________________________________________ BatchNorm2d [128, 8, 14, 14] 16 True ______________________________________________________________________ MergeLayer [128, 8, 14, 14] 0 False ______________________________________________________________________ Conv2d [128, 16, 7, 7] 1152 True ______________________________________________________________________ ReLU [128, 16, 7, 7] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 7, 7] 32 True ______________________________________________________________________ Conv2d [128, 16, 7, 7] 2304 True ______________________________________________________________________ ReLU [128, 16, 7, 7] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 7, 7] 32 True ______________________________________________________________________ Conv2d [128, 16, 7, 7] 2304 True ______________________________________________________________________ ReLU [128, 16, 7, 7] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 7, 7] 32 True ______________________________________________________________________ MergeLayer [128, 16, 7, 7] 0 False ______________________________________________________________________ Conv2d [128, 32, 4, 4] 4608 True ______________________________________________________________________ ReLU [128, 32, 4, 4] 0 False ______________________________________________________________________ BatchNorm2d [128, 32, 4, 4] 64 True ______________________________________________________________________ Conv2d [128, 32, 4, 4] 9216 True ______________________________________________________________________ ReLU [128, 32, 4, 4] 0 False ______________________________________________________________________ BatchNorm2d [128, 32, 4, 4] 64 True ______________________________________________________________________ Conv2d [128, 32, 4, 4] 9216 True ______________________________________________________________________ ReLU [128, 32, 4, 4] 0 False ______________________________________________________________________ BatchNorm2d [128, 32, 4, 4] 64 True ______________________________________________________________________ MergeLayer [128, 32, 4, 4] 0 False ______________________________________________________________________ Conv2d [128, 16, 2, 2] 4608 True ______________________________________________________________________ ReLU [128, 16, 2, 2] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 2, 2] 32 True ______________________________________________________________________ Conv2d [128, 16, 2, 2] 2304 True ______________________________________________________________________ ReLU [128, 16, 2, 2] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 2, 2] 32 True ______________________________________________________________________ Conv2d [128, 16, 2, 2] 2304 True ______________________________________________________________________ ReLU [128, 16, 2, 2] 0 False ______________________________________________________________________ BatchNorm2d [128, 16, 2, 2] 32 True ______________________________________________________________________ MergeLayer [128, 16, 2, 2] 0 False ______________________________________________________________________ Conv2d [128, 10, 1, 1] 1440 True ______________________________________________________________________ ReLU [128, 10, 1, 1] 0 False ______________________________________________________________________ BatchNorm2d [128, 10, 1, 1] 20 True ______________________________________________________________________ Lambda [128, 10] 0 False ______________________________________________________________________ Total params: 41132 Total trainable params: 41132 Total non-trainable params: 0