Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p torch
Sebastian Raschka CPython 3.6.8 IPython 7.2.0 torch 1.0.0
A convolutional autoencoder using nearest neighbor upscaling layers that compresses 768-pixel MNIST images down to a 7x7x8 (392 pixel) representation.
import time
import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
##########################
### SETTINGS
##########################
# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Device:', device)
# Hyperparameters
random_seed = 123
learning_rate = 0.05
num_epochs = 10
batch_size = 128
##########################
### MNIST DATASET
##########################
# Note transforms.ToTensor() scales input images
# to 0-1 range
train_dataset = datasets.MNIST(root='data',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.MNIST(root='data',
train=False,
transform=transforms.ToTensor())
train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
# Checking the dataset
for images, labels in train_loader:
print('Image batch dimensions:', images.shape)
print('Image label dimensions:', labels.shape)
break
Device: cuda:0 Image batch dimensions: torch.Size([128, 1, 28, 28]) Image label dimensions: torch.Size([128])
##########################
### MODEL
##########################
class ConvolutionalAutoencoder(torch.nn.Module):
def __init__(self):
super(ConvolutionalAutoencoder, self).__init__()
# calculate same padding:
# (w - k + 2*p)/s + 1 = o
# => p = (s(o-1) - w + k)/2
### ENCODER
# 28x28x1 => 28x28x4
self.conv_1 = torch.nn.Conv2d(in_channels=1,
out_channels=4,
kernel_size=(3, 3),
stride=(1, 1),
# (1(28-1) - 28 + 3) / 2 = 1
padding=1)
# 28x28x4 => 14x14x4
self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2),
# (2(14-1) - 28 + 2) / 2 = 0
padding=0)
# 14x14x4 => 14x14x8
self.conv_2 = torch.nn.Conv2d(in_channels=4,
out_channels=8,
kernel_size=(3, 3),
stride=(1, 1),
# (1(14-1) - 14 + 3) / 2 = 1
padding=1)
# 14x14x8 => 7x7x8
self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),
stride=(2, 2),
# (2(7-1) - 14 + 2) / 2 = 0
padding=0)
### DECODER
# 7x7x8 => 14x14x8
# Unpool
# 14x14x8 => 14x14x8
self.conv_3 = torch.nn.Conv2d(in_channels=8,
out_channels=4,
kernel_size=(3, 3),
stride=(1, 1),
# (1(14-1) - 14 + 3) / 2 = 1
padding=1)
# 14x14x4 => 28x28x4
# Unpool
# 28x28x4 => 28x28x1
self.conv_4 = torch.nn.Conv2d(in_channels=4,
out_channels=1,
kernel_size=(3, 3),
stride=(1, 1),
# (1(28-1) - 28 + 3) / 2 = 1
padding=1)
def forward(self, x):
### ENCODER
x = self.conv_1(x)
x = F.leaky_relu(x)
x = self.pool_1(x)
x = self.conv_2(x)
x = F.leaky_relu(x)
x = self.pool_2(x)
### DECODER
x = F.interpolate(x, scale_factor=2, mode='nearest')
x = self.conv_3(x)
x = F.leaky_relu(x)
x = F.interpolate(x, scale_factor=2, mode='nearest')
x = self.conv_4(x)
logits = F.leaky_relu(x)
probas = torch.sigmoid(logits)
return logits, probas
torch.manual_seed(random_seed)
model = ConvolutionalAutoencoder()
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
start_time = time.time()
for epoch in range(num_epochs):
for batch_idx, (features, targets) in enumerate(train_loader):
# don't need labels, only the images (features)
features = features.to(device)
### FORWARD AND BACK PROP
logits, decoded = model(features)
cost = F.binary_cross_entropy_with_logits(logits, features)
optimizer.zero_grad()
cost.backward()
### UPDATE MODEL PARAMETERS
optimizer.step()
### LOGGING
if not batch_idx % 50:
print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f'
%(epoch+1, num_epochs, batch_idx,
len(train_loader), cost))
print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))
print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))
Epoch: 001/010 | Batch 000/469 | Cost: 0.7008 Epoch: 001/010 | Batch 050/469 | Cost: 0.1782 Epoch: 001/010 | Batch 100/469 | Cost: 0.1394 Epoch: 001/010 | Batch 150/469 | Cost: 0.1240 Epoch: 001/010 | Batch 200/469 | Cost: 0.1173 Epoch: 001/010 | Batch 250/469 | Cost: 0.1105 Epoch: 001/010 | Batch 300/469 | Cost: 0.1045 Epoch: 001/010 | Batch 350/469 | Cost: 0.1060 Epoch: 001/010 | Batch 400/469 | Cost: 0.1051 Epoch: 001/010 | Batch 450/469 | Cost: 0.1002 Time elapsed: 0.12 min Epoch: 002/010 | Batch 000/469 | Cost: 0.0998 Epoch: 002/010 | Batch 050/469 | Cost: 0.1023 Epoch: 002/010 | Batch 100/469 | Cost: 0.1018 Epoch: 002/010 | Batch 150/469 | Cost: 0.0967 Epoch: 002/010 | Batch 200/469 | Cost: 0.0982 Epoch: 002/010 | Batch 250/469 | Cost: 0.0985 Epoch: 002/010 | Batch 300/469 | Cost: 0.0961 Epoch: 002/010 | Batch 350/469 | Cost: 0.0998 Epoch: 002/010 | Batch 400/469 | Cost: 0.0975 Epoch: 002/010 | Batch 450/469 | Cost: 0.0967 Time elapsed: 0.24 min Epoch: 003/010 | Batch 000/469 | Cost: 0.0953 Epoch: 003/010 | Batch 050/469 | Cost: 0.0921 Epoch: 003/010 | Batch 100/469 | Cost: 0.0955 Epoch: 003/010 | Batch 150/469 | Cost: 0.0960 Epoch: 003/010 | Batch 200/469 | Cost: 0.0959 Epoch: 003/010 | Batch 250/469 | Cost: 0.0929 Epoch: 003/010 | Batch 300/469 | Cost: 0.0952 Epoch: 003/010 | Batch 350/469 | Cost: 0.0935 Epoch: 003/010 | Batch 400/469 | Cost: 0.0939 Epoch: 003/010 | Batch 450/469 | Cost: 0.0946 Time elapsed: 0.36 min Epoch: 004/010 | Batch 000/469 | Cost: 0.0931 Epoch: 004/010 | Batch 050/469 | Cost: 0.0965 Epoch: 004/010 | Batch 100/469 | Cost: 0.0900 Epoch: 004/010 | Batch 150/469 | Cost: 0.0883 Epoch: 004/010 | Batch 200/469 | Cost: 0.0957 Epoch: 004/010 | Batch 250/469 | Cost: 0.0956 Epoch: 004/010 | Batch 300/469 | Cost: 0.0885 Epoch: 004/010 | Batch 350/469 | Cost: 0.0901 Epoch: 004/010 | Batch 400/469 | Cost: 0.0943 Epoch: 004/010 | Batch 450/469 | Cost: 0.0933 Time elapsed: 0.48 min Epoch: 005/010 | Batch 000/469 | Cost: 0.0885 Epoch: 005/010 | Batch 050/469 | Cost: 0.0953 Epoch: 005/010 | Batch 100/469 | Cost: 0.0896 Epoch: 005/010 | Batch 150/469 | Cost: 0.0944 Epoch: 005/010 | Batch 200/469 | Cost: 0.0935 Epoch: 005/010 | Batch 250/469 | Cost: 0.0914 Epoch: 005/010 | Batch 300/469 | Cost: 0.0898 Epoch: 005/010 | Batch 350/469 | Cost: 0.0883 Epoch: 005/010 | Batch 400/469 | Cost: 0.0906 Epoch: 005/010 | Batch 450/469 | Cost: 0.0924 Time elapsed: 0.60 min Epoch: 006/010 | Batch 000/469 | Cost: 0.0862 Epoch: 006/010 | Batch 050/469 | Cost: 0.0881 Epoch: 006/010 | Batch 100/469 | Cost: 0.0879 Epoch: 006/010 | Batch 150/469 | Cost: 0.0847 Epoch: 006/010 | Batch 200/469 | Cost: 0.0922 Epoch: 006/010 | Batch 250/469 | Cost: 0.0871 Epoch: 006/010 | Batch 300/469 | Cost: 0.0893 Epoch: 006/010 | Batch 350/469 | Cost: 0.0900 Epoch: 006/010 | Batch 400/469 | Cost: 0.0843 Epoch: 006/010 | Batch 450/469 | Cost: 0.0856 Time elapsed: 0.72 min Epoch: 007/010 | Batch 000/469 | Cost: 0.0898 Epoch: 007/010 | Batch 050/469 | Cost: 0.0840 Epoch: 007/010 | Batch 100/469 | Cost: 0.0898 Epoch: 007/010 | Batch 150/469 | Cost: 0.0928 Epoch: 007/010 | Batch 200/469 | Cost: 0.0920 Epoch: 007/010 | Batch 250/469 | Cost: 0.0864 Epoch: 007/010 | Batch 300/469 | Cost: 0.0860 Epoch: 007/010 | Batch 350/469 | Cost: 0.0839 Epoch: 007/010 | Batch 400/469 | Cost: 0.0856 Epoch: 007/010 | Batch 450/469 | Cost: 0.0843 Time elapsed: 0.84 min Epoch: 008/010 | Batch 000/469 | Cost: 0.0817 Epoch: 008/010 | Batch 050/469 | Cost: 0.0858 Epoch: 008/010 | Batch 100/469 | Cost: 0.0894 Epoch: 008/010 | Batch 150/469 | Cost: 0.0897 Epoch: 008/010 | Batch 200/469 | Cost: 0.0815 Epoch: 008/010 | Batch 250/469 | Cost: 0.0892 Epoch: 008/010 | Batch 300/469 | Cost: 0.0859 Epoch: 008/010 | Batch 350/469 | Cost: 0.0883 Epoch: 008/010 | Batch 400/469 | Cost: 0.0834 Epoch: 008/010 | Batch 450/469 | Cost: 0.0940 Time elapsed: 0.95 min Epoch: 009/010 | Batch 000/469 | Cost: 0.0888 Epoch: 009/010 | Batch 050/469 | Cost: 0.0856 Epoch: 009/010 | Batch 100/469 | Cost: 0.0881 Epoch: 009/010 | Batch 150/469 | Cost: 0.0853 Epoch: 009/010 | Batch 200/469 | Cost: 0.0868 Epoch: 009/010 | Batch 250/469 | Cost: 0.0795 Epoch: 009/010 | Batch 300/469 | Cost: 0.0868 Epoch: 009/010 | Batch 350/469 | Cost: 0.0899 Epoch: 009/010 | Batch 400/469 | Cost: 0.0882 Epoch: 009/010 | Batch 450/469 | Cost: 0.0950 Time elapsed: 1.07 min Epoch: 010/010 | Batch 000/469 | Cost: 0.0854 Epoch: 010/010 | Batch 050/469 | Cost: 0.0839 Epoch: 010/010 | Batch 100/469 | Cost: 0.0873 Epoch: 010/010 | Batch 150/469 | Cost: 0.0871 Epoch: 010/010 | Batch 200/469 | Cost: 0.0851 Epoch: 010/010 | Batch 250/469 | Cost: 0.0831 Epoch: 010/010 | Batch 300/469 | Cost: 0.0843 Epoch: 010/010 | Batch 350/469 | Cost: 0.0860 Epoch: 010/010 | Batch 400/469 | Cost: 0.0840 Epoch: 010/010 | Batch 450/469 | Cost: 0.0818 Time elapsed: 1.19 min Total Training Time: 1.19 min
%matplotlib inline
import matplotlib.pyplot as plt
##########################
### VISUALIZATION
##########################
n_images = 15
image_width = 28
fig, axes = plt.subplots(nrows=2, ncols=n_images,
sharex=True, sharey=True, figsize=(20, 2.5))
orig_images = features[:n_images]
decoded_images = decoded[:n_images]
for i in range(n_images):
for ax, img in zip(axes, [orig_images, decoded_images]):
curr_img = img[i].detach().to(torch.device('cpu'))
ax[i].imshow(curr_img.view((image_width, image_width)), cmap='binary')
%watermark -iv
numpy 1.15.4 torch 1.0.0