Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.
%load_ext watermark
%watermark -a 'Sebastian Raschka' -v -p tensorflow
Sebastian Raschka CPython 3.6.1 IPython 6.0.0 tensorflow 1.2.0
The VGG-16 Convolutional Neural Network Architecture [1] implemented in TensorFlow and trained on Cifar-10 [2, 3] images.
References:
##########################
### DATASET
##########################
# load utilities from ../helper.py
import sys
sys.path.insert(0, '..')
from helper import download_and_extract_cifar
from helper import Cifar10Loader
dest = download_and_extract_cifar('./cifar-10')
cifar = Cifar10Loader(dest, normalize=True,
zero_center=True,
channel_mean_center=False)
cifar.num_train
X, y = cifar.load_test()
half = cifar.num_test // 2
X_test, X_valid = X[:half], X[half:]
y_test, y_valid = y[:half], y[half:]
del X, y
--------------------------------------------------------------------------- TypeError Traceback (most recent call last) <ipython-input-1-0beb05071f51> in <module> 5 6 import sys ----> 7 sys.path.insert('..') 8 9 from helper import download_and_extract_cifar TypeError: insert() takes exactly 2 arguments (1 given)
import tensorflow as tf
import numpy as np
tf.test.gpu_device_name()
'/gpu:0'
##########################
### SETTINGS
##########################
# Hyperparameters
learning_rate = 0.001
training_epochs = 30
batch_size = 32
# Other
print_interval = 200
# Architecture
image_width, image_height, image_depth = 32, 32, 3
n_classes = 10
##########################
### WRAPPER FUNCTIONS
##########################
def conv_layer(input, input_channels, output_channels,
kernel_size, strides, scope, padding='SAME'):
with tf.name_scope(scope):
weights_shape = kernel_size + [input_channels, output_channels]
weights = tf.Variable(tf.truncated_normal(shape=weights_shape,
mean=0.0,
stddev=0.1,
dtype=tf.float32),
name='weights')
biases = tf.Variable(tf.zeros(shape=[output_channels]),
name='biases')
conv = tf.nn.conv2d(input=input,
filter=weights,
strides=strides,
padding=padding,
name='convolution')
out = tf.nn.bias_add(conv, biases, name='logits')
out = tf.nn.relu(out, name='activation')
return out
def fc_layer(input, output_nodes, scope,
activation=None, seed=None):
with tf.name_scope(scope):
shape = int(np.prod(input.get_shape()[1:]))
flat_input = tf.reshape(input, [-1, shape])
weights = tf.Variable(tf.truncated_normal(shape=[shape,
output_nodes],
mean=0.0,
stddev=0.1,
dtype=tf.float32,
seed=seed),
name='weights')
biases = tf.Variable(tf.zeros(shape=[output_nodes]),
name='biases')
act = tf.nn.bias_add(tf.matmul(flat_input, weights), biases,
name='logits')
if activation is not None:
act = activation(act, name='activation')
return act
##########################
### GRAPH DEFINITION
##########################
g = tf.Graph()
with g.as_default():
# Input data
tf_x = tf.placeholder(tf.float32, [None, image_width, image_height, image_depth], name='features')
tf_y = tf.placeholder(tf.float32, [None, n_classes], name='targets')
##########################
### VGG16 Model
##########################
# =========
# BLOCK 1
# =========
conv_layer_1 = conv_layer(input=tf_x,
input_channels=3,
output_channels=64,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv1')
conv_layer_2 = conv_layer(input=conv_layer_1,
input_channels=64,
output_channels=64,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv2')
pool_layer_1 = tf.nn.max_pool(conv_layer_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# =========
# BLOCK 2
# =========
conv_layer_3 = conv_layer(input=pool_layer_1,
input_channels=64,
output_channels=128,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv3')
conv_layer_4 = conv_layer(input=conv_layer_3,
input_channels=128,
output_channels=128,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv4')
pool_layer_2 = tf.nn.max_pool(conv_layer_4,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# =========
# BLOCK 3
# =========
conv_layer_5 = conv_layer(input=pool_layer_2,
input_channels=128,
output_channels=256,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv5')
conv_layer_6 = conv_layer(input=conv_layer_5,
input_channels=256,
output_channels=256,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv6')
conv_layer_7 = conv_layer(input=conv_layer_6,
input_channels=256,
output_channels=256,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv7')
pool_layer_3 = tf.nn.max_pool(conv_layer_7,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
# =========
# BLOCK 4
# =========
conv_layer_8 = conv_layer(input=pool_layer_3,
input_channels=256,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv8')
conv_layer_9 = conv_layer(input=conv_layer_8,
input_channels=512,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv9')
conv_layer_10 = conv_layer(input=conv_layer_9,
input_channels=512,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv10')
pool_layer_4 = tf.nn.max_pool(conv_layer_10,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
# =========
# BLOCK 5
# =========
conv_layer_11 = conv_layer(input=pool_layer_4,
input_channels=512,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv11')
conv_layer_12 = conv_layer(input=conv_layer_11,
input_channels=512,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv12')
conv_layer_13 = conv_layer(input=conv_layer_12,
input_channels=512,
output_channels=512,
kernel_size=[3, 3],
strides=[1, 1, 1, 1],
scope='conv13')
pool_layer_5 = tf.nn.max_pool(conv_layer_12,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool5')
# ===========
# CLASSIFIER
# ===========
fc_layer_1 = fc_layer(input=pool_layer_5,
output_nodes=4096,
activation=tf.nn.relu,
scope='fc1')
fc_layer_2 = fc_layer(input=fc_layer_1,
output_nodes=4096,
activation=tf.nn.relu,
scope='fc2')
out_layer = fc_layer(input=fc_layer_2,
output_nodes=n_classes,
activation=None,
scope='output_layer')
# Loss and optimizer
loss = tf.nn.softmax_cross_entropy_with_logits(logits=out_layer, labels=tf_y)
cost = tf.reduce_mean(loss, name='cost')
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train = optimizer.minimize(cost, name='train')
# Prediction
correct_prediction = tf.equal(tf.argmax(tf_y, 1), tf.argmax(out_layer, 1),
name='correct_predictions')
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')
# Saver to save session for reuse
saver = tf.train.Saver()
##########################
### TRAINING & EVALUATION
##########################
with tf.Session(graph=g) as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(training_epochs):
avg_cost = 0.
mbatch_cnt = 0
for batch_x, batch_y in cifar.load_train_epoch(shuffle=True, batch_size=batch_size):
mbatch_cnt += 1
_, c = sess.run(['train', 'cost:0'], feed_dict={'features:0': batch_x,
'targets:0': batch_y})
avg_cost += c
if not mbatch_cnt % print_interval:
print("Minibatch: %04d | Cost: %.3f" % (mbatch_cnt, c))
# ===================
# Training Accuracy
# ===================
n_predictions, n_correct = 0, 0
for batch_x, batch_y in cifar.load_train_epoch(batch_size=batch_size):
p = sess.run('correct_predictions:0',
feed_dict={'features:0': batch_x,
'targets:0': batch_y})
n_correct += np.sum(p)
n_predictions += p.shape[0]
train_acc = n_correct / n_predictions
# ===================
# Validation Accuracy
# ===================
#valid_acc = sess.run('accuracy:0', feed_dict={'features:0': X_valid,
# 'targets:0': y_valid})
# ---------------------------------------
# workaround for GPUs with <= 4 Gb memory
n_predictions, n_correct = 0, 0
indices = np.arange(y_valid.shape[0])
chunksize = 500
for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):
index_slice = indices[start_idx:start_idx + chunksize]
p = sess.run('correct_predictions:0',
feed_dict={'features:0': X_valid[index_slice],
'targets:0': y_valid[index_slice]})
n_correct += np.sum(p)
n_predictions += p.shape[0]
valid_acc = n_correct / n_predictions
# ---------------------------------------
print("Epoch: %03d | AvgCost: %.3f" % (epoch + 1, avg_cost / mbatch_cnt), end="")
print(" | Train/Valid ACC: %.3f/%.3f" % (train_acc, valid_acc))
saver.save(sess, save_path='./convnet-vgg16.ckpt')
Minibatch: 0200 | Cost: 9.004 Minibatch: 0400 | Cost: 3.877 Minibatch: 0600 | Cost: 3.213 Minibatch: 0800 | Cost: 2.410 Minibatch: 1000 | Cost: 2.384 Minibatch: 1200 | Cost: 2.250 Minibatch: 1400 | Cost: 2.352 Epoch: 001 | AvgCost: 1897.994 | Train/Valid ACC: 0.152/0.145 Minibatch: 0200 | Cost: 2.433 Minibatch: 0400 | Cost: 2.426 Minibatch: 0600 | Cost: 2.202 Minibatch: 0800 | Cost: 2.221 Minibatch: 1000 | Cost: 2.112 Minibatch: 1200 | Cost: 2.182 Minibatch: 1400 | Cost: 2.382 Epoch: 002 | AvgCost: 2.219 | Train/Valid ACC: 0.176/0.168 Minibatch: 0200 | Cost: 2.145 Minibatch: 0400 | Cost: 2.089 Minibatch: 0600 | Cost: 2.221 Minibatch: 0800 | Cost: 1.903 Minibatch: 1000 | Cost: 2.296 Minibatch: 1200 | Cost: 2.169 Minibatch: 1400 | Cost: 2.164 Epoch: 003 | AvgCost: 2.168 | Train/Valid ACC: 0.166/0.173 Minibatch: 0200 | Cost: 2.094 Minibatch: 0400 | Cost: 2.015 Minibatch: 0600 | Cost: 2.217 Minibatch: 0800 | Cost: 2.028 Minibatch: 1000 | Cost: 2.265 Minibatch: 1200 | Cost: 2.103 Minibatch: 1400 | Cost: 2.060 Epoch: 004 | AvgCost: 2.144 | Train/Valid ACC: 0.189/0.188 Minibatch: 0200 | Cost: 2.310 Minibatch: 0400 | Cost: 1.986 Minibatch: 0600 | Cost: 1.844 Minibatch: 0800 | Cost: 2.183 Minibatch: 1000 | Cost: 2.040 Minibatch: 1200 | Cost: 2.004 Minibatch: 1400 | Cost: 1.987 Epoch: 005 | AvgCost: 2.075 | Train/Valid ACC: 0.180/0.181 Minibatch: 0200 | Cost: 1.849 Minibatch: 0400 | Cost: 1.898 Minibatch: 0600 | Cost: 1.996 Minibatch: 0800 | Cost: 1.968 Minibatch: 1000 | Cost: 1.987 Minibatch: 1200 | Cost: 2.003 Minibatch: 1400 | Cost: 2.041 Epoch: 006 | AvgCost: 2.010 | Train/Valid ACC: 0.176/0.176 Minibatch: 0200 | Cost: 2.419 Minibatch: 0400 | Cost: 2.109 Minibatch: 0600 | Cost: 1.902 Minibatch: 0800 | Cost: 1.937 Minibatch: 1000 | Cost: 1.813 Minibatch: 1200 | Cost: 1.990 Minibatch: 1400 | Cost: 1.662 Epoch: 007 | AvgCost: 1.913 | Train/Valid ACC: 0.256/0.251 Minibatch: 0200 | Cost: 2.036 Minibatch: 0400 | Cost: 1.690 Minibatch: 0600 | Cost: 1.866 Minibatch: 0800 | Cost: 1.574 Minibatch: 1000 | Cost: 1.886 Minibatch: 1200 | Cost: 1.720 Minibatch: 1400 | Cost: 1.937 Epoch: 008 | AvgCost: 1.769 | Train/Valid ACC: 0.270/0.267 Minibatch: 0200 | Cost: 1.569 Minibatch: 0400 | Cost: 1.800 Minibatch: 0600 | Cost: 1.541 Minibatch: 0800 | Cost: 1.499 Minibatch: 1000 | Cost: 1.596 Minibatch: 1200 | Cost: 1.994 Minibatch: 1400 | Cost: 1.283 Epoch: 009 | AvgCost: 1.653 | Train/Valid ACC: 0.343/0.337 Minibatch: 0200 | Cost: 1.415 Minibatch: 0400 | Cost: 1.648 Minibatch: 0600 | Cost: 1.487 Minibatch: 0800 | Cost: 1.454 Minibatch: 1000 | Cost: 1.266 Minibatch: 1200 | Cost: 1.267 Minibatch: 1400 | Cost: 1.589 Epoch: 010 | AvgCost: 1.526 | Train/Valid ACC: 0.469/0.459 Minibatch: 0200 | Cost: 1.585 Minibatch: 0400 | Cost: 1.449 Minibatch: 0600 | Cost: 1.340 Minibatch: 0800 | Cost: 1.297 Minibatch: 1000 | Cost: 1.767 Minibatch: 1200 | Cost: 1.088 Minibatch: 1400 | Cost: 1.159 Epoch: 011 | AvgCost: 1.342 | Train/Valid ACC: 0.536/0.517 Minibatch: 0200 | Cost: 1.408 Minibatch: 0400 | Cost: 1.510 Minibatch: 0600 | Cost: 1.424 Minibatch: 0800 | Cost: 1.019 Minibatch: 1000 | Cost: 1.069 Minibatch: 1200 | Cost: 1.465 Minibatch: 1400 | Cost: 1.289 Epoch: 012 | AvgCost: 1.203 | Train/Valid ACC: 0.590/0.576 Minibatch: 0200 | Cost: 0.670 Minibatch: 0400 | Cost: 0.993 Minibatch: 0600 | Cost: 0.998 Minibatch: 0800 | Cost: 1.103 Minibatch: 1000 | Cost: 1.127 Minibatch: 1200 | Cost: 0.969 Minibatch: 1400 | Cost: 1.216 Epoch: 013 | AvgCost: 1.111 | Train/Valid ACC: 0.634/0.603 Minibatch: 0200 | Cost: 1.262 Minibatch: 0400 | Cost: 1.147 Minibatch: 0600 | Cost: 0.660 Minibatch: 0800 | Cost: 0.970 Minibatch: 1000 | Cost: 0.716 Minibatch: 1200 | Cost: 1.017 Minibatch: 1400 | Cost: 1.264 Epoch: 014 | AvgCost: 1.036 | Train/Valid ACC: 0.638/0.617 Minibatch: 0200 | Cost: 0.846 Minibatch: 0400 | Cost: 0.843 Minibatch: 0600 | Cost: 0.792 Minibatch: 0800 | Cost: 0.718 Minibatch: 1000 | Cost: 1.141 Minibatch: 1200 | Cost: 1.024 Minibatch: 1400 | Cost: 0.877 Epoch: 015 | AvgCost: 0.949 | Train/Valid ACC: 0.683/0.654 Minibatch: 0200 | Cost: 0.463 Minibatch: 0400 | Cost: 0.854 Minibatch: 0600 | Cost: 0.820 Minibatch: 0800 | Cost: 0.952 Minibatch: 1000 | Cost: 0.819 Minibatch: 1200 | Cost: 0.648 Minibatch: 1400 | Cost: 1.030 Epoch: 016 | AvgCost: 0.872 | Train/Valid ACC: 0.738/0.697 Minibatch: 0200 | Cost: 0.862 Minibatch: 0400 | Cost: 1.010 Minibatch: 0600 | Cost: 0.991 Minibatch: 0800 | Cost: 0.664 Minibatch: 1000 | Cost: 0.604 Minibatch: 1200 | Cost: 0.879 Minibatch: 1400 | Cost: 0.791 Epoch: 017 | AvgCost: 0.823 | Train/Valid ACC: 0.753/0.698 Minibatch: 0200 | Cost: 0.823 Minibatch: 0400 | Cost: 0.619 Minibatch: 0600 | Cost: 0.764 Minibatch: 0800 | Cost: 1.127 Minibatch: 1000 | Cost: 0.623 Minibatch: 1200 | Cost: 0.796 Minibatch: 1400 | Cost: 0.694 Epoch: 018 | AvgCost: 0.771 | Train/Valid ACC: 0.769/0.709 Minibatch: 0200 | Cost: 0.431 Minibatch: 0400 | Cost: 0.688 Minibatch: 0600 | Cost: 0.406 Minibatch: 0800 | Cost: 0.906 Minibatch: 1000 | Cost: 0.607 Minibatch: 1200 | Cost: 0.445 Minibatch: 1400 | Cost: 0.660 Epoch: 019 | AvgCost: 0.721 | Train/Valid ACC: 0.780/0.707 Minibatch: 0200 | Cost: 0.740 Minibatch: 0400 | Cost: 0.542 Minibatch: 0600 | Cost: 0.435 Minibatch: 0800 | Cost: 0.953 Minibatch: 1000 | Cost: 0.954 Minibatch: 1200 | Cost: 0.544 Minibatch: 1400 | Cost: 0.855 Epoch: 020 | AvgCost: 0.672 | Train/Valid ACC: 0.779/0.717 Minibatch: 0200 | Cost: 0.432 Minibatch: 0400 | Cost: 0.376 Minibatch: 0600 | Cost: 0.477 Minibatch: 0800 | Cost: 0.476 Minibatch: 1000 | Cost: 0.950 Minibatch: 1200 | Cost: 0.468 Minibatch: 1400 | Cost: 0.595 Epoch: 021 | AvgCost: 0.636 | Train/Valid ACC: 0.818/0.737 Minibatch: 0200 | Cost: 1.114 Minibatch: 0400 | Cost: 0.519 Minibatch: 0600 | Cost: 0.623 Minibatch: 0800 | Cost: 0.721 Minibatch: 1000 | Cost: 0.880 Minibatch: 1200 | Cost: 0.654 Minibatch: 1400 | Cost: 0.378 Epoch: 022 | AvgCost: 0.596 | Train/Valid ACC: 0.825/0.742 Minibatch: 0200 | Cost: 0.541 Minibatch: 0400 | Cost: 0.478 Minibatch: 0600 | Cost: 0.642 Minibatch: 0800 | Cost: 0.871 Minibatch: 1000 | Cost: 0.710 Minibatch: 1200 | Cost: 0.682 Minibatch: 1400 | Cost: 0.671 Epoch: 023 | AvgCost: 0.663 | Train/Valid ACC: 0.776/0.712 Minibatch: 0200 | Cost: 0.498 Minibatch: 0400 | Cost: 0.379 Minibatch: 0600 | Cost: 0.535 Minibatch: 0800 | Cost: 0.435 Minibatch: 1000 | Cost: 0.562 Minibatch: 1200 | Cost: 0.777 Minibatch: 1400 | Cost: 0.635 Epoch: 024 | AvgCost: 0.539 | Train/Valid ACC: 0.853/0.759 Minibatch: 0200 | Cost: 0.331 Minibatch: 0400 | Cost: 0.419 Minibatch: 0600 | Cost: 0.497 Minibatch: 0800 | Cost: 0.619 Minibatch: 1000 | Cost: 0.620 Minibatch: 1200 | Cost: 0.196 Minibatch: 1400 | Cost: 0.648 Epoch: 025 | AvgCost: 0.498 | Train/Valid ACC: 0.849/0.738 Minibatch: 0200 | Cost: 0.247 Minibatch: 0400 | Cost: 0.263 Minibatch: 0600 | Cost: 0.183 Minibatch: 0800 | Cost: 0.251 Minibatch: 1000 | Cost: 0.251 Minibatch: 1200 | Cost: 0.447 Minibatch: 1400 | Cost: 0.443 Epoch: 026 | AvgCost: 0.469 | Train/Valid ACC: 0.850/0.744 Minibatch: 0200 | Cost: 0.390 Minibatch: 0400 | Cost: 0.325 Minibatch: 0600 | Cost: 0.404 Minibatch: 0800 | Cost: 0.363 Minibatch: 1000 | Cost: 0.655 Minibatch: 1200 | Cost: 0.195 Minibatch: 1400 | Cost: 0.647 Epoch: 027 | AvgCost: 0.485 | Train/Valid ACC: 0.855/0.751 Minibatch: 0200 | Cost: 0.580 Minibatch: 0400 | Cost: 0.177 Minibatch: 0600 | Cost: 0.317 Minibatch: 0800 | Cost: 0.249 Minibatch: 1000 | Cost: 0.287 Minibatch: 1200 | Cost: 0.282 Minibatch: 1400 | Cost: 0.429 Epoch: 028 | AvgCost: 0.416 | Train/Valid ACC: 0.887/0.759 Minibatch: 0200 | Cost: 0.233 Minibatch: 0400 | Cost: 0.118 Minibatch: 0600 | Cost: 0.218 Minibatch: 0800 | Cost: 0.195 Minibatch: 1000 | Cost: 0.208 Minibatch: 1200 | Cost: 0.188 Minibatch: 1400 | Cost: 0.509 Epoch: 029 | AvgCost: 0.488 | Train/Valid ACC: 0.874/0.767 Minibatch: 0200 | Cost: 0.313 Minibatch: 0400 | Cost: 0.508 Minibatch: 0600 | Cost: 0.553 Minibatch: 0800 | Cost: 0.385 Minibatch: 1000 | Cost: 0.272 Minibatch: 1200 | Cost: 0.366 Minibatch: 1400 | Cost: 0.156 Epoch: 030 | AvgCost: 0.404 | Train/Valid ACC: 0.909/0.776
##########################
### RELOAD & TEST
##########################
with tf.Session(graph=g) as sess:
saver.restore(sess, save_path='./convnet-vgg16.ckpt')
# test_acc = sess.run('accuracy:0', feed_dict={'features:0': X_test,
# 'targets:0': y_test})
# ---------------------------------------
# workaround for GPUs with <= 4 Gb memory
n_predictions, n_correct = 0, 0
indices = np.arange(y_test.shape[0])
chunksize = 500
for start_idx in range(0, indices.shape[0] - chunksize + 1, chunksize):
index_slice = indices[start_idx:start_idx + chunksize]
p = sess.run('correct_predictions:0',
feed_dict={'features:0': X_test[index_slice],
'targets:0': y_test[index_slice]})
n_correct += np.sum(p)
n_predictions += p.shape[0]
test_acc = n_correct / n_predictions
# ---------------------------------------
print('Test ACC: %.3f' % test_acc)
INFO:tensorflow:Restoring parameters from ./convnet-vgg16.ckpt Test ACC: 0.772