from keras import backend as K
from keras.applications import inception_v3
from keras.preprocessing.image import load_img, img_to_array
import keras
import numpy as np
import scipy
from IPython.display import Image
Using Theano backend.
base_image_path = "monalisa.jpg"
result_prefix = "monolisa-generated"
These are the names of the layers for which we try to maximize activation, as well as their weight in the final loss we try to maximize.
You can tweak these setting to obtain new visual effects.
settings = {
'features': {
'mixed2': 0.2,
'mixed3': 0.5,
'mixed4': 2.,
'mixed5': 1.5,
},
}
def preprocess_image(image_path):
# Util function to open, resize and format pictures
# into appropriate tensors.
img = load_img(image_path)
img = img_to_array(img)
img = np.expand_dims(img, axis=0)
img = inception_v3.preprocess_input(img)
return img
def deprocess_image(x):
# Util function to convert a tensor into a valid image.
if K.image_data_format() == 'channels_first':
x = x.reshape((3, x.shape[2], x.shape[3]))
x = x.transpose((1, 2, 0))
else:
x = x.reshape((x.shape[1], x.shape[2], 3))
x /= 2.
x += 0.5
x *= 255.
x = np.clip(x, 0, 255).astype('uint8')
return x
def eval_loss_and_grads(x):
outs = fetch_loss_and_grads([x])
loss_value = outs[0]
grad_values = outs[1]
return loss_value, grad_values
def resize_img(img, size):
img = np.copy(img)
if K.image_data_format() == 'channels_first':
factors = (1, 1,
float(size[0]) / img.shape[2],
float(size[1]) / img.shape[3])
else:
factors = (1,
float(size[0]) / img.shape[1],
float(size[1]) / img.shape[2],
1)
return scipy.ndimage.zoom(img, factors, order=1)
def gradient_ascent(x, iterations, step, max_loss=None):
for i in range(iterations):
loss_value, grad_values = eval_loss_and_grads(x)
if max_loss is not None and loss_value > max_loss:
break
print('..Loss value at', i, ':', loss_value)
x += step * grad_values
return x
def save_img(img, fname):
pil_img = deprocess_image(np.copy(img))
scipy.misc.imsave(fname, pil_img)
K.set_learning_phase(0)
# Build the InceptionV3 network with our placeholder.
# The model will be loaded with pre-trained ImageNet weights.
model = inception_v3.InceptionV3(weights='imagenet',
include_top=False)
dream = model.input
print('Model loaded.')
Model loaded.
from conx import import_keras_model
conx, version 3.4.3
network = import_keras_model(model, "Inception V3")
for clayer in network.layers:
if clayer.kind() == "hidden":
clayer.visible = False
img = preprocess_image(base_image_path)
network.dataset._inputs = [img]
network.layers[1].visible = True
network.dashboard()
Failed to display Jupyter Widget of type Dashboard
.
If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean that the widgets JavaScript is still loading. If this message persists, it likely means that the widgets JavaScript library is either not installed or not enabled. See the Jupyter Widgets Documentation for setup instructions.
If you're reading this message in another frontend (for example, a static rendering on GitHub or NBViewer), it may mean that your frontend doesn't currently support widgets.
network.propagate_to_features("conv2d_3", img[0])
output = network.propagate(img[0])
from conx import shape
shape(output)
(6, 6, 2048)
network.layers[1].name
'conv2d_1'
output = network.propagate_to("conv2d_1", img[0])
shape(output)
(127, 127, 32)
shape(network.dataset.inputs[0])
(256, 256, 3)
# Define the loss.
loss = K.variable(0.)
for layer_name in settings['features']:
# Add the L2 norm of the features of a layer to the loss.
coeff = settings['features'][layer_name]
x = network[layer_name].keras_layer.output
# We avoid border artifacts by only involving non-border pixels in the loss.
scaling = K.prod(K.cast(K.shape(x), 'float32'))
if K.image_data_format() == 'channels_first':
loss += coeff * K.sum(K.square(x[:, :, 2: -2, 2: -2])) / scaling
else:
loss += coeff * K.sum(K.square(x[:, 2: -2, 2: -2, :])) / scaling
# Compute the gradients of the dream wrt the loss.
grads = K.gradients(loss, dream)[0]
# Normalize gradients.
grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
# Set up function to retrieve the value
# of the loss and gradients given an input image.
outputs = [loss, grads]
fetch_loss_and_grads = K.function([dream], outputs)
Process:
To obtain the detail lost during upscaling, we simply take the original image, shrink it down, upscale it, and compare the result to the (resized) original image.
# Playing with these hyperparameters will also allow you to achieve new effects
step = 0.01 # Gradient ascent step size
num_octave = 3 # Number of scales at which to run gradient ascent
octave_scale = 1.4 # Size ratio between scales
iterations = 20 # Number of ascent steps per scale
max_loss = 10.
if K.image_data_format() == 'channels_first':
original_shape = img.shape[2:]
else:
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, num_octave):
shape = tuple([int(dim / (octave_scale ** i)) for dim in original_shape])
successive_shapes.append(shape)
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
output1 = model.predict(img)[0]
output2 = network.propagate(img[0])
(output1 == output2).all()
True
from conx import shape, array2image
shape(output2)
(6, 6, 2048)
for shape in successive_shapes:
print('Processing image shape', shape)
img = resize_img(img, shape)
img = gradient_ascent(img,
iterations=iterations,
step=step,
max_loss=max_loss)
upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)
same_size_original = resize_img(original_img, shape)
lost_detail = same_size_original - upscaled_shrunk_original_img
img += lost_detail
shrunk_original_img = resize_img(original_img, shape)
save_img(img, fname=result_prefix + '.png')
Processing image shape (130, 130) ..Loss value at 0 : 0.3545876443386078 ..Loss value at 1 : 0.4975617229938507 ..Loss value at 2 : 0.8025295734405518 ..Loss value at 3 : 1.158888339996338 ..Loss value at 4 : 1.4930806159973145 ..Loss value at 5 : 1.6510992050170898 ..Loss value at 6 : 1.923414945602417 ..Loss value at 7 : 2.1840977668762207 ..Loss value at 8 : 2.2317416667938232 ..Loss value at 9 : 2.5300564765930176 ..Loss value at 10 : 2.722439765930176 ..Loss value at 11 : 2.878614902496338 ..Loss value at 12 : 3.0598573684692383 ..Loss value at 13 : 3.2777538299560547 ..Loss value at 14 : 3.3470568656921387 ..Loss value at 15 : 3.5682644844055176 ..Loss value at 16 : 3.7422337532043457 ..Loss value at 17 : 4.028820037841797 ..Loss value at 18 : 4.099969863891602 ..Loss value at 19 : 4.302164554595947 Processing image shape (182, 182) ..Loss value at 0 : 0.9539443254470825 ..Loss value at 1 : 1.6061757802963257 ..Loss value at 2 : 2.113076686859131 ..Loss value at 3 : 2.3465652465820312 ..Loss value at 4 : 2.776820421218872 ..Loss value at 5 : 2.999680519104004 ..Loss value at 6 : 3.3221709728240967 ..Loss value at 7 : 3.717674732208252 ..Loss value at 8 : 4.081844329833984 ..Loss value at 9 : 4.312065124511719 ..Loss value at 10 : 4.548281192779541 ..Loss value at 11 : 4.772209167480469 ..Loss value at 12 : 4.857020854949951 ..Loss value at 13 : 5.2547760009765625 ..Loss value at 14 : 5.270005226135254 ..Loss value at 15 : 5.968417167663574 ..Loss value at 16 : 6.207927703857422 ..Loss value at 17 : 6.2207350730896 ..Loss value at 18 : 6.272695541381836 ..Loss value at 19 : 6.817244529724121 Processing image shape (256, 256) ..Loss value at 0 : 1.117199182510376 ..Loss value at 1 : 1.7699036598205566 ..Loss value at 2 : 2.3564746379852295 ..Loss value at 3 : 2.848952531814575 ..Loss value at 4 : 3.228632688522339 ..Loss value at 5 : 3.652087688446045 ..Loss value at 6 : 4.0550384521484375 ..Loss value at 7 : 4.45245361328125 ..Loss value at 8 : 4.75450325012207 ..Loss value at 9 : 5.174521446228027 ..Loss value at 10 : 5.522565841674805 ..Loss value at 11 : 5.904690265655518 ..Loss value at 12 : 6.451308250427246 ..Loss value at 13 : 6.94564962387085 ..Loss value at 14 : 7.622781753540039 ..Loss value at 15 : 8.32858943939209
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:53: DeprecationWarning: `imsave` is deprecated! `imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0. Use ``imageio.imwrite`` instead.
Image(filename="./monalisa-generated.png")