Important: This notebook will only work with fastai-0.7.x. Do not try to run any fastai-1.x code from this path in the repository because it will load fastai-0.7.x
In [ ]:
%matplotlib inline
%reload_ext autoreload
%autoreload 2
Devise¶
In [ ]:
from fastai.conv_learner import *
torch.backends.cudnn.benchmark=True
import fastText as ft
In [ ]:
import torchvision.transforms as transforms
In [ ]:
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
tfms = transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
In [ ]:
fname = 'valid/n01440764/ILSVRC2012_val_00007197.JPEG'
In [ ]:
PATH = Path('data/imagenet/')
TMP_PATH = PATH/'tmp'
TRANS_PATH = Path('data/translate/')
PATH_TRN = PATH/'train'
In [ ]:
img = Image.open(PATH/fname)
In [ ]:
import fastai
In [ ]:
fastai.dataloader.DataLoader
In [ ]:
arch=resnet50
ttfms,vtfms = tfms_from_model(arch, 224, transforms_side_on, max_zoom=1.1)
def to_array(x,y): return np.array(x).astype(np.float32)/255,None
def TT(x,y): return torch.from_numpy(x),None
ttfms.tfms = [to_array] + ttfms.tfms# + [TT]
In [ ]:
ttfms(img)
Out[ ]:
( 0 ,.,.) =
-1.2303e+00 -8.5119e-01 -5.9588e-01 ... 2.6367e-01 2.9539e-01 -6.1730e-03
-3.7076e-01 -2.6681e-01 -8.4245e-01 ... 2.7774e-02 -2.1419e-01 -3.8521e-02
-4.4917e-01 -8.2077e-01 -9.1286e-01 ... -1.8501e-01 -3.3902e-01 -2.7731e-01
... ⋱ ...
-2.3891e-01 -6.1463e-01 -7.5132e-01 ... -3.6532e-01 -3.1892e-01 -5.1175e-01
-3.5185e-01 -6.9721e-01 -9.4736e-01 ... -4.4875e-01 -4.6495e-01 -3.5350e-01
-6.0510e-01 -1.2168e+00 -7.6353e-01 ... -2.4128e-01 -3.2143e-01 -4.5569e-01
( 1 ,.,.) =
-1.0955e+00 -3.7223e-01 6.5788e-02 ... 3.6881e-01 5.9299e-01 1.9598e-01
2.8945e-01 6.3827e-01 -1.3719e-01 ... 1.1291e-01 7.1316e-02 2.3313e-01
3.5067e-01 -1.5094e-01 -3.2621e-01 ... -1.5517e-01 -9.9879e-02 1.8858e-01
... ⋱ ...
8.9915e-02 1.1407e-01 6.0954e-02 ... 5.4866e-02 -3.2704e-02 -2.1574e-01
3.8106e-03 -3.5581e-01 -4.8731e-01 ... -1.6228e-01 -1.9376e-01 -7.6189e-02
-3.5776e-01 -1.0699e+00 -5.1589e-01 ... 3.9278e-02 -1.8401e-02 -1.4476e-01
( 2 ,.,.) =
-1.5169e+00 -1.2942e+00 -1.2443e+00 ... -2.9852e-01 -3.4353e-01 -3.4875e-01
-8.5310e-01 -7.1368e-01 -1.1537e+00 ... -3.7036e-01 -7.6493e-01 -5.2110e-01
-7.2973e-01 -1.1940e+00 -1.2884e+00 ... -6.7519e-01 -8.5094e-01 -8.4855e-01
... ⋱ ...
-1.1173e+00 -1.4027e+00 -1.5842e+00 ... -2.8738e-01 8.6017e-02 6.4270e-03
-1.1104e+00 -1.1831e+00 -1.4715e+00 ... 1.0541e-01 1.8515e-01 2.9458e-01
-1.3824e+00 -1.5386e+00 -1.4586e+00 ... 4.5452e-01 3.5802e-01 5.8026e-02
[torch.FloatTensor of size 3x224x224]
In [ ]:
In [ ]:
ft_vecs = ft.load_model(str((TRANS_PATH/'wiki.en.bin')))
In [ ]:
ft_vecs.get_word_vector('king')
Out[ ]:
array([ 0.03259, -0.18164, -0.29049, -0.10506, -0.16712, -0.07748, -0.5661 , -0.08622, -0.00216, 0.15366,
0.12189, -0.14722, 0.01511, 0.07209, -0.02156, -0.20612, -0.02104, -0.01999, -0.15506, 0.00802,
-0.22746, 0.33518, -0.10629, -0.50318, -0.1582 , 0.27829, 0.05752, -0.32697, 0.04766, 0.01076,
0.13972, -0.12445, -0.18989, 0.32969, -0.32513, 0.10958, 0.21962, -0.47215, 0.03422, -0.2207 ,
0.02177, 0.0832 , -0.04776, -0.48873, 0.05207, -0.15001, -0.19203, 0.06177, 0.15535, -0.05598,
0.11071, 0.39161, -0.17716, 0.05449, 0.25898, -0.13954, 0.4272 , -0.07273, -0.4714 , 0.04993,
0.29526, -0.05319, 0.03451, -0.10583, -0.30137, 0.16372, 0.07541, 0.21018, -0.11459, 0.10976,
0.04923, 0.17688, 0.45658, -0.59762, -0.0039 , 0.08866, 0.53103, 0.153 , -0.1673 , 0.13121,
-0.05547, -0.03582, -0.34535, 0.09128, 0.03323, 0.45211, -0.16894, 0.21139, 0.24153, 0.51014,
-0.01474, -0.47179, 0.2235 , -0.34668, 0.12126, 0.23727, -0.08424, 0.04555, -0.07698, 0.0428 ,
-0.13887, 0.29286, -0.28864, 0.53446, 0.02677, -0.04119, 0.40156, 0.38334, 0.01935, 0.02089,
0.02142, -0.11958, -0.44997, 0.13685, -0.12185, -0.00509, 0.60342, 0.65889, -0.16251, 0.46393,
0.19732, 0.19346, -0.07765, 0.17387, 0.07279, 0.04365, -0.01246, 0.4392 , 0.03182, 0.34927,
-0.13155, 0.41265, 0.1348 , 0.03162, 0.17821, 0.20899, -0.03224, -0.37799, 0.23646, 0.10512,
-0.00483, 0.33617, 0.43214, 0.28264, 0.01725, 0.35155, 0.28504, -0.41468, -0.20859, 0.08935,
-0.08568, -0.3982 , -0.61611, 0.574 , -0.34191, 0.03569, 0.08309, 0.02758, 0.30767, -0.14426,
-0.23718, 0.19269, 0.12444, 0.20298, -0.08636, -0.30212, 0.06119, 0.08865, 0.60565, 0.23092,
-0.16018, -0.44802, -0.14103, 0.08389, 0.08604, 0.17387, -0.11659, 0.15751, -0.25178, 0.12577,
0.28713, -0.00183, 0.05259, -0.0495 , -0.03082, 0.13133, -0.00867, 0.00691, 0.30406, 0.18153,
-0.05479, -0.39295, 0.29229, 0.27204, 0.01185, 0.02325, 0.02535, -0.21103, -0.45489, 0.10004,
0.26659, -0.12585, -0.03636, -0.1304 , -0.10385, -0.35109, -0.04138, 0.20202, 0.08724, -0.22088,
0.25375, 0.08034, 0.0022 , -0.14621, -0.16164, 0.12694, -0.01651, -0.11299, -0.06235, 0.15739,
-0.20588, -0.09687, -0.22731, -0.10299, -0.02208, 0.1705 , -0.41714, 0.13382, -0.09988, -0.35683,
0.49678, -0.00604, -0.09917, 0.28355, 0.27951, 0.09213, 0.12555, 0.12955, 0.05188, -0.14202,
-0.18416, -0.48024, -0.02423, 0.10908, -0.04117, -0.20895, -0.30235, 0.47612, -0.22305, -0.41871,
-0.03084, 0.02981, 0.21836, -0.04544, -0.24222, 0.0735 , -0.16438, -0.05721, 0.31028, 0.26954,
0.20621, 0.04835, 0.10146, -0.2655 , 0.00589, -0.0269 , 0.05519, 0.2096 , -0.21835, 0.12025,
-0.44548, 0.05322, -0.23166, 0.03323, 0.13661, -0.39058, 0.1834 , 0.01626, -0.19765, 0.14757,
-0.06413, 0.34661, 0.31601, 0.13334, -0.53255, 0.26908, 0.27234, -0.1101 , -0.11572, -0.42586,
0.21509, -0.23383, 0.07461, 0.30356, 0.0955 , -0.30532, -0.2858 , 0.27764, 0.04028, -0.09576],
dtype=float32)
In [ ]:
np.corrcoef(ft_vecs.get_word_vector('jeremy'), ft_vecs.get_word_vector('Jeremy'))
Out[ ]:
array([[1. , 0.60866],
[0.60866, 1. ]])
In [ ]:
np.corrcoef(ft_vecs.get_word_vector('banana'), ft_vecs.get_word_vector('Jeremy'))
Out[ ]:
array([[1. , 0.14482],
[0.14482, 1. ]])
Map imagenet classes to word vectors¶
In [ ]:
ft_words = ft_vecs.get_words(include_freq=True)
ft_word_dict = {k:v for k,v in zip(*ft_words)}
ft_words = sorted(ft_word_dict.keys(), key=lambda x: ft_word_dict[x])
len(ft_words)
Out[ ]:
2519370
In [ ]:
from fastai.io import get_data
In [ ]:
CLASSES_FN = 'imagenet_class_index.json'
get_data(f'http://files.fast.ai/models/{CLASSES_FN}', TMP_PATH/CLASSES_FN)
imagenet_class_index.json: 41.0kB [00:00, 56.2kB/s]
In [ ]:
WORDS_FN = 'classids.txt'
get_data(f'http://files.fast.ai/data/{WORDS_FN}', PATH/WORDS_FN)
classids.txt: 1.74MB [00:02, 765kB/s]
In [ ]:
class_dict = json.load((TMP_PATH/CLASSES_FN).open())
classids_1k = dict(class_dict.values())
nclass = len(class_dict); nclass
Out[ ]:
1000
In [ ]:
class_dict['0']
Out[ ]:
['n01440764', 'tench']
In [ ]:
classid_lines = (PATH/WORDS_FN).open().readlines()
classid_lines[:5]
Out[ ]:
['n00001740 entity\n', 'n00001930 physical_entity\n', 'n00002137 abstraction\n', 'n00002452 thing\n', 'n00002684 object\n']
In [ ]:
classids = dict(l.strip().split() for l in classid_lines)
len(classids),len(classids_1k)
Out[ ]:
(82115, 1000)
In [ ]:
lc_vec_d = {w.lower(): ft_vecs.get_word_vector(w) for w in ft_words[-1000000:]}
In [ ]:
syn_wv = [(k, lc_vec_d[v.lower()]) for k,v in classids.items()
if v.lower() in lc_vec_d]
syn_wv_1k = [(k, lc_vec_d[v.lower()]) for k,v in classids_1k.items()
if v.lower() in lc_vec_d]
syn2wv = dict(syn_wv)
len(syn2wv)
Out[ ]:
49469
In [ ]:
pickle.dump(syn2wv, (TMP_PATH/'syn2wv.pkl').open('wb'))
pickle.dump(syn_wv_1k, (TMP_PATH/'syn_wv_1k.pkl').open('wb'))
In [ ]:
syn2wv = pickle.load((TMP_PATH/'syn2wv.pkl').open('rb'))
syn_wv_1k = pickle.load((TMP_PATH/'syn_wv_1k.pkl').open('rb'))
In [ ]:
images = []
img_vecs = []
for d in (PATH/'train').iterdir():
if d.name not in syn2wv: continue
vec = syn2wv[d.name]
for f in d.iterdir():
images.append(str(f.relative_to(PATH)))
img_vecs.append(vec)
n_val=0
for d in (PATH/'valid').iterdir():
if d.name not in syn2wv: continue
vec = syn2wv[d.name]
for f in d.iterdir():
images.append(str(f.relative_to(PATH)))
img_vecs.append(vec)
n_val += 1
In [ ]:
n_val
Out[ ]:
28650
In [ ]:
img_vecs = np.stack(img_vecs)
img_vecs.shape
In [ ]:
pickle.dump(images, (TMP_PATH/'images.pkl').open('wb'))
pickle.dump(img_vecs, (TMP_PATH/'img_vecs.pkl').open('wb'))
In [ ]:
images = pickle.load((TMP_PATH/'images.pkl').open('rb'))
img_vecs = pickle.load((TMP_PATH/'img_vecs.pkl').open('rb'))
In [ ]:
arch = resnet50
In [ ]:
n = len(images); n
Out[ ]:
766876
In [ ]:
val_idxs = list(range(n-28650, n))
In [ ]:
tfms = tfms_from_model(arch, 224, transforms_side_on, max_zoom=1.1)
md = ImageClassifierData.from_names_and_array(PATH, images, img_vecs, val_idxs=val_idxs,
classes=None, tfms=tfms, continuous=True, bs=256)
In [ ]:
x,y = next(iter(md.val_dl))
In [ ]:
models = ConvnetBuilder(arch, md.c, is_multi=False, is_reg=True, xtra_fc=[1024], ps=[0.2,0.2])
learn = ConvLearner(md, models, precompute=True)
learn.opt_fn = partial(optim.Adam, betas=(0.9,0.99))
/home/ubuntu/fastai/courses/dl2/fastai/initializers.py:6: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_. if hasattr(m, 'weight'): init_fn(m.weight)
In [ ]:
def cos_loss(inp,targ): return 1 - F.cosine_similarity(inp,targ).mean()
learn.crit = cos_loss
In [ ]:
learn.lr_find(start_lr=1e-4, end_lr=1e15)
In [ ]:
learn.sched.plot()
In [ ]:
lr = 1e-2
wd = 1e-7
In [ ]:
learn.precompute=True
In [ ]:
learn.fit(lr, 1, cycle_len=20, wds=wd, use_clr=(20,10))
HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))
epoch trn_loss val_loss
0 0.104692 0.125685
1 0.112455 0.129307
2 0.110631 0.126568
3 0.108629 0.127338
4 0.110791 0.125033
5 0.108859 0.125186
6 0.106582 0.123875
7 0.103227 0.123945
8 0.10396 0.12304
9 0.105898 0.124894
10 0.10498 0.122582
11 0.104983 0.122906
12 0.102317 0.121171
13 0.10017 0.121816
14 0.099454 0.119647
15 0.100425 0.120914
16 0.097226 0.119724
17 0.094666 0.118746
18 0.094137 0.118744
19 0.090076 0.117908
Out[ ]:
[0.11790786389489033]
In [ ]:
learn.bn_freeze(True)
In [ ]:
learn.fit(lr, 1, cycle_len=20, wds=wd, use_clr=(20,10))
HBox(children=(IntProgress(value=0, description='Epoch', max=20), HTML(value='')))
epoch trn_loss val_loss
0 0.104692 0.125685
1 0.112455 0.129307
2 0.110631 0.126568
3 0.108629 0.127338
4 0.110791 0.125033
5 0.108859 0.125186
6 0.106582 0.123875
7 0.103227 0.123945
8 0.10396 0.12304
9 0.105898 0.124894
10 0.10498 0.122582
11 0.104983 0.122906
12 0.102317 0.121171
13 0.10017 0.121816
14 0.099454 0.119647
15 0.100425 0.120914
16 0.097226 0.119724
17 0.094666 0.118746
18 0.094137 0.118744
19 0.090076 0.117908
Out[ ]:
[0.11790786389489033]
In [ ]:
lrs = np.array([lr/1000,lr/100,lr])
In [ ]:
learn.precompute=False
learn.freeze_to(1)
In [ ]:
learn.save('pre0')
In [ ]:
learn.load('pre0')
Image search¶
Search imagenet classes¶
In [ ]:
syns, wvs = list(zip(*syn_wv_1k))
wvs = np.array(wvs)
In [ ]:
%time pred_wv = learn.predict()
CPU times: user 18.4 s, sys: 7.91 s, total: 26.3 s Wall time: 7.17 s
In [ ]:
start=300
In [ ]:
denorm = md.val_ds.denorm
def show_img(im, figsize=None, ax=None):
if not ax: fig,ax = plt.subplots(figsize=figsize)
ax.imshow(im)
ax.axis('off')
return ax
def show_imgs(ims, cols, figsize=None):
fig,axes = plt.subplots(len(ims)//cols, cols, figsize=figsize)
for i,ax in enumerate(axes.flat): show_img(ims[i], ax=ax)
plt.tight_layout()
In [ ]:
show_imgs(denorm(md.val_ds[start:start+25][0]), 5, (10,10))
In [ ]:
import nmslib
def create_index(a):
index = nmslib.init(space='angulardist')
index.addDataPointBatch(a)
index.createIndex()
return index
def get_knns(index, vecs):
return zip(*index.knnQueryBatch(vecs, k=10, num_threads=4))
def get_knn(index, vec): return index.knnQuery(vec, k=10)
In [ ]:
nn_wvs = create_index(wvs)
In [ ]:
idxs,dists = get_knns(nn_wvs, pred_wv)
In [ ]:
[[classids[syns[id]] for id in ids[:3]] for ids in idxs[start:start+10]]
Out[ ]:
[['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['spoonbill', 'bustard', 'oystercatcher'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill']]
Search all wordnet noun classes¶
In [ ]:
all_syns, all_wvs = list(zip(*syn2wv.items()))
all_wvs = np.array(all_wvs)
In [ ]:
nn_allwvs = create_index(all_wvs)
In [ ]:
idxs,dists = get_knns(nn_allwvs, pred_wv)
In [ ]:
[[classids[all_syns[id]] for id in ids[:3]] for ids in idxs[start:start+10]]
Out[ ]:
[['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['spoonbill', 'bustard', 'oystercatcher'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill'], ['limpkin', 'oystercatcher', 'spoonbill']]
Text -> image search¶
In [ ]:
nn_predwv = create_index(pred_wv)
In [ ]:
en_vecd = pickle.load(open(TRANS_PATH/'wiki.en.pkl','rb'))
In [ ]:
vec = en_vecd['boat']
In [ ]:
idxs,dists = get_knn(nn_predwv, vec)
show_imgs([open_image(PATH/md.val_ds.fnames[i]) for i in idxs[:3]], 3, figsize=(9,3));
In [ ]:
vec = (en_vecd['engine'] + en_vecd['boat'])/2
In [ ]:
idxs,dists = get_knn(nn_predwv, vec)
show_imgs([open_image(PATH/md.val_ds.fnames[i]) for i in idxs[:3]], 3, figsize=(9,3));
In [ ]:
vec = (en_vecd['sail'] + en_vecd['boat'])/2
In [ ]:
idxs,dists = get_knn(nn_predwv, vec)
show_imgs([open_image(PATH/md.val_ds.fnames[i]) for i in idxs[:3]], 3, figsize=(9,3));
Image->image¶
In [ ]:
fname = 'valid/n01440764/ILSVRC2012_val_00007197.JPEG'
In [ ]:
img = open_image(PATH/fname)
In [ ]:
show_img(img);
In [ ]:
t_img = md.val_ds.transform(img)
pred = learn.predict_array(t_img[None])
In [ ]:
idxs,dists = get_knn(nn_predwv, pred)
show_imgs([open_image(PATH/md.val_ds.fnames[i]) for i in idxs[1:4]], 3, figsize=(9,3));
In [ ]: