Important: This notebook will only work with fastai-0.7.x. Do not try to run any fastai-1.x code from this path in the repository because it will load fastai-0.7.x
%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.imports import *
from fastai.torch_imports import *
from fastai.core import *
from fastai.model import fit
from fastai.dataset import *
import torchtext
from torchtext import vocab, data
from torchtext.datasets import language_modeling
from fastai.rnn_reg import *
from fastai.rnn_train import *
from fastai.nlp import *
from fastai.lm_rnn import *
PATH='data/wikitext-2/'
%ls {PATH}
models/ tmp/ wiki.test.tokens wiki.train.tokens wiki.valid.tokens
!head -5 {PATH}wiki.train.tokens
= Valkyria Chronicles III = Senjō no Valkyria 3 : <unk> Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . <unk> the same fusion of tactical and real @-@ time gameplay as its predecessors , the story runs parallel to the first game and follows the " Nameless " , a penal military unit serving the nation of Gallia during the Second Europan War who perform secret black operations and are pitted against the Imperial unit " <unk> Raven " . The game began development in 2010 , carrying over a large portion of the work done on Valkyria Chronicles II . While it retained the standard features of the series , it also underwent multiple adjustments , such as making the game more <unk> for series newcomers . Character designer <unk> Honjou and composer Hitoshi Sakimoto both returned from previous entries , along with Valkyria Chronicles II director Takeshi Ozawa . A large team of writers handled the script . The game 's opening theme was sung by May 'n .
!wc -lwc {PATH}wiki.train.tokens
36718 2051910 10797148 data/wikitext-2/wiki.train.tokens
!wc -lwc {PATH}wiki.valid.tokens
3760 213886 1121681 data/wikitext-2/wiki.valid.tokens
TEXT = data.Field(lower=True)
FILES = dict(train='wiki.train.tokens', validation='wiki.valid.tokens', test='wiki.test.tokens')
bs,bptt = 80,70
md = LanguageModelData(PATH, TEXT, **FILES, bs=bs, bptt=bptt, min_freq=10)
len(md.trn_dl), md.nt, len(md.trn_ds), len(md.trn_ds[0].text)
(371, 12981, 1, 2088628)
#md.trn_ds[0].text[:12], next(iter(md.trn_dl))
em_sz = 200
nh = 500
nl = 3
learner = md.get_model(SGD_Momentum(0.7), bs, em_sz, nh, nl)
reg_fn=partial(seq2seq_reg, alpha=2, beta=1)
clip=0.3
learner.fit(10, 1, wds=1e-6, reg_fn=reg_fn, clip=clip)
A Jupyter Widget
[ 0. 5.8902 5.5658]
learner.fit(10, 6, wds=1e-6, reg_fn=reg_fn, cycle_len=1, cycle_mult=2, clip=clip)
A Jupyter Widget
[ 0. 5.5214 5.3178] [ 1. 5.36 5.1435] [ 2. 5.2619 5.0643] [ 3. 5.166 4.9541] [ 4. 5.0275 4.8416] [ 5. 4.9453 4.7837] [ 6. 4.9306 4.7692] [ 7. 4.9195 4.7462] [ 8. 4.8391 4.6762] [ 9. 4.767 4.6317] [ 10. 4.718 4.5859] [ 11. 4.6623 4.5497] [ 12. 4.6262 4.5275] [ 13. 4.6103 4.5196] [ 14. 4.6206 4.5137] [ 15. 4.6716 4.5614] [ 16. 4.6237 4.5333] [ 17. 4.5918 4.4998] [ 18. 4.5519 4.47 ] [ 19. 4.5188 4.4514] [ 20. 4.4907 4.4342] [ 21. 4.464 4.4094] [ 22. 4.4473 4.4016] [ 23. 4.4196 4.3848] [ 24. 4.3956 4.3653] [ 25. 4.384 4.3596] [ 26. 4.3692 4.3493] [ 27. 4.3546 4.3422] [ 28. 4.3548 4.34 ] [ 29. 4.3457 4.3364] [ 30. 4.3517 4.3371] [ 31. 4.4461 4.401 ] [ 32. 4.4343 4.39 ] [ 33. 4.4064 4.3759] [ 34. 4.401 4.374] [ 35. 4.3683 4.3583] [ 36. 4.3535 4.3478] [ 37. 4.3403 4.3402] [ 38. 4.3303 4.3356] [ 39. 4.3214 4.3346] [ 40. 4.2934 4.3226] [ 41. 4.2919 4.3188] [ 42. 4.2742 4.3055] [ 43. 4.2566 4.3009] [ 44. 4.2374 4.2973] [ 45. 4.2297 4.2854] [ 46. 4.217 4.2823] [ 47. 4.2086 4.2732] [ 48. 4.1899 4.2642] [ 49. 4.196 4.2624] [ 50. 4.1726 4.2562] [ 51. 4.182 4.2528] [ 52. 4.1619 4.2527] [ 53. 4.1868 4.248 ] [ 54. 4.1704 4.2455] [ 55. 4.156 4.2425] [ 56. 4.138 4.2428] [ 57. 4.1476 4.2367] [ 58. 4.1454 4.2349] [ 59. 4.1273 4.2312] [ 60. 4.1331 4.2317] [ 61. 4.1471 4.231 ] [ 62. 4.1283 4.2335]
learner.save('lm_420')
learner.fit(10, 6, wds=1e-6, reg_fn=reg_fn, cycle_len=1, cycle_mult=2, clip=clip)
A Jupyter Widget
[ 0. 4.2051 4.242 ] [ 1. 4.2466 4.2633] [ 2. 4.2073 4.2377] [ 3. 4.2657 4.2765] [ 4. 4.2209 4.2627] [ 5. 4.1965 4.2458] [ 6. 4.1746 4.2335] [ 7. 4.2657 4.2897] [ 8. 4.2401 4.277 ] [ 9. 4.2266 4.269 ] [ 10. 4.189 4.2546] [ 11. 4.1639 4.2369] [ 12. 4.1531 4.2288] [ 13. 4.1233 4.2223] [ 14. 4.1328 4.2161] [ 15. 4.2515 4.2909] [ 16. 4.2246 4.2872] [ 17. 4.2081 4.2727] [ 18. 4.1989 4.2651] [ 19. 4.1836 4.2634] [ 20. 4.1853 4.2647] [ 21. 4.1435 4.2518] [ 22. 4.1407 4.2388] [ 23. 4.1227 4.2285] [ 24. 4.1024 4.2244] [ 25. 4.0866 4.218 ] [ 26. 4.0916 4.2135] [ 27. 4.0695 4.2084] [ 28. 4.0615 4.2036] [ 29. 4.0672 4.2034] [ 30. 4.0859 4.2014] [ 31. 4.1937 4.2731] [ 32. 4.1961 4.2628] [ 33. 4.1725 4.2582] [ 34. 4.1909 4.2662] [ 35. 4.1622 4.2586] [ 36. 4.1525 4.2556] [ 37. 4.1601 4.2616] [ 38. 4.1521 4.2604] [ 39. 4.1471 4.2571] [ 40. 4.1131 4.2425] [ 41. 4.1072 4.2362] [ 42. 4.0949 4.2399] [ 43. 4.1014 4.2315] [ 44. 4.0926 4.2266] [ 45. 4.0721 4.2225] [ 46. 4.0694 4.219 ] [ 47. 4.0742 4.2188] [ 48. 4.0433 4.2181] [ 49. 4.0666 4.2083] [ 50. 4.0395 4.2066] [ 51. 4.0591 4.2007] [ 52. 4.0166 4.2046] [ 53. 4.0279 4.1973] [ 54. 4.0178 4.1957] [ 55. 4.0241 4.1937] [ 56. 3.9977 4.1901] [ 57. 3.997 4.1909] [ 58. 4.0194 4.189 ] [ 59. 3.9832 4.1833] [ 60. 4.0069 4.1856] [ 61. 3.9893 4.1873] [ 62. 3.986 4.1828]
learner.save('lm_419')
learner.fit(10, 6, wds=1e-6, reg_fn=reg_fn, cycle_len=1, cycle_mult=2, clip=clip)
A Jupyter Widget
[ 0. 4.0781 4.2001] [ 1. 4.1227 4.2212] [ 2. 4.1012 4.1981] [ 3. 4.1524 4.243 ] [ 4. 4.1137 4.2235] [ 5. 4.0761 4.1977] [ 6. 4.072 4.1966] [ 7. 4.1489 4.2491] [ 8. 4.1355 4.2431] [ 9. 4.1167 4.2323] [ 10. 4.0797 4.2189] [ 11. 4.048 4.2054] [ 12. 4.0284 4.2013] [ 13. 4.0436 4.1854] [ 14. 4.0416 4.1864] [ 15. 4.131 4.2443] [ 16. 4.1205 4.2414] [ 17. 4.116 4.2427] [ 18. 4.1111 4.2443] [ 19. 4.1028 4.2313] [ 20. 4.0806 4.224 ] [ 21. 4.0879 4.2192] [ 22. 4.0452 4.2083] [ 23. 4.0272 4.2042] [ 24. 4.0196 4.2019] [ 25. 4.0273 4.1909] [ 26. 4.0178 4.1868] [ 27. 3.9833 4.1847] [ 28. 3.9777 4.1805] [ 29. 3.9841 4.1805] [ 30. 4.0016 4.181 ] [ 31. 4.1057 4.236 ] [ 32. 4.126 4.2456] [ 33. 4.1018 4.2358] [ 34. 4.0977 4.2421] [ 35. 4.1049 4.2377] [ 36. 4.1079 4.2429] [ 37. 4.0851 4.2365] [ 38. 4.0778 4.2304] [ 39. 4.0849 4.2331] [ 40. 4.058 4.2278] [ 41. 4.047 4.2225] [ 42. 4.0585 4.2251] [ 43. 4.0324 4.2226] [ 44. 4.0459 4.2161] [ 45. 4.0147 4.2054] [ 46. 4.0259 4.2097] [ 47. 3.9967 4.205 ] [ 48. 4.0014 4.1984] [ 49. 3.9926 4.1985] [ 50. 3.968 4.195] [ 51. 3.98 4.1888] [ 52. 3.9527 4.1912] [ 53. 3.9452 4.1864] [ 54. 3.9587 4.1848] [ 55. 3.9286 4.184 ] [ 56. 3.9302 4.1778] [ 57. 3.9271 4.1751] [ 58. 3.9452 4.1743] [ 59. 3.9443 4.1792] [ 60. 3.924 4.176] [ 61. 3.9347 4.1745] [ 62. 3.924 4.1718]
learner.save('lm_418')
math.exp(4.17)
64.71545210740304
m=learner.model
s=[""". <eos> The game began development in 2010 , carrying over a large portion of the work
done on Valkyria Chronicles II . While it retained the standard features of """.split()]
t=TEXT.numericalize(s)
m[0].bs=1
m.reset(False)
res,*_ = m(t)
nexts = torch.topk(res[-1], 10)[1]
[TEXT.vocab.itos[o] for o in to_np(nexts)]
['the', '<unk>', 'a', '"', 'an', 'its', 'this', 'their', 'all', 'other']
for i in range(20):
n=res[-1].topk(2)[1]
n = n[1] if n.data[0]==0 else n[0]
print(TEXT.vocab.itos[n.data[0]], end=' ')
res,*_ = m(n[0].unsqueeze(0))
the film , and the first of the two @-@ year @-@ old , the " black " , a