#!/usr/bin/env python
# coding: utf-8

# <br></br>
# # **Project4 | word2vec**
# <br></br>
# ## **1 데이터 전처리**

# In[ ]:


# 살인의 추억 텍스트 불러오기
filename = './data/movie_memories_of_murder_2003.txt'
with open(filename, 'r') as f:
    texts = f.read()
texts[:500]


# In[ ]:


from txtutil import txtnoun
skips = {'두만':'박두만', '태윤':'서태윤', '용구':'조용구', '귀옥':'권귀옥', 
         '희봉':'구희봉', '동철':'신동철', '광호':'백광호', '병순':'조병순', 
         '해일':'박해일', '광호의':'백광호', '백광호의':'백광호'}
get_ipython().run_line_magic('time', "texts  = txtnoun(filename, skip=skips, tags=['Noun'])")
texts[:500]


# In[ ]:


# 명사 Token 작업된 자료를 ssResport.txt 로 저장 
texts_file = './data/mom_noun_script.txt'
with open(texts_file, 'w', encoding='utf-8') as file:
    file.write(texts)


# In[ ]:


# ! cat ./data/ssResport.txt | head -n 10


# <br></br>
# ## **2 Word 2 vec 객체 만들기**
# gensim

# In[ ]:


# ! pip3 install gensim


# In[ ]:


get_ipython().run_cell_magic('time', '', 'texts_file = \'./data/mom_noun_script.txt\'\n\nfrom gensim.models import word2vec\ndata  = word2vec.LineSentence(texts_file)\nmodel = word2vec.Word2Vec(data, size=30, window=2, min_count=10, hs=1,\n                          workers=4, iter=100, sg=1)\nmodel.save("./data/mom_script.model")\nprint("model saved.")\n')


# <br></br>
# ## **3 저장된 모델 불러오기 및 확인**
# gensim

# In[ ]:


get_ipython().run_line_magic('reset', '')


# In[ ]:


get_ipython().run_line_magic('who', '')


# In[ ]:


from gensim.models import word2vec
model = word2vec.Word2Vec.load('./data/mom_script.model')
len(model.wv.vocab.keys())


# In[ ]:


list(model.wv.index2word)


# <br></br>
# ## **4 Word2Vec 모델 내용 확인**
# gensim

# In[ ]:


model.wv.most_similar(positive=['범인'])


# In[ ]:


model.wv.most_similar(negative=['범인'])


# In[ ]:


model.wv.most_similar(positive=['피해자'])


# In[ ]:


model.wv.most_similar(negative=['피해자'])


# In[ ]:


model.wv.most_similar(positive=['박두만'])


# In[ ]:


model.wv.most_similar(positive=['서태윤'])


# In[ ]:


model.wv.most_similar(positive=['조용구'])


# In[ ]:


model.wv.most_similar(positive=['박두만','서태윤'])


# In[ ]:


model.wv.most_similar(positive=['박두만','서태윤'],
                      negative=['피해자'])


# In[ ]:


model.wv.most_similar(positive=['박두만','서태윤'],
                      negative=['박해일'])


# In[ ]:


model.wv.most_similar(positive=['박두만','서태윤'],
                      negative=['범인'])


# <br></br>
# ## **5 Visulaization**
# gensim

# In[ ]:


list(model.wv.vocab.keys())[:10]


# In[ ]:


# model.wv.vocab : { word: object of numeric vector }
vocab  = list(model.wv.vocab)
X      = model[vocab]
X.shape


# In[ ]:


get_ipython().run_cell_magic('time', '', 'from sklearn.manifold import TSNE\ntsne   = TSNE(n_components= 2)\nX_tsne = tsne.fit_transform(X)\n')


# In[ ]:


import pandas as pd
df = pd.DataFrame(X_tsne, 
                  index = vocab, 
                  columns=['x', 'y'])
df.head()


# In[ ]:


get_ipython().run_line_magic('matplotlib', 'inline')
from matplotlib import rc
rc('font', family=['NanumGothic','Malgun Gothic'])

import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12,12))
ax  = fig.add_subplot(1, 1, 1)
ax.scatter(df['x'], df['y'])
for word, pos in df.iterrows():
    ax.annotate(word, pos, fontsize=15)
plt.grid(True)