#!/usr/bin/env python
# coding: utf-8

# <br></br>
# # **gensim | word2vec**
# <br></br>
# ## **1 데이터 전처리**

# In[1]:


import glob
from txtutil import txtnoun
# 2015 ~ 2018 지속가능 경영 보고서 Token을 수집
filelist = glob.glob('../data/kr-Report_201?.txt')
print(filelist)

# 불러온 Document 명사Token만 추출
skiplist = {'갤러시':'갤럭시', '가치창출':'가치창출'}
texts    = [txtnoun(file, skip=skiplist)  for file in filelist]
texts    = " ".join(texts)
texts[:300]


# In[2]:


# 명사 Token 작업된 자료를 ssResport.txt 로 저장 
texts_file = '../data/ssResport.txt'
with open(texts_file, 'w', encoding='utf-8') as file:
    file.write(texts)


# In[3]:


# ! cat ./data/ssResport.txt | head -n 10


# <br></br>
# ## **2 Word 2 vec 객체 만들기**
# gensim

# In[4]:


# ! pip3 install gensim


# In[5]:


get_ipython().run_cell_magic('time', '', 'texts_file = \'../data/ssResport.txt\'\n\nfrom gensim.models import word2vec\ndata  = word2vec.LineSentence(texts_file)\nmodel = word2vec.Word2Vec(data, size=200, window=2, min_count=20, hs=1,\n                          workers=4, iter=100, sg=1)\nmodel.save("../data/ssReport.model")\nprint("model saved.")\n')


# <br></br>
# ## **3 저장된 객체 활용**
# gensim

# In[6]:


get_ipython().run_line_magic('reset', '')


# In[7]:


get_ipython().run_line_magic('who', '')


# In[8]:


from gensim.models import word2vec
model = word2vec.Word2Vec.load('../data/ssReport.model')
len(model.wv.vocab.keys())


# In[9]:


list(model.wv.index2word)[:10]


# In[10]:


model.wv.most_similar(positive=['삼성전자'])


# In[11]:


model.wv.most_similar(negative=['삼성전자'])


# In[12]:


model.wv.most_similar(positive=['글로벌'])


# In[13]:


model.wv.most_similar(negative=['글로벌'])


# In[14]:


model.wv.most_similar(positive=['삼성전자','경영활동'], 
                      negative=['근무환경']) # 담당자, 직원


# <br></br>
# ## **04 Visulaization**
# gensim

# In[15]:


list(model.wv.vocab.keys())[:10]


# In[16]:


# model.wv.vocab : { word: object of numeric vector }
vocab  = list(model.wv.vocab)
X      = model[vocab]
X.shape


# In[21]:


get_ipython().run_cell_magic('time', '', 'from sklearn.manifold import TSNE\ntsne   = TSNE(n_components = 2)\nX_tsne = tsne.fit_transform(X)\n')


# In[22]:


import pandas as pd
df = pd.DataFrame(X_tsne, 
                  index = vocab, 
                  columns=['x', 'y'])
df.head()


# In[23]:


get_ipython().run_line_magic('matplotlib', 'inline')
from matplotlib import rc
rc('font', family=['NanumGothic','Malgun Gothic'])

import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12,12))
ax  = fig.add_subplot(1, 1, 1)
ax.scatter(df['x'], df['y'])
for word, pos in df.iterrows():
    ax.annotate(word, pos)
plt.grid(True)