Notebook

tf-idf¶

1 Document 자료를 불러오기¶

In [1]:

# Document 자료를 불러온다 : 2017년 연간결산 리포트
with open('../data/News2017.txt', 'r', encoding='utf-8') as f:
    texts = f.read()

texts = texts.lower()
texts[:300]

Out[1]:

'samsung electronics posted krw 65.98 trillion in consolidated revenue and krw 15.15 trillion in operating profit for the fourth quarter of 2017.\n \noverall, the company reported full-year revenue of krw 239.58 trillion and full-year operating profit of krw 53.65 trillion.\n \nfourth quarter earnings we'

In [2]:

# 영문 Token만 추출한다
from nltk.tokenize import RegexpTokenizer
re_capt  = RegexpTokenizer(r'[a-z]\w+')
tokens   = re_capt.tokenize(texts)
document = " ".join(tokens)
document[:300]

Out[2]:

'samsung electronics posted krw trillion in consolidated revenue and krw trillion in operating profit for the fourth quarter of overall the company reported full year revenue of krw trillion and full year operating profit of krw trillion fourth quarter earnings were driven by the components business '

In [3]:

# 추출한 Token의 빈도를 계산한다
from nltk import FreqDist
import pandas as pd
token_freq = FreqDist(tokens)
token_freq = pd.Series(token_freq).sort_values(ascending=False)
token_freq[:10]

Out[3]:

the       153
and       100
to         95
in         77
for        74
of         53
will       37
as         35
demand     33
by         32
dtype: int64

2 sklean 을 활용한 tf idf 계산¶

sklearn의 기본 데이터를 활용하여 tf-idf 결과값 출력

In [4]:

# ! pip3 install sklearn

In [5]:

# ! pip3 install scipy

In [6]:

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vec   = TfidfVectorizer(stop_words='english')
transformed = tfidf_vec.fit_transform(raw_documents = [document])
transformed = np.array(transformed.todense())
transformed

Out[6]:

array([[0.01705916, 0.00852958, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.01705916, 0.00852958, 0.03411832,
        0.02558874, 0.01705916, 0.00852958, 0.00852958, 0.00852958,
        0.03411832, 0.02558874, 0.03411832, 0.01705916, 0.00852958,
        0.01705916, 0.0426479 , 0.03411832, 0.00852958, 0.00852958,
        0.00852958, 0.05970706, 0.00852958, 0.0426479 , 0.01705916,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.01705916,
        0.00852958, 0.03411832, 0.02558874, 0.00852958, 0.00852958,
        0.03411832, 0.01705916, 0.00852958, 0.03411832, 0.01705916,
        0.00852958, 0.00852958, 0.00852958, 0.01705916, 0.27294655,
        0.0426479 , 0.01705916, 0.00852958, 0.00852958, 0.01705916,
        0.00852958, 0.00852958, 0.00852958, 0.01705916, 0.00852958,
        0.00852958, 0.01705916, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.01705916, 0.02558874, 0.00852958, 0.00852958,
        0.01705916, 0.15353243, 0.00852958, 0.00852958, 0.03411832,
        0.05970706, 0.00852958, 0.01705916, 0.00852958, 0.00852958,
        0.03411832, 0.00852958, 0.00852958, 0.0426479 , 0.02558874,
        0.01705916, 0.03411832, 0.01705916, 0.00852958, 0.01705916,
        0.00852958, 0.00852958, 0.0426479 , 0.01705916, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.0426479 , 0.01705916,
        0.00852958, 0.02558874, 0.01705916, 0.02558874, 0.02558874,
        0.00852958, 0.01705916, 0.05970706, 0.28147613, 0.00852958,
        0.06823664, 0.03411832, 0.00852958, 0.01705916, 0.01705916,
        0.01705916, 0.00852958, 0.00852958, 0.03411832, 0.00852958,
        0.06823664, 0.01705916, 0.00852958, 0.05970706, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.05117748, 0.00852958,
        0.01705916, 0.02558874, 0.00852958, 0.00852958, 0.01705916,
        0.18765075, 0.00852958, 0.00852958, 0.00852958, 0.01705916,
        0.00852958, 0.02558874, 0.05117748, 0.00852958, 0.10235496,
        0.00852958, 0.01705916, 0.01705916, 0.00852958, 0.01705916,
        0.02558874, 0.00852958, 0.0852958 , 0.07676622, 0.03411832,
        0.00852958, 0.14500285, 0.05117748, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.0852958 , 0.00852958,
        0.03411832, 0.00852958, 0.00852958, 0.09382538, 0.02558874,
        0.00852958, 0.02558874, 0.01705916, 0.00852958, 0.00852958,
        0.00852958, 0.03411832, 0.05970706, 0.01705916, 0.01705916,
        0.0426479 , 0.00852958, 0.00852958, 0.00852958, 0.02558874,
        0.02558874, 0.02558874, 0.0426479 , 0.01705916, 0.10235496,
        0.0426479 , 0.00852958, 0.13647327, 0.01705916, 0.00852958,
        0.00852958, 0.02558874, 0.00852958, 0.00852958, 0.03411832,
        0.00852958, 0.00852958, 0.05117748, 0.01705916, 0.00852958,
        0.00852958, 0.07676622, 0.06823664, 0.07676622, 0.05970706,
        0.00852958, 0.01705916, 0.02558874, 0.00852958, 0.00852958,
        0.00852958, 0.03411832, 0.02558874, 0.00852958, 0.01705916,
        0.01705916, 0.00852958, 0.13647327, 0.05970706, 0.00852958,
        0.01705916, 0.01705916, 0.00852958, 0.03411832, 0.10235496,
        0.00852958, 0.0426479 , 0.03411832, 0.05117748, 0.00852958,
        0.00852958, 0.00852958, 0.0426479 , 0.00852958, 0.00852958,
        0.05117748, 0.05970706, 0.02558874, 0.00852958, 0.02558874,
        0.02558874, 0.02558874, 0.00852958, 0.01705916, 0.0426479 ,
        0.01705916, 0.00852958, 0.00852958, 0.06823664, 0.00852958,
        0.00852958, 0.01705916, 0.00852958, 0.14500285, 0.0426479 ,
        0.02558874, 0.02558874, 0.00852958, 0.00852958, 0.00852958,
        0.01705916, 0.05970706, 0.00852958, 0.03411832, 0.00852958,
        0.02558874, 0.00852958, 0.01705916, 0.16206201, 0.00852958,
        0.0426479 , 0.00852958, 0.00852958, 0.00852958, 0.05970706,
        0.01705916, 0.00852958, 0.01705916, 0.02558874, 0.01705916,
        0.15353243, 0.01705916, 0.0426479 , 0.0426479 , 0.01705916,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.09382538,
        0.00852958, 0.00852958, 0.01705916, 0.06823664, 0.00852958,
        0.00852958, 0.00852958, 0.01705916, 0.01705916, 0.01705916,
        0.00852958, 0.00852958, 0.02558874, 0.00852958, 0.00852958,
        0.00852958, 0.07676622, 0.06823664, 0.00852958, 0.00852958,
        0.01705916, 0.01705916, 0.01705916, 0.00852958, 0.00852958,
        0.00852958, 0.01705916, 0.01705916, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.06823664, 0.00852958,
        0.00852958, 0.00852958, 0.14500285, 0.01705916, 0.00852958,
        0.03411832, 0.01705916, 0.00852958, 0.02558874, 0.0852958 ,
        0.05117748, 0.02558874, 0.22176907, 0.06823664, 0.05117748,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.01705916,
        0.00852958, 0.00852958, 0.00852958, 0.01705916, 0.01705916,
        0.07676622, 0.00852958, 0.00852958, 0.19618033, 0.00852958,
        0.00852958, 0.02558874, 0.01705916, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.03411832,
        0.01705916, 0.02558874, 0.02558874, 0.01705916, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.01705916, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.07676622, 0.02558874,
        0.00852958, 0.01705916, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.17912117, 0.24735781, 0.02558874, 0.00852958,
        0.01705916, 0.01705916, 0.02558874, 0.00852958, 0.13647327,
        0.00852958, 0.01705916, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.03411832, 0.00852958, 0.03411832,
        0.00852958, 0.0426479 , 0.05117748, 0.02558874, 0.01705916,
        0.01705916, 0.05970706, 0.00852958, 0.02558874, 0.01705916,
        0.00852958, 0.01705916, 0.00852958, 0.06823664, 0.09382538,
        0.00852958, 0.00852958, 0.05117748, 0.00852958, 0.03411832,
        0.00852958, 0.00852958, 0.02558874, 0.02558874, 0.02558874,
        0.01705916, 0.00852958, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.01705916, 0.00852958, 0.0426479 , 0.00852958,
        0.05970706, 0.00852958, 0.12794369, 0.00852958, 0.03411832,
        0.00852958, 0.00852958, 0.00852958, 0.01705916, 0.05970706,
        0.00852958, 0.05970706, 0.05117748, 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.12794369, 0.00852958, 0.05970706,
        0.05970706, 0.00852958, 0.0426479 , 0.00852958, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.00852958,
        0.00852958, 0.03411832, 0.00852958, 0.00852958, 0.01705916,
        0.00852958, 0.00852958, 0.02558874, 0.12794369, 0.00852958,
        0.00852958, 0.00852958, 0.00852958, 0.00852958, 0.06823664,
        0.00852958, 0.06823664]])

In [7]:

index_value   = {i[1]:i[0] for i in tfidf_vec.vocabulary_.items()}
fully_indexed = {index_value[column]:value  for row in transformed  
                                            for (column,value) in enumerate(row)}

token_tfidf = pd.Series(fully_indexed).sort_values(ascending=False)
token_tfidf[:10]

Out[7]:

demand      0.281476
business    0.272947
samsung     0.247358
products    0.221769
quarter     0.196180
earnings    0.187651
sales       0.179121
mobile      0.162062
new         0.153532
company     0.153532
dtype: float64