# -*- encoding:utf8 -*-
import konlpy
from konlpy.tag import Mecab
print('konlpy version = %s' % konlpy.__version__)

sent = '트와이스와 아이오아이가 활동시절 박근혜 최순실의 국정농단 사태는 대통령탄핵으로 이어졌습니다'
"MeCab() 명사추출 :", Mecab().nouns(sent)[:6]

%%time
from soynlp.utils import DoublespaceLineCorpus
from soynlp.noun import LRNounExtractor_v2

corpus_path = './data/news_2016-10-20.txt'
corpus = DoublespaceLineCorpus(corpus_path, iter_sent=True) # Soynlp Corpus 변환
noun_extractor = LRNounExtractor_v2(verbose=True)           # Soynlp 모델의 정의
nouns = noun_extractor.train_extract(corpus)                # 모델의 훈련

list(noun_extractor._compounds_components.items())[:5]

nouns['뉴스'] # NounScore(frequency=4319, score=1.0)

nouns['부산과학기술기획평가']

# nouns['두바이월드센터시카고옵션거래소']
noun_extractor.decompose_compound('두바이월드센터시카고옵션거래소를')

# nouns['두바이월드센터시카고옵션거래소']
noun_extractor.decompose_compound('햄버거짬뽕')

noun_extractor.decompose_compound('한국아이티비지니스진흥협회의')

# 단 영어가 포함된 경우에는 엔트로피 측정이 잘 안됨에 주의
noun_extractor.decompose_compound('한국IT비지니스진흥협회에서')

# 학습내용과 무관한 단어들은 구분능력이 떨어짐 (당연하지..)
noun_extractor.decompose_compound('복만두를강냉이와')

noun_extractor.lrgraph.get_r('아이오아이')

# topk=10 으로 설정되어 있습니다. topk < 0 으로 설정하면 모든 R set 이 출력됩니다.
noun_extractor.lrgraph.get_r('아이오아이', topk=-1)

# L-R 구조의 L parts 도 확인할 수 있습니다. 이 역시 topk=10 으로 기본값이 설정되어 있습니다.
noun_extractor.lrgraph.get_l('었다고')

%%time
from soynlp.word import WordExtractor

word_extractor = WordExtractor(
    min_frequency=100,
    min_cohesion_forward=0.05, 
    min_right_branching_entropy=0.0
)

print('num sentences = {:,}'.format(len(corpus)))
word_extractor.train(corpus)
words = word_extractor.extract()
len(words)

import math
def word_score(score):
    return (score.cohesion_forward * math.exp(score.right_branching_entropy))

print('단어   (빈도수, cohesion, branching entropy)\n')
for word, score in sorted(words.items(), 
                          key = lambda x:word_score(x[1]), 
                          reverse = True)[:30]:
    print('{:}\t({:5,}, {:.3f}, {:.3f})'.format(
            word, 
            score.leftside_frequency, 
            score.cohesion_forward,
            score.right_branching_entropy))

%%time
# Cohesion score, Branching Entropy, Accessor Variety 에 대하여 각각의 점수만 이용하고 싶은 경우에는 다음의 함수를 이용합니다
cohesion_scores = word_extractor.all_cohesion_scores()
cohesion_scores['아이오아이'] # (cohesion_forward, cohesion_backward)

%%time
# Cohesion score, Branching Entropy, Accessor Variety 에 대하여 각각의 점수만 이용하고 싶은 경우에는 다음의 함수를 이용합니다
cohesion_scores = word_extractor.all_cohesion_scores()
cohesion_scores['아이오아이'] # (cohesion_forward, cohesion_backward)

%%time
branching_entropy = word_extractor.all_branching_entropy()
branching_entropy['아이오아이'] # (left_branching_entropy, right_branching_entropy)

%%time
accessor_variety = word_extractor.all_accessor_variety()
accessor_variety['아이오아이'] # (left_accessor_variety, right_accessor_variety)

%%time
from soynlp.tokenizer import NounLMatchTokenizer, NounMatchTokenizer

# noun_scores[noun] = (score, frequency) : 학습을 위한 데이터로 변환 합니다.
noun_scores      = noun_extractor.train_extract(corpus)
lmatch_tokenizer = NounLMatchTokenizer(nouns = noun_scores)

lmatch_tokenizer.tokenize('오늘의 연합뉴스날씨입니다.')

lmatch_tokenizer.tokenize('마파두부야채볶음')

lmatch_tokenizer.tokenize('오늘의 연합뉴스날씨입니다.', compose_compound=False)

lmatch_tokenizer.tokenize('오늘의 연합뉴스 날씨입니다.')

lmatch_tokenizer.tokenize('오늘의 ㅋ연합뉴스날씨입니다.')

# NounMatchTokenizer() 는 위 함수대로 작동하지 않습니다,
# match_tokenizer  = NounMatchTokenizer(noun_scores = noun_scores)
# match_tokenizer.tokenize('오늘의 연합뉴스날씨입니다.')
# match_tokenizer.tokenize('오늘의 ㅋ연합뉴스 날씨입니다.')
# match_tokenizer.tokenize('오늘의 ㅋ연합뉴스날씨입니다.', compose_compound=False)

import re
import pandas as pd
df = pd.read_csv('./data/petition.csv', parse_dates=['start', 'end'])
p  = r'.*(대출|집값|부동산|LTV).*'
estate = df[ df['title'].str.match(p) | df['content'].str.match(
                p, flags=re.MULTILINE) ].reset_index(drop=True)
print(estate.shape)

# 특정 인덱스 문서를 Sample 로 추출 합니다.
sample_index   = 15  # 샘플문서 인덱스
sample_title   = estate['title'][sample_index]
sample_content = estate['content'][sample_index]
sample_title,  sample_content[:100]

# 토큰화
from soynlp.tokenizer import RegexTokenizer, LTokenizer, MaxScoreTokenizer
tokenizer       = RegexTokenizer()
tokened_title   = tokenizer.tokenize(sample_title)
tokened_content = tokenizer.tokenize(sample_content)

print(len(tokened_title), len(tokened_content))
tokened_title, tokened_content[:10]

%%time
# Preprocessing (개행문자 제거)
def preprocessing(text):
    text = re.sub('\\\\n', ' ', text)
    return text

sentences = estate['content'].apply(preprocessing)
tokens    = sentences.apply(tokenizer.tokenize)
tokens[:3], tokens[sample_index][:10]

%%time
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def displayWordCloud(data = None, backgroundcolor = 'white', width=800, height=300):
    wordcloud = WordCloud( font_path = fontpath,
                           background_color = backgroundcolor, 
                           width = width, height = height).generate(data)
    plt.figure(figsize = (12 , 8))
    plt.imshow(wordcloud); plt.axis("off"); plt.show()

%matplotlib inline
import matplotlib.font_manager as fm
fontpath = "/home/markbaum/.local/share/fonts/D2Coding.ttf"
font = fm.FontProperties(fname=fontpath, size=9)
displayWordCloud(' '.join(sentences))

%%time
# 추출된 명사를 찍어봅니다.
from soynlp.noun import LRNounExtractor
noun_extractor = LRNounExtractor(verbose=True)
noun_extractor.train(sentences)
nouns = noun_extractor.extract()

print("추출한 명사갯수 {:,}".format(len(nouns)))
displayWordCloud(' '.join(nouns))