# 개별 단어내 의미를 가는 단위로 분할 합니다
text = "자연어 처리는 인공지능의 한 줄기 입니다"

from konlpy.tag import Mecab
Mecab().morphs(text)

# 문서 내 Token의 빈도수 계산
import pandas as pd
def get_term_frequency(document, word_dict=None):
    if word_dict is None:
        word_dict = {}
    words = document.split()

    for w in words:
        word_dict[w] = 1 + (0 if word_dict.get(w) is None else word_dict[w])
    return pd.Series(word_dict).sort_values(ascending=False)

get_term_frequency(doc1)

# 개별 token 이 몇개의 문서에서 출현하였는지를 Counting
def get_document_frequency(documents):
    dicts, df = [], {}
    vocab = set([])
    for d in documents:
        tf    = get_term_frequency(d)
        dicts += [tf]
        vocab = vocab | set(tf.keys())
    
    for v in list(vocab):
        df[v] = 0
        for dict_d in dicts:
            if dict_d.get(v) is not None:
                df[v] += 1
    return pd.Series(df).sort_values(ascending=False)

get_document_frequency([doc1, doc2])

# 전체 문서의 token 별 tf-idf 계산하는 함수
from operator import itemgetter
import numpy as np

def get_tfidf(docs):
    vocab, tfs = {}, []
    for d in docs:
        vocab =  get_term_frequency(d, vocab)
        tfs  += [get_term_frequency(d)]
    df = get_document_frequency(docs)

    stats = []
    for word, freq in vocab.items():
        tfidfs = []
        for idx in range(len(docs)):
            if tfs[idx].get(word) is not None:
                tfidfs += [tfs[idx][word] * np.log(len(docs) / df[word])]
            else: tfidfs += [0]
        stats.append((word, freq, *tfidfs, max(tfidfs)))
    
    _col = ('word','frequency','doc1','doc2','doc3','max')
    return pd.DataFrame(stats, columns=_col
            ).sort_values('max', ascending=False).reset_index(drop=True)

get_tfidf([doc1, doc2, doc3]).head()

# 전체 문서의 tf 벡터 만들기 (tf-idf 와 비교활용이 가능)
def get_tf(docs):
    vocab, tfs = {}, []
    for d in docs:
        vocab = get_term_frequency(d, vocab)
        tfs  += [get_term_frequency(d)]

    stats = []
    for word, freq in vocab.items():
        tf_v = []
        for idx in range(len(docs)):
            if tfs[idx].get(word) is not None:
                tf_v += [tfs[idx][word]]
            else: tf_v += [0]
        stats.append((word, freq, *tf_v))
    _col = ('word','frequency','doc1','doc2','doc3')
    return pd.DataFrame(stats, columns=_col).sort_values('frequency', ascending=False)

get_tf([doc1, doc2, doc3]).head()

%%time
from collections import defaultdict
import pandas as pd
with open('../backup/ted.aligned.ko.refined.tok.txt') as f:
    lines = [l.strip() for l in f.read().splitlines() if l.strip()]

# context window 를 생성하는 함수
def get_context_counts(lines, w_size=2):
    co_dict   = defaultdict(int)
    for line in lines:
        words = line.split()
        for i, w in enumerate(words):
            for c in words[i - w_size:i + w_size]:
                if w != c: co_dict[(w, c)] += 1
    return pd.Series(co_dict)

co_dict = get_context_counts(lines)
co_dict

tfs = get_term_frequency(' '.join(lines))
tfs

tfs = tfs[tfs < 100000]
tfs.head()

%%time
# 앞에서 작성한 동시발생 정보를 활용하여 벡터 데이터를 생성합니다
def co_occurrence(co_dict, vocab):
    data = []
    for word1 in vocab:
        row = []
        for word2 in vocab:
            try: count = co_dict[(word1, word2)]
            except KeyError: count = 0
            row.append(count)   
        data.append(row)
    return pd.DataFrame(data, index=vocab, columns=vocab)
      
import torch
co = co_occurrence(co_dict, tfs.index[:1000])
torch.save(co, 'co.pth')  # 결과를 co.pth 로 저장 합니다
co.head()

import torch
def get_nearest(query, dataframe, metric, top_k, ascending=True):
    vector = torch.FloatTensor(dataframe.loc[query].values)
    distances = dataframe.apply(lambda x: metric(vector, torch.FloatTensor(x.values)), axis=1)
    top_distances = distances.sort_values(ascending=ascending)[:top_k]
    print(', '.join([f'{k} ({v:.1f})' for k, v in top_distances.items()]))

# "우리" 단어와 l1 거리측정 함수를 활용한 계산  
def get_l1_distance(x1, x2):
    return ((x1 - x2).abs()).sum()

print('L1 distance:')
get_nearest('우리', co, get_l1_distance, 30)

# l2 거리측정 함수
def get_l1_distance(x1, x2):
    return ((x1 - x2).abs()).sum()

print('L1 distance:')
get_nearest('우리', co, get_l1_distance, 30)

def get_infinity_distance(x1, x2):
    return ((x1 - x2).abs()).max()

print('\nInfinity distance:')
get_nearest('우리', co, get_infinity_distance, 30)

def get_cosine_similarity(x1, x2):
    return (x1 * x2).sum() / ((x1**2).sum()**.5 * (x2**2).sum()**.5)

print('\nCosine similarity:')
get_nearest('우리', co, get_cosine_similarity, 30, ascending=False)

def get_jaccard_similarity(x1, x2):
    return torch.stack([x1, x2]).min(dim=0)[0].sum() / torch.stack([x1, x2]).max(dim=0)[0].sum()

print('\nJaccard similarity:')
get_nearest('우리', co, get_jaccard_similarity, 30, ascending=False)

from nltk.corpus import wordnet as wn
for ss in wn.synsets('bass'):
    print(ss, ss.definition())

from nltk.wsd import lesk
def lesk_token(sentence, word):
    sentence = sentence.split()
    best_synset = lesk(sentence, word)
    return best_synset, best_synset.definition()

sentence1 = 'I went fishing last weekend and I got a bass and cooked it'
sentence2 = 'I love the music from the speaker which has strong beat and bass'
sentence3 = 'I think the bass is more important than guitar'

word = 'bass'
for _ in [sentence1, sentence2, sentence3]:
    print(lesk_token(str(_), word))

%%time
# 데이터 불러오기
with open('../backup/ted.aligned.ko.refined.tok.txt') as f:
    lines = [l.strip() for l in f.read().splitlines() if l.strip()]

# Seen(w) 함수의 계산 : 문장 내 서술어("VV") 와 표제어("NNG") 를 찾는다 
from konlpy.tag import Mecab
def count_seen_headwords(lines, predicate='VV', headword='NNG'):
    tagger, seen_dict  = Mecab(), {}    
    for line in lines:
        pos_result     = tagger.pos(line)
        word_h, word_p = None, None
        for word, pos in pos_result:
            if pos == predicate or pos[:3] == predicate + '+':
                word_p = word; break
            if pos == headword:
                word_h = word
        
        if word_h is not None and word_p is not None:
            seen_dict[word_p] = [word_h] +\
                ([] if seen_dict.get(word_p) is None else seen_dict[word_p])            
    return seen_dict

seen_headwords = count_seen_headwords(lines)
list(seen_headwords.keys())[:10] # key 값들 살펴보기
seen_headwords[list(seen_headwords.keys())[0]][:10]

co = torch.load('co.pth')

# 계산 결과를 활용하여 주어진 술어에 대해 올바른 headword 를 찾는 wsd 함수
def get_cosine_similarity(x1, x2):
    return (x1 * x2).sum() / ((x1**2).sum()**.5 * (x2**2).sum()**.5)

def wsd(predicate, headwords):
    selectional_associations = []
    for h in headwords:
        selectional_associations += [get_selectional_association(
            predicate, h, lines, co, get_cosine_similarity)]
    print(selectional_associations)
    
wsd('가', ['학교', '사람', '질문'])
# wsd('피우', ['담배', '맥주', '사과'])