import nltk
from nltk import CFG
from nltk.tree import Tree

# Part 1: NLTK 기본 모듈을 사용한 결과출력
def definegrammar_pasrereult():
    Grammar = nltk.CFG.fromstring(""" 
    S -> NP VP 
    PP -> P NP 
    NP -> Det N | Det N PP | 'I' 
    VP -> V NP | VP PP 
    Det -> 'an' | 'my' 
    N -> 'elephant' | 'pajamas' 
    V -> 'shot' 
    P -> 'in' 
    """)
    sent   = "I shot an elephant".split()
    parser = nltk.ChartParser(Grammar)
    trees  = parser.parse(sent)
    for tree in trees:
        print (tree)

print ("\n--------Parsing result as per defined grammar-------")
definegrammar_pasrereult()

# Part 2: Tree 함수를 사용한 parse tree 그리기
def draw_parser_tree():
    dp1  = Tree('dp', [Tree('d', ['the']), Tree('np', ['dog'])])
    dp2  = Tree('dp', [Tree('d', ['the']), Tree('np', ['cat'])])
    vp   = Tree('vp', [Tree('v', ['chased']), dp2])
    tree = Tree('s', [dp1, vp])
    print(tree)
    print(tree.pformat_latex_qtree())
    tree.pretty_print()

print ("\n--------Drawing Parse Tree-------")
draw_parser_tree()

import spacy
nlp = spacy.load('en')
doc = nlp(u'London is a big city in the United Kingdom.')
print ("-------Example 1 ------")
for _ in doc.ents:
    print("{} : {}".format(_.label_, _.text))

doc1 = nlp(u'While in France, Christine Lagarde discussed short-term stimulus efforts in a '
           u'recent interview on 5:00 P.M. with the Wall Street Journal')
print ("-------Example 2 ------")
for _ in doc1.ents:
    print("{} : {}".format(_.label_, _.text))

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer

ngram_vectorizer = CountVectorizer(analyzer='char_wb', ngram_range=(2, 2), min_df=1)
counts = ngram_vectorizer.fit_transform(['words', 'wprds'])
ngram_vectorizer.get_feature_names() == ([' w', 'ds', 'or', 'pr', 'rd', 's ', 'wo', 'wp'])
counts.toarray().astype(int)

ngram_vectorizer

text  = 'tf idf, short form of term frequency, inverse document frequency'
text2 = 'is a numerical statistic that is intended to reflect how important'
text3 = 'a word is to a document in a collection or corpus'

from textblob import TextBlob
blob     = TextBlob(text)
blob2    = TextBlob(text2)
blob3    = TextBlob(text3)
bloblist = [blob, blob2, blob3]
bloblist

import math
def tf(word, blob):
    return blob.words.count(word) / len(blob.words)

def idf(word, bloblist):
    # n_containing (0이 없도록 1 smoothing)
    x = 1 + sum(1 for blob in bloblist if word in blob) 
    return math.log(len(bloblist) / (x if x else 1))

def tfidf(word, blob, bloblist):
    return tf(word, blob) * idf(word, bloblist)

tf_score    = tf('short', blob)
idf_score   = idf('short', bloblist)
tfidf_score = tfidf('short', blob, bloblist)
print ("""tf : {:.7}\nidf : {:.7}\ntf*idf : {:.7}""".format(
    str(tf_score), str(idf_score), str(tfidf_score)))

# 문서 1개 호출한 뒤 Token의 생성
import nltk, string, os
from collections import Counter

# 전처리 : 소문자 처리, 문장기호 삭제 등
def get_tokens(file):
    with open(file, 'r') as shakes:
        text = shakes.read()
    
    lowers = text.lower() # 소문자 전처리
    no_punctuation = lowers.translate(str.maketrans('','',string.punctuation)) # 문장기호 삭제
    tokens = no_punctuation.split(' ') # 문장 내용을 Token 으로 분리
    return tokens

tokens = get_tokens('./data/shakes1.txt')
count  = Counter(tokens)
count.most_common(5)

# nlkt 의 stopword 를 활용한 Token 정규화
from nltk.corpus import stopwords

filtered = [w for w in tokens if not w in stopwords.words('english')]
count    = Counter(filtered)
count.most_common(5)

# PorterStemmer 사용 Stemming 활용
from nltk.stem.porter import PorterStemmer

def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

stemmer = PorterStemmer()
stemmed = stem_tokens(filtered, stemmer)
count   = Counter(stemmed)
count.most_common(5)

# this can take some time
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf      = TfidfVectorizer(tokenizer=tokenize, stop_words='english')
tfs        = tfidf.fit_transform(token_dict.values())
sample_str = 'this sentence has unseen text such as computer but also king lord juliet'
response   = tfidf.transform([sample_str])
response.data

feature_names = tfidf.get_feature_names()
for col in response.nonzero()[1]:
    print (feature_names[col], ' - ', response[0, col])

import numpy as np
feature_array = np.array(tfidf.get_feature_names())
tfidf_sorting = np.argsort(response.toarray()).flatten()[::-1]
top_n_tri     = feature_array[tfidf_sorting][: 3]
top_n_forth   = feature_array[tfidf_sorting][: 4]
top_n_tri,  top_n_forth

import pandas as pd
from sklearn.feature_extraction import DictVectorizer
df = pd.DataFrame([['rick','young'],['phil','old']], columns=['name','age-group'])
df

# pandas 모듈을 활용한 Encoding
pd.get_dummies(df)

# Scikit-learn 모듈을 사용한 Tf-IDF
X = pd.DataFrame({'income': [100000,110000,90000,30000,14000,50000],
                  'country':['US', 'CAN', 'US', 'CAN', 'MEX', 'US'],
                  'race':['White', 'Black', 'Latino', 'White', 'White', 'Black']})

v = DictVectorizer()
qualitative_features = ['country']
X_qual = v.fit_transform(X[qualitative_features].to_dict('records'))
v.vocabulary_ ,"\n", X_qual.toarray()