Ch11 문서 분석 시스템 만들기¶

vocabulary = {} # 단어집을 처리
import numpy as np
with open('data/SMSSpamCollection') as file_handle:
    for line in file_handle:   # 파일을 한 줄씩 읽는다
        splits = line.split()  # 한 줄을 빈 칸으로 쪼개서 리스트로 변환
        label  = splits[0]     # 맨 앞 단어는 레이블로 구분
        text   = splits[1:]
        for word in text:      # 단어 단위로 고유번호를 부여
            lower = word.lower()
            if not lower in vocabulary:
                vocabulary[lower] = len(vocabulary)

# 단어집의 내용을 출력
print("전체 단어의 수", len(vocabulary))
for no, word in enumerate(vocabulary):
    if no < 5: print(no, word)

features = []  # features 리스트 생성
with open('data/SMSSpamCollection') as file_handle:
    for line in file_handle:                
        splits  = line.split()
        feature = np.zeros(len(vocabulary))
        text    = splits[1:]
        for word in text:   # Vocabulary 피처 갯수
            lower = word.lower()
            feature[vocabulary[lower]] += 1

        # 단어 빈도 피처를 문서 총 단어 갯수로 벡터를 나누어 피처를 생성
        feature = feature / sum(feature)
        features.append(feature)

print("단어빈도 피쳐의 수", len(features))
features[:5]

labels = [] # 레이블을 생성 합니다.
with open('data/SMSSpamCollection') as file_handle:
    for line in file_handle: # 파일을 한 줄씩 읽습니다
        splits = line.split()
        label  = splits[0]
        # 맨 앞 단어(label)이 spam 1, 아니면 0
        if label == 'spam': labels.append(1)
        else: labels.append(0)

print("스팸 레이블 피쳐의 수", len(labels))
labels[:15]

import pickle, warnings
warnings.simplefilter("ignore", category=FutureWarning)
with open('data/processed.pickle', 'rb') as file_handle:
    vocabulary, features, labels = pickle.load(file_handle)

total_number   = len(labels)
middle_index   = total_number//2 # 50%를 Train, 나머지를 Test

# 학습 데이터 분류 후 학습
train_features = features[ :middle_index, :]
train_labels   = labels[ :middle_index]

from sklearn.linear_model import LogisticRegression
classifier     = LogisticRegression()
classifier.fit(train_features, train_labels)

# 검증 데이터 분류허가
test_features  = features[middle_index: , :]
test_labels    = labels[middle_index: ]

print("train accuracy: {:.5f}\ntest accuracy : {:.5f}".format(
    classifier.score(train_features, train_labels),
    classifier.score(test_features, test_labels)))

# 어떤 항목이 판별에 영향을 많이 줬는지 찾아보기
weights, pairs = classifier.coef_[0, :], []
for index, value in enumerate(weights):
    pairs.append( (abs(value), vocabulary[index]) )

# 생성된 pairs 자료를 점수 순서대로 정렬합니다
pairs.sort(key=lambda x: x[0], reverse=True)
for pair in pairs[:10]:
    print("score {:.4f} 단어: {}".format(
        pair[0], pair[1]))

spam_header, no_spam_header = 'spam\t', 'ham\t'
documents                   = []

# 단순히 문서를 추출
with open('data/SMSSpamCollection') as file_handle:
    for line in file_handle:
        if line.startswith(spam_header):
            documents.append(line[len(spam_header):])
        elif line.startswith(no_spam_header):
            documents.append(line[len(no_spam_header):])

# 단어 feacture를 만듭니다
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
warnings.simplefilter("ignore", category=PendingDeprecationWarning)
vectorizer  = CountVectorizer(stop_words='english', max_features=2000)
term_counts = vectorizer.fit_transform(documents)
vocabulary  = vectorizer.get_feature_names()
            
# LDA는 단어출현 갯수로 동작하므로 CountVectorizer를 활용
from sklearn.decomposition import LatentDirichletAllocation
topic_model = LatentDirichletAllocation(n_components=10)
topic_model.fit(term_counts) # 토픽 모델을 학습

# 학습된 토픽들을 하나씩 출력합니다.
topics = topic_model.components_
for topic_id, weights in enumerate(topics):
    print('topic %d' % topic_id, end=': ')
    pairs = [(abs(value), vocabulary[term_id]) for term_id, value in enumerate(weights)]
    pairs.sort(key=lambda x: x[0], reverse=True)
    for pair in pairs[:10]:
        print(pair[1], end=',')
    print()

# 분석할 Document
text = """One day in November 2016, the two authors of this book, 
Seungyeon and Youngjoo, had a coffee at Red Rock cafe, 
which is a very popular place in Mountain View."""

from nltk.tag import StanfordPOSTagger
from nltk.tokenize import word_tokenize
STANFORD_POS_MODEL_PATH = 'data/stanford-postagger-2018-10-16/models/english-bidirectional-distsim.tagger'
STANFORD_POS_JAR_PATH   = 'data/stanford-postagger-2018-10-16/stanford-postagger-3.9.2.jar'
pos_tagger = StanfordPOSTagger(STANFORD_POS_MODEL_PATH, STANFORD_POS_JAR_PATH)
tokens     = word_tokenize(text)
print(tokens, '\n\n', pos_tagger.tag(tokens))

# 동사와 명사만 뽑아봅시다.
noun_and_verbs = [token[0]  for token in pos_tagger.tag(tokens)
                  if token[1].startswith('V') or token[1].startswith('N')]
print("\n동사, 명사추출: ", ', '.join(noun_and_verbs))

# 분석할 Document
text = """One day in November 2016, the two authors of this book, 
Seungyeon and Youngjoo, had a coffee at Red Rock cafe, 
which is a very popular place in Mountain View."""

STANFORD_NER_CLASSIFER_PATH = 'data/stanford-ner-2018-10-16/classifiers/english.muc.7class.distsim.crf.ser.gz'
STANFORD_NER_JAR_PATH       = 'data/stanford-ner-2018-10-16/stanford-ner-3.9.2.jar'

from nltk.tag import StanfordNERTagger
from nltk.tokenize import word_tokenize
ner_tagger = StanfordNERTagger(STANFORD_NER_CLASSIFER_PATH, STANFORD_NER_JAR_PATH)
tokens     = word_tokenize(text)
print(ner_tagger.tag(tokens))

# 장소에 해당하는 단어만 출력합니다.
all_locations = [token[0]  for token in ner_tagger.tag(tokens) 
                 if token[1] == 'LOCATION']
print(', '.join(all_locations))