SVM을 이용한 뉴스 Topic 분류기¶

# 앞에서 진행한 내용 복습
# 정상매일은 0, 스펨매일은 1
import glob,os

emails, labels = [], []
for no, file_path in enumerate(['./data/enron1/ham/','./data/enron1/spam/']):
    for filename in glob.glob(os.path.join(file_path, '*.txt')):
        with open(filename, 'r', encoding = "ISO-8859-1") as infile:
            emails.append(infile.read())
            labels.append(no)

from nltk.corpus import names
from nltk.stem import WordNetLemmatizer

def letters_only(astr):
    for c in astr:
        if not c.isalpha():
            return False
    return True

def clean_text(docs):
    cleaned_docs = []
    for doc in docs:
        cleaned_docs.append(' '.join([lemmatizer.lemmatize(word.lower())
                                      for word in doc.split()
                      if letters_only(word) and word not in all_names]))
    return cleaned_docs

all_names      = set(names.words())
lemmatizer     = WordNetLemmatizer()
cleaned_emails = clean_text(emails)

from sklearn.model_selection import StratifiedKFold
import numpy as np

k                 = 10
k_fold            = StratifiedKFold(n_splits=k)
cleaned_emails_np = np.array(cleaned_emails)
labels_np         = np.array(labels)

smoothing_factor_option = [1.0, 2.0, 3.0, 4.0, 5.0]

from collections import defaultdict
auc_record = defaultdict(float)

%timeit
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import roc_auc_score

for train_indices, test_indices in k_fold.split(cleaned_emails, labels):
    X_train, X_test  = cleaned_emails_np[train_indices], cleaned_emails_np[test_indices]
    Y_train, Y_test  = labels_np[train_indices], labels_np[test_indices]
    tfidf_vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, stop_words='english', max_features=8000)
    term_docs_train  = tfidf_vectorizer.fit_transform(X_train)
    term_docs_test   = tfidf_vectorizer.transform(X_test)
    for smoothing_factor in smoothing_factor_option:
        clf = MultinomialNB(alpha=smoothing_factor, fit_prior=True)
        clf.fit(term_docs_train, Y_train)
        prediction_prob = clf.predict_proba(term_docs_test)
        pos_prob        = prediction_prob[:, 1]
        auc             = roc_auc_score(Y_test, pos_prob)
        auc_record[smoothing_factor] += auc

# 구분할 2개의 카테고리 데이터를 불러온다
from sklearn.datasets import fetch_20newsgroups

categories = ['comp.graphics', 'sci.space']
data_train = fetch_20newsgroups(subset='train', categories=categories, random_state=42)
data_test  = fetch_20newsgroups(subset='test',  categories=categories, random_state=42)

# 불러온 데이터들을 전처리 작업한다
cleaned_train = clean_text(data_train.data)
label_train   = data_train.target
cleaned_test  = clean_text(data_test.data)
label_test    = data_test.target

# 데이터를 Tf-IDF 로 임베딩 변환한다
tfidf_vectorizer = TfidfVectorizer(sublinear_tf = True, 
                                   max_df = 0.5, 
                                   stop_words = 'english', 
                                   max_features = 8000)
term_docs_train  = tfidf_vectorizer.fit_transform(cleaned_train)
term_docs_test   = tfidf_vectorizer.transform(cleaned_test)

# 다양한 클래스의 뉴스데이터 불러오기
categories = [
    'alt.atheism',
    'talk.religion.misc',
    'comp.graphics',
    'sci.space',
    'rec.sport.hockey']

data_train = fetch_20newsgroups(subset='train', categories=categories, random_state=42)
data_test  = fetch_20newsgroups(subset='test',  categories=categories, random_state=42)

# 학습할 데이터를 불러온다
categories = None
data_train = fetch_20newsgroups(subset='train', categories=categories, random_state=42)
data_test  = fetch_20newsgroups(subset='test', categories=categories, random_state=42)

# 데이터를 전처리 작업을 진행한다
cleaned_train = clean_text(data_train.data)
label_train   = data_train.target
cleaned_test  = clean_text(data_test.data)
label_test    = data_test.target

# Tf-IDF로 텍스트를 임베딩 작업 진행한다
tfidf_vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5, stop_words='english', max_features=8000)
term_docs_train  = tfidf_vectorizer.fit_transform(cleaned_train)
term_docs_test   = tfidf_vectorizer.transform(cleaned_test)

# 텍스트 데이터를 분류하는 경우 선형함수를 사용한다
arameters  = {'C': [0.1, 1, 10, 100]}
svc_libsvm = SVC(kernel='linear')

# 전체 20개의 파라미터를 순차적 검증하기 위하여 for 반복문 대신에
from sklearn.model_selection import GridSearchCV
grid_search = GridSearchCV(svc_libsvm, parameters, n_jobs=-1, cv=3)

# Tf-IDF 추출기와 SVM 분류기를 1개의 PipeLine로 묶는다
from sklearn.pipeline import Pipeline
pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english')),
    ('svc', LinearSVC()), ])

# Pipe Line 단계의 이름과 파라미터를 묶어서 키로 설정한다
parameters_pipeline = {
    'tfidf__max_df': (0.25, 0.5),
    'tfidf__max_features': (40000, 50000),
    'tfidf__sublinear_tf': (True, False),
    'tfidf__smooth_idf': (True, False),
    'svc__C': (0.1, 1, 10, 100),}

# Test를 위해 20% 데이터를 분리한다
from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(
    X, Y, test_size=.2, random_state=42)