ch7 추천엔진 과 모델의 평가¶

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

ratings = pd.read_csv("data/ml-latest-small/ratings.csv")
print (ratings.head(2))
movies = pd.read_csv("data/ml-latest-small/movies.csv")
print (movies.head(2))

# 영화제목과 평점정보를 1개의 DataFrame으로 묶음
ratings = pd.merge(ratings[['userId', 'movieId', 'rating']], 
                   movies[['movieId', 'title']],
                   how='left', left_on='movieId', right_on='movieId')
ratings.head(3)

# 영화명 필드, 사용자 인덱스 Pivot Table을 생성합니다
rp = ratings.pivot_table(columns = ['movieId'], 
                         index   = ['userId'], values = 'rating')
rp = rp.fillna(0)
rp.head(3)

# 연산이 용이하도록 numpy matrix로 변환합니다
rp_mat = rp.values # as_matrix()
rp_mat

# The cosine of the angle between them is about 0.822.
from scipy.spatial.distance import cosine
a = np.asarray([2, 1, 0, 2, 0, 1, 1, 1])
b = np.asarray([2, 1, 1, 1, 1, 0, 1, 1])
print ("A 과 B 행렬의 Cosin 유사도 : {:.3f}".format(1-cosine(a, b)))

%%time
# User similarity matrix
m, n      = rp.shape
mat_users = np.zeros((m, m))
for i in range(m):
    for j in range(m):
        if i != j: 
            mat_users[i][j] = (1-cosine(rp_mat[i,:], rp_mat[j,:]))
        else: 
            mat_users[i][j] = 0.
            
pd_users = pd.DataFrame(mat_users, index=rp.index, columns=rp.index )
print(pd_users.head(2))

# 사용자 기반 유사도 측정
def topn_simusers(uid=16, n=5):
    users      = pd_users.loc[uid, :].sort_values(ascending=False)
    topn_users = users.iloc[:n, ]
    topn_users = topn_users.rename('score')    
    print ("Similar users as user:", uid)
    return pd.DataFrame(topn_users)

# 17번 사용자와 유사한 10명의 ID 정보를 출력
print(topn_simusers(uid=17, n=10))

# 사용자가 선호하는 영화목록을 출력하는 함수
def topn_movieratings(uid=355, n_ratings=10):    
    uid_ratings = ratings.loc[ratings['userId'] == uid]
    uid_ratings = uid_ratings.sort_values(by='rating', ascending=[False])
    print ("Top {} movie ratings of user: {}".format(n_ratings, uid))
    return uid_ratings.iloc[:n_ratings, ]    

# 596번 사용자가 선호하는 영화목록 10개
print(topn_movieratings(uid=596, n_ratings=10))

# cosine_sim 코싸인 유사도 행렬
# TF-IDF Vectorizer간 Dot Product 계산시 Cosine Similarity Score 제공
from sklearn.metrics.pairwise import linear_kernel
mat_movies = linear_kernel(rp_mat, rp_mat)
mat_movies

from scipy.spatial.distance import cdist
mat_movies = cdist(rp_mat, rp_mat)
mat_movies.shape

from sklearn.metrics import pairwise_distances
mat_movies = pairwise_distances(rp_mat, metric='manhattan')
mat_movies.shape

pd_movies = pd.read_csv("data/pd_movies.csv",index_col='movieId')

# Finding similar movies
def topn_simovies(mid = 588,n=15):
    mid_ratings = pd_movies.loc[mid,:].sort_values(ascending = False)
    topn_movies = pd.DataFrame(mid_ratings.iloc[:n,])
    topn_movies['index1'] = topn_movies.index
    topn_movies['index1'] = topn_movies['index1'].astype('int64')
    topn_movies = pd.merge(topn_movies, movies[['movieId','title']],
                           how='left', left_on='index1', right_on='movieId')
    print ("Movies similar to movie id: {}, {}, are".format(
        mid, 
        movies['title'][movies['movieId']==mid].to_string(index=False)))
    del topn_movies['index1']
    return topn_movies

# 589번 사용자가 유사한 영화목록 15개 출력
print (topn_simovies(mid=589, n=15))

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# ratings = pd.read_csv("data/ml-latest-small/ratings.csv")
# movies  = pd.read_csv("data/ml-latest-small/movies.csv")
# rp = ratings.pivot_table(columns=['movieId'], index=['userId'], values='rating')
# rp = rp.fillna(0)
A = rp.values
print ("\nShape of Original Sparse Matrix",  A.shape)
A

# 0.5~5 평점이 존재하면1, 없으면 0의 희소행렬을 생성
W = A > 0.5
W [W == True]  = 1
W [W == False] = 0
W = W.astype(np.float64,copy=False)
W

# W 와 반대조건의 Table을 생성
# 예측된 평가행렬을 W_pred 와 곱하면 0 이 되도록 만들기 위함
# 연산 후 내림차순으로 정렬을 쉽게 도와주는 행렬
W_pred = A < 0.5
W_pred[W_pred==True]  = 1
W_pred[W_pred==False] = 0
W_pred = W_pred.astype(np.float64, copy=False)
np.fill_diagonal(W_pred, val=0)
W_pred

# Parameters
m, n         = A.shape
n_iterations = 70   # 학습을 위한 반복횟수
n_factors    = 100   # 잠재요인
lmbda        = 0.1   # 학습률

X = 5 * np.random.rand(m, n_factors)
Y = 5 * np.random.rand(n_factors, n)

# RMSE 오차계산 함수를 정의합니다
def get_error(A, X, Y, W):
    return np.sqrt(np.sum((W * (A - np.dot(X, Y)))**2)/np.sum(W))

errors = []
for itr in range(n_iterations):
    X = np.linalg.solve(np.dot(Y,Y.T) + lmbda*np.eye(n_factors), np.dot(Y,A.T)).T
    Y = np.linalg.solve(np.dot(X.T,X) + lmbda*np.eye(n_factors), np.dot(X.T,A))    
    if itr % 10 == 0:
        print("{:3} 반복완료 RMSE is: {:.4f}".format(itr,get_error(A,X,Y,W)))
    errors.append(get_error(A, X, Y, W))

# 최종 예측행렬을 생성 합니다
A_hat = np.dot(X, Y)
print ("영화폄점 최종 모델의 RMSE: ",get_error(A,X,Y,W))    

%matplotlib inline
from matplotlib import rc
from matplotlib import pyplot as plt
rc('font', family=['NanumGothic','Malgun Gothic'])

plt.plot(errors)
plt.ylim([0, 3.5])
plt.xlabel("반복 학습횟수 (Number of Iterations)")
plt.ylabel("RMSE 값")
plt.title("No.of Iterations vs. RMSE")
plt.show()

# 위에서 반복학습한 행렬을 곱하여 최종 예측행렬을 출력합니다
def print_recommovies(uid=315, n_movies=15, pred_mat=A_hat, wpred_mat=W_pred):
    pred_recos   = pred_mat * wpred_mat
    pd_predrecos = pd.DataFrame(pred_recos, index=rp.index ,columns=rp.columns)
    pred_ratings = pd_predrecos.loc[uid,:].sort_values(ascending = False)
    pred_topratings = pred_ratings[:n_movies,]
    pred_topratings = pred_topratings.rename('pred_ratings')  
    pred_topratings = pd.DataFrame(pred_topratings)
    pred_topratings['index1'] = pred_topratings.index
    pred_topratings['index1'] = pred_topratings['index1'].astype('int64')
    pred_topratings = pd.merge(pred_topratings,movies[['movieId','title']],how = 'left',left_on ='index1' ,right_on = 'movieId')
    del pred_topratings['index1']    
    print ("\nTop",n_movies,"movies predicted for the user:",uid," based on collaborative filtering\n")
    return pred_topratings

# 355번 사용자가 선호하는 영화목록 10편을 출력합니다
# 연산에 활용할 예측행렬, 희소행렬을 입력합니다
print_recommovies(uid=355, n_movies=10, pred_mat=A_hat, wpred_mat=W_pred)

# 11번 사용자가 선호하는 영화목록 5편을 출력합니다
print_recommovies(uid=11, n_movies=5, pred_mat=A_hat, wpred_mat=W_pred)

%%time
# Grid Search on Collaborative Filtering
def get_error(A, X, Y, W):
    return np.sqrt(np.sum((W *(A-np.dot(X, Y)))**2)/np.sum(W))

init_error = float("inf")
niters     = [20, 50, 100, 200]
factors    = [30, 50, 70,  100]
lambdas    = [0.001, 0.01, 0.05, 0.1]
print("ALS 행렬 파라미터 Grid Search:\n")

for niter in niters:
    for facts in factors:
        for lmbd in lambdas:                
            X = 5 * np.random.rand(m, facts)
            Y = 5 * np.random.rand(facts, n)
            for itr in range(niter):
                X = np.linalg.solve(np.dot(Y, Y.T)+lmbd*np.eye(facts), np.dot(Y, A.T)).T
                Y = np.linalg.solve(np.dot(X.T, X)+lmbd*np.eye(facts), np.dot(X.T, A))
            error = get_error(A, X, Y, W)
            if error < init_error:
                print("iters: {:3} Factor 수: {:3} lambda: {} RMSE 값 {:.4f}".format(
                    niter, facts, lmbd ,error))
                init_error = error

# PipeLine 생성 예제
from sklearn.svm import SVC
from sklearn.pipeline import Pipeline
from sklearn.datasets import load_digits
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV, validation_curve

digits      = load_digits()
X, y        = digits.data, digits.target
pipe_svc    = Pipeline([('scl', StandardScaler()), ('clf', SVC(random_state=1))])
param_range = [0.0001, 0.001, 0.01] #, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid  = [
    {'clf__C': param_range, 'clf__kernel': ['linear']},
    {'clf__C': param_range, 'clf__gamma': param_range, 'clf__kernel': ['rbf']}]
gs          = GridSearchCV(estimator=pipe_svc, param_grid=param_grid,
                           scoring='accuracy', cv=10, n_jobs=1)
%time gs = gs.fit(X, y)

# make PipeLine 생성 예제2
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline

cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, random_state=0)

pipe       = make_pipeline(StandardScaler(), LogisticRegression(solver='liblinear'))
param_grid = {'logisticregression__C': [0.01, 0.1, 1, 10, 100]}
X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, random_state=4)
grid = GridSearchCV(pipe, param_grid, cv=5)
grid.fit(X_train, y_train)