from preamble import *
%matplotlib inline
from sklearn.datasets import make_blobs
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
# create a synthetic dataset
X, y = make_blobs(random_state=0)
print("X.shape:", X.shape)
print("y.shape:", y.shape)
# split data and labels into a training and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
# instantiate a model and fit it to the training set
logreg = LogisticRegression().fit(X_train, y_train)
# evaluate the model on the test set
print("Test set score: {:.2f}".format(logreg.score(X_test, y_test)))
X.shape: (100, 2) y.shape: (100,) Test set score: 0.88
import matplotlib
print(matplotlib.__version__)
mglearn.plots.plot_cross_validation()
2.1.2
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
iris = load_iris()
print("iris.data.shape:", iris.data.shape)
print("iris.target.shape:", iris.target.shape)
logreg = LogisticRegression()
scores = cross_val_score(logreg, iris.data, iris.target)
print("Cross-validation scores: {}".format(scores))
iris.data.shape: (150, 4) iris.target.shape: (150,) Cross-validation scores: [0.961 0.922 0.958]
scores = cross_val_score(logreg, iris.data, iris.target, cv=5)
print("Cross-validation scores: {}".format(scores))
Cross-validation scores: [1. 0.967 0.933 0.9 1. ]
print("Average cross-validation score: {:.2f}".format(scores.mean()))
Average cross-validation score: 0.96
from sklearn.datasets import load_iris
iris = load_iris()
print("Iris labels:\n{}".format(iris.target))
Iris labels: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2]
mglearn.plots.plot_stratified_cross_validation()
from sklearn.model_selection import KFold
kfold = KFold(n_splits=5) #교차 검증 분할기의 역할 수행
print("Cross-validation scores:\n{}".format(cross_val_score(logreg, iris.data, iris.target, cv=kfold)))
Cross-validation scores: [1. 0.933 0.433 0.967 0.433]
kfold = KFold(n_splits=3)
print("Cross-validation scores:\n{}".format(cross_val_score(logreg, iris.data, iris.target, cv=kfold)))
Cross-validation scores: [0. 0. 0.]
kfold = KFold(n_splits=3, shuffle=True, random_state=0)
print("Cross-validation scores:\n{}".format(cross_val_score(logreg, iris.data, iris.target, cv=kfold)))
Cross-validation scores: [0.9 0.96 0.96]
from sklearn.model_selection import LeaveOneOut
loo = LeaveOneOut()
scores = cross_val_score(logreg, iris.data, iris.target, cv=loo)
print("Number of cv iterations: ", len(scores))
print("Mean accuracy: {:.2f}".format(scores.mean()))
Number of cv iterations: 150 Mean accuracy: 0.95
임의 분할 교차 검증
아래 그림 예제
mglearn.plots.plot_shuffle_split()
from sklearn.model_selection import ShuffleSplit
shuffle_split = ShuffleSplit(n_splits=10, test_size=.5, train_size=.5)
scores = cross_val_score(logreg, iris.data, iris.target, cv=shuffle_split)
print("Cross-validation scores:\n{}".format(scores))
print("Mean accuracy: {:.2f}".format(scores.mean()))
Cross-validation scores: [1. 0.973 0.96 0.933 0.867 0.88 0.893 0.96 0.933 0.96 ] Mean accuracy: 0.94
from sklearn.model_selection import StratifiedShuffleSplit
shuffle_split = StratifiedShuffleSplit(n_splits=10, test_size=.5, train_size=.5)
scores = cross_val_score(logreg, iris.data, iris.target, cv=shuffle_split)
print("Cross-validation scores:\n{}".format(scores))
print("Mean accuracy: {:.2f}".format(scores.mean()))
Cross-validation scores: [0.973 0.947 0.933 0.947 0.96 0.96 0.92 0.96 0.933 1. ] Mean accuracy: 0.95
mglearn.plots.plot_group_kfold()
from sklearn.model_selection import GroupKFold
# create synthetic dataset
X, y = make_blobs(n_samples=12, random_state=0)
# assume the first three samples belong to the same group,
# then the next four, etc
groups = [0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3]
scores = cross_val_score(logreg, X, y, groups, cv=GroupKFold(n_splits=3))
print("Cross-validation scores:\n{}".format(scores))
print("Mean accuracy: {:.2f}".format(scores.mean()))
Cross-validation scores: [0.75 0.8 0.667] Mean accuracy: 0.74
C=0.001 | C=0.01 | C=0.1 | C=1 | C=10 | C=100 | |
gamma=0.001 | SVC(C=0.001, gamma=0.001) | SVC(C=0.01, gamma=0.001) | SVC(C=0.1, gamma=0.001) | SVC(C=1, gamma=0.001) | SVC(C=10, gamma=0.001) | SVC(C=100, gamma=0.001) |
gamma=0.01 | SVC(C=0.001, gamma=0.01) | SVC(C=0.01, gamma=0.01) | SVC(C=0.1, gamma=0.01) | SVC(C=1, gamma=0.01) | SVC(C=10, gamma=0.01) | SVC(C=100, gamma=0.01) |
gamma=0.1 | SVC(C=0.001, gamma=0.1) | SVC(C=0.01, gamma=0.1) | SVC(C=0.1, gamma=0.1) | SVC(C=1, gamma=0.1) | SVC(C=10, gamma=0.1) | SVC(C=100, gamma=0.1) |
gamma=1 | SVC(C=0.001, gamma=1) | SVC(C=0.01, gamma=1) | SVC(C=0.1, gamma=1) | SVC(C=1, gamma=1) | SVC(C=10, gamma=1) | SVC(C=100, gamma=1) |
gamma=10 | SVC(C=0.001, gamma=10) | SVC(C=0.01, gamma=10) | SVC(C=0.1, gamma=10) | SVC(C=1, gamma=10) | SVC(C=10, gamma=10) | SVC(C=100, gamma=10) |
gamma=100 | SVC(C=0.001, gamma=100) | SVC(C=0.01, gamma=100) | SVC(C=0.1, gamma=100) | SVC(C=1, gamma=100) | SVC(C=10, gamma=100) | SVC(C=100, gamma=100) |
# naive grid search implementation
from sklearn.svm import SVC
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=0)
print("Size of training set: {} size of test set: {}".format(X_train.shape[0], X_test.shape[0]))
best_score = 0
for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
# for each combination of parameters, train an SVC
svm = SVC(gamma=gamma, C=C)
svm.fit(X_train, y_train)
# evaluate the SVC on the test set
score = svm.score(X_test, y_test)
# if we got a better score, store the score and parameters
if score > best_score:
best_score = score
best_parameters = {'C': C, 'gamma': gamma}
print("Best score: {:.2f}".format(best_score))
print("Best parameters: {}".format(best_parameters))
Size of training set: 112 size of test set: 38 Best score: 0.97 Best parameters: {'C': 100, 'gamma': 0.001}
mglearn.plots.plot_threefold_split()
from sklearn.svm import SVC
# split data into train+validation set and test set
X_trainval, X_test, y_trainval, y_test = train_test_split(iris.data, iris.target, random_state=0)
# split train+validation set into training and validation sets
X_train, X_valid, y_train, y_valid = train_test_split(X_trainval, y_trainval, random_state=1)
print("Size of training set: {}, size of validation set: {},size of test set: {}\n".format(
X_train.shape[0],
X_valid.shape[0],
X_test.shape[0]))
best_score = 0
for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
# for each combination of parameters train an SVC
svm = SVC(gamma=gamma, C=C)
svm.fit(X_train, y_train)
# evaluate the SVC on the validation set
score = svm.score(X_valid, y_valid)
# if we got a better score, store the score and parameters
if score > best_score:
best_score = score
best_parameters = {'C': C, 'gamma': gamma}
# rebuild a model on the combined training and validation set,
# and evaluate it on the test set
svm = SVC(**best_parameters)
#[NOTE] 훈련 데이터와 검증 데이터를 합쳐서 다시 모델을 구성함
svm.fit(X_trainval, y_trainval)
test_score = svm.score(X_test, y_test)
print("Best score on validation set: {:.2f}".format(best_score))
print("Best parameters: ", best_parameters)
print("Test set score with best parameters: {:.2f}".format(test_score))
Size of training set: 84, size of validation set: 28,size of test set: 38 Best score on validation set: 0.96 Best parameters: {'C': 10, 'gamma': 0.001} Test set score with best parameters: 0.92
X_trainval, X_test, y_trainval, y_test = train_test_split(iris.data, iris.target, random_state=0)
X_train, X_valid, y_train, y_valid = train_test_split(X_trainval, y_trainval, random_state=1)
print("Size of training set: {}, size of validation set: {},size of test set: {}\n".format(
X_train.shape[0],
X_valid.shape[0],
X_test.shape[0]))
# reference: manual_grid_search_cv
for gamma in [0.001, 0.01, 0.1, 1, 10, 100]:
for C in [0.001, 0.01, 0.1, 1, 10, 100]:
# for each combination of parameters,
# train an SVC
svm = SVC(gamma=gamma, C=C)
# perform cross-validation
scores = cross_val_score(svm, X_trainval, y_trainval, cv=5)
# compute mean cross-validation accuracy
score = np.mean(scores)
# if we got a better score, store the score and parameters
if score > best_score:
best_score = score
best_parameters = {'C': C, 'gamma': gamma}
# rebuild a model on the combined training and validation set
svm = SVC(**best_parameters)
#[NOTE] 훈련 데이터와 검증 데이터를 합쳐서 다시 모델을 구성함
svm.fit(X_trainval, y_trainval)
test_score = svm.score(X_test, y_test)
print("Best score on validation set: {:.2f}".format(best_score))
print("Best parameters: ", best_parameters)
print("Test set score with best parameters: {:.2f}".format(test_score))
Size of training set: 84, size of validation set: 28,size of test set: 38 Best score on validation set: 0.97 Best parameters: {'C': 100, 'gamma': 0.01} Test set score with best parameters: 0.97
mglearn.plots.plot_cross_val_selection()
/Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs) /Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True warnings.warn(*warn_args, **warn_kwargs)
param_grid = {
'C': [0.001, 0.01, 0.1, 1, 10, 100],
'gamma': [0.001, 0.01, 0.1, 1, 10, 100]
}
print("Parameter grid:\n{}".format(param_grid))
Parameter grid: {'C': [0.001, 0.01, 0.1, 1, 10, 100], 'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
estimator = SVC()
grid_search = GridSearchCV(
estimator = estimator,
param_grid = param_grid,
n_jobs = -1,
cv = 5,
return_train_score = True
)
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=0)
grid_search.fit(X_train, y_train)
GridSearchCV(cv=5, error_score='raise', estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf', max_iter=-1, probability=False, random_state=None, shrinking=True, tol=0.001, verbose=False), fit_params=None, iid=True, n_jobs=-1, param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100], 'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}, pre_dispatch='2*n_jobs', refit=True, return_train_score=True, scoring=None, verbose=0)
print("Test set score: {:.2f}".format(grid_search.score(X_test, y_test)))
Test set score: 0.97
print("Best parameters: {}".format(grid_search.best_params_))
print("Best cross-validation score: {:.2f}".format(grid_search.best_score_))
Best parameters: {'C': 100, 'gamma': 0.01} Best cross-validation score: 0.97
print("Best estimator:\n{}".format(grid_search.best_estimator_))
Best estimator: SVC(C=100, cache_size=200, class_weight=None, coef0=0.0, decision_function_shape='ovr', degree=3, gamma=0.01, kernel='rbf', max_iter=-1, probability=False, random_state=None, shrinking=True, tol=0.001, verbose=False)
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=0)
estimator = SVC()
param_grid = {
'C': [0.001, 0.01, 0.1, 1, 10, 100],
'gamma': [0.001, 0.01, 0.1, 1, 10, 100]
}
grid_search = GridSearchCV(
estimator = estimator,
param_grid = param_grid,
n_jobs = -1,
cv = 5,
return_train_score = True
)
grid_search.fit(X_train, y_train)
print("Test set score: {:.2f}".format(grid_search.score(X_test, y_test)))
Test set score: 0.97
import pandas as pd
# convert to Dataframe
results = pd.DataFrame(grid_search.cv_results_)
pd.options.display.float_format = '{:,.7f}'.format
print(results.columns)
# show the first 5 rows
# display(results.head(5))
display(results)
Index(['mean_fit_time', 'mean_score_time', 'mean_test_score', 'mean_train_score', 'param_C', 'param_gamma', 'params', 'rank_test_score', 'split0_test_score', 'split0_train_score', 'split1_test_score', 'split1_train_score', 'split2_test_score', 'split2_train_score', 'split3_test_score', 'split3_train_score', 'split4_test_score', 'split4_train_score', 'std_fit_time', 'std_score_time', 'std_test_score', 'std_train_score'], dtype='object')
mean_fit_time | mean_score_time | mean_test_score | mean_train_score | ... | std_fit_time | std_score_time | std_test_score | std_train_score | |
---|---|---|---|---|---|---|---|---|---|
0 | 0.0035735 | 0.0012889 | 0.3660714 | 0.3660787 | ... | 0.0010096 | 0.0004829 | 0.0113708 | 0.0028518 |
1 | 0.0013528 | 0.0004713 | 0.3660714 | 0.3660787 | ... | 0.0003804 | 0.0000942 | 0.0113708 | 0.0028518 |
2 | 0.0009893 | 0.0004484 | 0.3660714 | 0.3660787 | ... | 0.0002081 | 0.0001443 | 0.0113708 | 0.0028518 |
3 | 0.0014754 | 0.0005429 | 0.3660714 | 0.3660787 | ... | 0.0004951 | 0.0000276 | 0.0113708 | 0.0028518 |
4 | 0.0013918 | 0.0010472 | 0.3660714 | 0.3660787 | ... | 0.0004632 | 0.0009545 | 0.0113708 | 0.0028518 |
5 | 0.0045546 | 0.0012807 | 0.3660714 | 0.3660787 | ... | 0.0066814 | 0.0016633 | 0.0113708 | 0.0028518 |
6 | 0.0016893 | 0.0007505 | 0.3660714 | 0.3660787 | ... | 0.0013267 | 0.0006236 | 0.0113708 | 0.0028518 |
7 | 0.0013759 | 0.0005324 | 0.3660714 | 0.3660787 | ... | 0.0006139 | 0.0000450 | 0.0113708 | 0.0028518 |
8 | 0.0012942 | 0.0006999 | 0.3660714 | 0.3660787 | ... | 0.0001966 | 0.0002061 | 0.0113708 | 0.0028518 |
9 | 0.0011333 | 0.0012735 | 0.3660714 | 0.3660787 | ... | 0.0001514 | 0.0014780 | 0.0113708 | 0.0028518 |
10 | 0.0016042 | 0.0006387 | 0.3660714 | 0.3660787 | ... | 0.0005756 | 0.0001327 | 0.0113708 | 0.0028518 |
11 | 0.0014416 | 0.0022371 | 0.3660714 | 0.3660787 | ... | 0.0004339 | 0.0025022 | 0.0113708 | 0.0028518 |
12 | 0.0034446 | 0.0004577 | 0.3660714 | 0.3660787 | ... | 0.0047159 | 0.0000184 | 0.0113708 | 0.0028518 |
13 | 0.0015449 | 0.0004823 | 0.6964286 | 0.6964237 | ... | 0.0005018 | 0.0000942 | 0.0131963 | 0.0032580 |
14 | 0.0010179 | 0.0005842 | 0.9196429 | 0.9197442 | ... | 0.0001187 | 0.0000258 | 0.0440102 | 0.0212659 |
15 | 0.0011138 | 0.0004836 | 0.9553571 | 0.9598457 | ... | 0.0001235 | 0.0000593 | 0.0401043 | 0.0113043 |
16 | 0.0014143 | 0.0005417 | 0.3660714 | 0.3817097 | ... | 0.0005119 | 0.0001906 | 0.0113708 | 0.0213374 |
17 | 0.0020837 | 0.0004833 | 0.3660714 | 0.3660787 | ... | 0.0016379 | 0.0000902 | 0.0113708 | 0.0028518 |
18 | 0.0018477 | 0.0004734 | 0.6964286 | 0.6964237 | ... | 0.0016144 | 0.0000679 | 0.0131963 | 0.0032580 |
19 | 0.0009147 | 0.0004317 | 0.9285714 | 0.9353247 | ... | 0.0001314 | 0.0000665 | 0.0429827 | 0.0078884 |
20 | 0.0008145 | 0.0005105 | 0.9642857 | 0.9776501 | ... | 0.0001012 | 0.0001073 | 0.0340769 | 0.0100842 |
21 | 0.0009450 | 0.0004607 | 0.9464286 | 0.9843928 | ... | 0.0001512 | 0.0000481 | 0.0324799 | 0.0088664 |
22 | 0.0016514 | 0.0005848 | 0.9196429 | 1.0000000 | ... | 0.0002375 | 0.0001800 | 0.0647906 | 0.0000000 |
23 | 0.0013955 | 0.0004817 | 0.5089286 | 1.0000000 | ... | 0.0000500 | 0.0000798 | 0.0464350 | 0.0000000 |
24 | 0.0008643 | 0.0004160 | 0.9285714 | 0.9353247 | ... | 0.0000664 | 0.0000253 | 0.0429827 | 0.0078884 |
25 | 0.0006838 | 0.0004532 | 0.9642857 | 0.9776757 | ... | 0.0000423 | 0.0001550 | 0.0340769 | 0.0070319 |
26 | 0.0008232 | 0.0004055 | 0.9642857 | 0.9865662 | ... | 0.0002632 | 0.0000344 | 0.0177687 | 0.0083555 |
27 | 0.0007598 | 0.0004054 | 0.9375000 | 0.9865906 | ... | 0.0001178 | 0.0000655 | 0.0452528 | 0.0083624 |
28 | 0.0010057 | 0.0003399 | 0.9196429 | 1.0000000 | ... | 0.0001139 | 0.0000584 | 0.0647906 | 0.0000000 |
29 | 0.0010963 | 0.0004542 | 0.5625000 | 1.0000000 | ... | 0.0001579 | 0.0001312 | 0.0496678 | 0.0000000 |
30 | 0.0005068 | 0.0002547 | 0.9642857 | 0.9776757 | ... | 0.0000904 | 0.0000352 | 0.0340769 | 0.0070319 |
31 | 0.0005447 | 0.0002956 | 0.9732143 | 0.9843684 | ... | 0.0002228 | 0.0000414 | 0.0223995 | 0.0054851 |
32 | 0.0005828 | 0.0005119 | 0.9553571 | 0.9887884 | ... | 0.0001083 | 0.0003010 | 0.0495662 | 0.0099945 |
33 | 0.0005792 | 0.0002865 | 0.9464286 | 1.0000000 | ... | 0.0000584 | 0.0000391 | 0.0519227 | 0.0000000 |
34 | 0.0010027 | 0.0003521 | 0.9196429 | 1.0000000 | ... | 0.0001354 | 0.0000996 | 0.0647906 | 0.0000000 |
35 | 0.0010271 | 0.0003221 | 0.5625000 | 1.0000000 | ... | 0.0001011 | 0.0000409 | 0.0496678 | 0.0000000 |
36 rows × 22 columns
results2 = results[['rank_test_score', 'params', 'mean_test_score', 'std_test_score',
'mean_train_score', 'std_train_score']]
results2 = results2.sort_values('rank_test_score')
display(results2)
rank_test_score | params | mean_test_score | std_test_score | mean_train_score | std_train_score | |
---|---|---|---|---|---|---|
31 | 1 | {'C': 100, 'gamma': 0.01} | 0.9732143 | 0.0223995 | 0.9843684 | 0.0054851 |
20 | 2 | {'C': 1, 'gamma': 0.1} | 0.9642857 | 0.0340769 | 0.9776501 | 0.0100842 |
30 | 2 | {'C': 100, 'gamma': 0.001} | 0.9642857 | 0.0340769 | 0.9776757 | 0.0070319 |
26 | 2 | {'C': 10, 'gamma': 0.1} | 0.9642857 | 0.0177687 | 0.9865662 | 0.0083555 |
25 | 2 | {'C': 10, 'gamma': 0.01} | 0.9642857 | 0.0340769 | 0.9776757 | 0.0070319 |
32 | 6 | {'C': 100, 'gamma': 0.1} | 0.9553571 | 0.0495662 | 0.9887884 | 0.0099945 |
15 | 6 | {'C': 0.1, 'gamma': 1} | 0.9553571 | 0.0401043 | 0.9598457 | 0.0113043 |
33 | 8 | {'C': 100, 'gamma': 1} | 0.9464286 | 0.0519227 | 1.0000000 | 0.0000000 |
21 | 8 | {'C': 1, 'gamma': 1} | 0.9464286 | 0.0324799 | 0.9843928 | 0.0088664 |
27 | 10 | {'C': 10, 'gamma': 1} | 0.9375000 | 0.0452528 | 0.9865906 | 0.0083624 |
24 | 11 | {'C': 10, 'gamma': 0.001} | 0.9285714 | 0.0429827 | 0.9353247 | 0.0078884 |
19 | 11 | {'C': 1, 'gamma': 0.01} | 0.9285714 | 0.0429827 | 0.9353247 | 0.0078884 |
22 | 13 | {'C': 1, 'gamma': 10} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
14 | 13 | {'C': 0.1, 'gamma': 0.1} | 0.9196429 | 0.0440102 | 0.9197442 | 0.0212659 |
28 | 13 | {'C': 10, 'gamma': 10} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
34 | 13 | {'C': 100, 'gamma': 10} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
13 | 17 | {'C': 0.1, 'gamma': 0.01} | 0.6964286 | 0.0131963 | 0.6964237 | 0.0032580 |
18 | 17 | {'C': 1, 'gamma': 0.001} | 0.6964286 | 0.0131963 | 0.6964237 | 0.0032580 |
29 | 19 | {'C': 10, 'gamma': 100} | 0.5625000 | 0.0496678 | 1.0000000 | 0.0000000 |
35 | 19 | {'C': 100, 'gamma': 100} | 0.5625000 | 0.0496678 | 1.0000000 | 0.0000000 |
23 | 21 | {'C': 1, 'gamma': 100} | 0.5089286 | 0.0464350 | 1.0000000 | 0.0000000 |
0 | 22 | {'C': 0.001, 'gamma': 0.001} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
12 | 22 | {'C': 0.1, 'gamma': 0.001} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
11 | 22 | {'C': 0.01, 'gamma': 100} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
10 | 22 | {'C': 0.01, 'gamma': 10} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
9 | 22 | {'C': 0.01, 'gamma': 1} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
8 | 22 | {'C': 0.01, 'gamma': 0.1} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
7 | 22 | {'C': 0.01, 'gamma': 0.01} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
6 | 22 | {'C': 0.01, 'gamma': 0.001} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
5 | 22 | {'C': 0.001, 'gamma': 100} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
4 | 22 | {'C': 0.001, 'gamma': 10} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
3 | 22 | {'C': 0.001, 'gamma': 1} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
2 | 22 | {'C': 0.001, 'gamma': 0.1} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
1 | 22 | {'C': 0.001, 'gamma': 0.01} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
16 | 22 | {'C': 0.1, 'gamma': 10} | 0.3660714 | 0.0113708 | 0.3817097 | 0.0213374 |
17 | 22 | {'C': 0.1, 'gamma': 100} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
print([x for x in results.mean_test_score])
print()
print(results.mean_test_score.shape)
scores = np.array(results.mean_test_score).reshape(6, 6)
# plot the mean cross-validation scores
mglearn.tools.heatmap(
scores,
xlabel='gamma',
xticklabels=param_grid['gamma'],
ylabel='C',
yticklabels=param_grid['C'],
cmap="viridis"
)
[0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.36607142857142855, 0.6964285714285714, 0.9196428571428571, 0.9553571428571429, 0.36607142857142855, 0.36607142857142855, 0.6964285714285714, 0.9285714285714286, 0.9642857142857143, 0.9464285714285714, 0.9196428571428571, 0.5089285714285714, 0.9285714285714286, 0.9642857142857143, 0.9642857142857143, 0.9375, 0.9196428571428571, 0.5625, 0.9642857142857143, 0.9732142857142857, 0.9553571428571429, 0.9464285714285714, 0.9196428571428571, 0.5625] (36,)
<matplotlib.collections.PolyCollection at 0x10f14edd8>
fig, axes = plt.subplots(1, 3, figsize=(20, 5))
param_grid_linear = {'C': np.linspace(1, 2, 6), 'gamma': np.linspace(1, 2, 6)}
param_grid_one_log = {'C': np.linspace(1, 2, 6), 'gamma': np.logspace(-3, 2, 6)}
param_grid_range = {'C': np.logspace(-3, 2, 6), 'gamma': np.logspace(-7, -2, 6)}
for param_grid, ax in zip([param_grid_linear, param_grid_one_log, param_grid_range], axes):
grid_search = GridSearchCV(SVC(), param_grid, n_jobs=-1, cv=5)
grid_search.fit(X_train, y_train)
scores = grid_search.cv_results_['mean_test_score'].reshape(6, 6)
# plot the mean cross-validation scores
scores_image = mglearn.tools.heatmap(
scores, xlabel='gamma', ylabel='C', xticklabels=param_grid['gamma'],
yticklabels=param_grid['C'], cmap="viridis", ax=ax)
plt.colorbar(scores_image, ax=axes.tolist())
<matplotlib.colorbar.Colorbar at 0x10f188630>
첫번째 그래프
두번째 그래프
세번째 그래프
param_grid = [{'kernel': ['rbf'],
'C': [0.001, 0.01, 0.1, 1, 10, 100],
'gamma': [0.001, 0.01, 0.1, 1, 10, 100]},
{'kernel': ['linear'],
'C': [0.001, 0.01, 0.1, 1, 10, 100]}]
print("List of grids:\n{}".format(param_grid))
List of grids: [{'kernel': ['rbf'], 'C': [0.001, 0.01, 0.1, 1, 10, 100], 'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}, {'kernel': ['linear'], 'C': [0.001, 0.01, 0.1, 1, 10, 100]}]
grid_search = GridSearchCV(SVC(), param_grid, n_jobs=-1, cv=5, return_train_score=True)
grid_search.fit(X_train, y_train)
print("Best parameters: {}".format(grid_search.best_params_))
print("Best cross-validation score: {:.2f}".format(grid_search.best_score_))
Best parameters: {'C': 100, 'gamma': 0.01, 'kernel': 'rbf'} Best cross-validation score: 0.97
results = pd.DataFrame(grid_search.cv_results_)
results2 = results[['rank_test_score', 'params', 'mean_test_score', 'std_test_score',
'mean_train_score', 'std_train_score']]
results2 = results2.sort_values('rank_test_score')
display(results2)
rank_test_score | params | mean_test_score | std_test_score | mean_train_score | std_train_score | |
---|---|---|---|---|---|---|
39 | 1 | {'C': 1, 'kernel': 'linear'} | 0.9732143 | 0.0223995 | 0.9843684 | 0.0054851 |
31 | 1 | {'C': 100, 'gamma': 0.01, 'kernel': 'rbf'} | 0.9732143 | 0.0223995 | 0.9843684 | 0.0054851 |
20 | 3 | {'C': 1, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9642857 | 0.0340769 | 0.9776501 | 0.0100842 |
30 | 3 | {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'} | 0.9642857 | 0.0340769 | 0.9776757 | 0.0070319 |
26 | 3 | {'C': 10, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9642857 | 0.0177687 | 0.9865662 | 0.0083555 |
25 | 3 | {'C': 10, 'gamma': 0.01, 'kernel': 'rbf'} | 0.9642857 | 0.0340769 | 0.9776757 | 0.0070319 |
40 | 3 | {'C': 10, 'kernel': 'linear'} | 0.9642857 | 0.0338387 | 0.9888134 | 0.0070280 |
41 | 3 | {'C': 100, 'kernel': 'linear'} | 0.9642857 | 0.0338387 | 0.9932579 | 0.0055055 |
32 | 9 | {'C': 100, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9553571 | 0.0495662 | 0.9887884 | 0.0099945 |
15 | 9 | {'C': 0.1, 'gamma': 1, 'kernel': 'rbf'} | 0.9553571 | 0.0401043 | 0.9598457 | 0.0113043 |
38 | 11 | {'C': 0.1, 'kernel': 'linear'} | 0.9464286 | 0.0332185 | 0.9665385 | 0.0121316 |
21 | 11 | {'C': 1, 'gamma': 1, 'kernel': 'rbf'} | 0.9464286 | 0.0324799 | 0.9843928 | 0.0088664 |
33 | 11 | {'C': 100, 'gamma': 1, 'kernel': 'rbf'} | 0.9464286 | 0.0519227 | 1.0000000 | 0.0000000 |
27 | 14 | {'C': 10, 'gamma': 1, 'kernel': 'rbf'} | 0.9375000 | 0.0452528 | 0.9865906 | 0.0083624 |
24 | 15 | {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'} | 0.9285714 | 0.0429827 | 0.9353247 | 0.0078884 |
19 | 15 | {'C': 1, 'gamma': 0.01, 'kernel': 'rbf'} | 0.9285714 | 0.0429827 | 0.9353247 | 0.0078884 |
22 | 17 | {'C': 1, 'gamma': 10, 'kernel': 'rbf'} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
14 | 17 | {'C': 0.1, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9196429 | 0.0440102 | 0.9197442 | 0.0212659 |
34 | 17 | {'C': 100, 'gamma': 10, 'kernel': 'rbf'} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
28 | 17 | {'C': 10, 'gamma': 10, 'kernel': 'rbf'} | 0.9196429 | 0.0647906 | 1.0000000 | 0.0000000 |
37 | 21 | {'C': 0.01, 'kernel': 'linear'} | 0.8482143 | 0.0547783 | 0.8550694 | 0.0503114 |
18 | 22 | {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'} | 0.6964286 | 0.0131963 | 0.6964237 | 0.0032580 |
13 | 22 | {'C': 0.1, 'gamma': 0.01, 'kernel': 'rbf'} | 0.6964286 | 0.0131963 | 0.6964237 | 0.0032580 |
35 | 24 | {'C': 100, 'gamma': 100, 'kernel': 'rbf'} | 0.5625000 | 0.0496678 | 1.0000000 | 0.0000000 |
29 | 24 | {'C': 10, 'gamma': 100, 'kernel': 'rbf'} | 0.5625000 | 0.0496678 | 1.0000000 | 0.0000000 |
23 | 26 | {'C': 1, 'gamma': 100, 'kernel': 'rbf'} | 0.5089286 | 0.0464350 | 1.0000000 | 0.0000000 |
36 | 27 | {'C': 0.001, 'kernel': 'linear'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
2 | 27 | {'C': 0.001, 'gamma': 0.1, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
3 | 27 | {'C': 0.001, 'gamma': 1, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
1 | 27 | {'C': 0.001, 'gamma': 0.01, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
8 | 27 | {'C': 0.01, 'gamma': 0.1, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
5 | 27 | {'C': 0.001, 'gamma': 100, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
7 | 27 | {'C': 0.01, 'gamma': 0.01, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
6 | 27 | {'C': 0.01, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
17 | 27 | {'C': 0.1, 'gamma': 100, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
16 | 27 | {'C': 0.1, 'gamma': 10, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3817097 | 0.0213374 |
12 | 27 | {'C': 0.1, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
11 | 27 | {'C': 0.01, 'gamma': 100, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
10 | 27 | {'C': 0.01, 'gamma': 10, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
9 | 27 | {'C': 0.01, 'gamma': 1, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
4 | 27 | {'C': 0.001, 'gamma': 10, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
0 | 27 | {'C': 0.001, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3660714 | 0.0113708 | 0.3660787 | 0.0028518 |
from sklearn.model_selection import StratifiedShuffleSplit
shuffle_split = StratifiedShuffleSplit(test_size=.8, n_splits=1)
grid_search = GridSearchCV(SVC(), param_grid, cv=shuffle_split, return_train_score=True)
grid_search.fit(X_train, y_train)
print("Best parameters: {}".format(grid_search.best_params_))
print("Best cross-validation score: {:.2f}".format(grid_search.best_score_))
Best parameters: {'C': 10, 'gamma': 0.1, 'kernel': 'rbf'} Best cross-validation score: 0.92
results = pd.DataFrame(grid_search.cv_results_)
results2 = results[['rank_test_score', 'params', 'mean_test_score', 'std_test_score',
'mean_train_score', 'std_train_score']]
results2 = results2.sort_values('rank_test_score')
display(results2)
rank_test_score | params | mean_test_score | std_test_score | mean_train_score | std_train_score | |
---|---|---|---|---|---|---|
41 | 1 | {'C': 100, 'kernel': 'linear'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
39 | 1 | {'C': 1, 'kernel': 'linear'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
32 | 1 | {'C': 100, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
31 | 1 | {'C': 100, 'gamma': 0.01, 'kernel': 'rbf'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
26 | 1 | {'C': 10, 'gamma': 0.1, 'kernel': 'rbf'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
40 | 1 | {'C': 10, 'kernel': 'linear'} | 0.9222222 | 0.0000000 | 1.0000000 | 0.0000000 |
38 | 7 | {'C': 0.1, 'kernel': 'linear'} | 0.9111111 | 0.0000000 | 1.0000000 | 0.0000000 |
33 | 8 | {'C': 100, 'gamma': 1, 'kernel': 'rbf'} | 0.9000000 | 0.0000000 | 1.0000000 | 0.0000000 |
30 | 8 | {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'} | 0.9000000 | 0.0000000 | 1.0000000 | 0.0000000 |
27 | 8 | {'C': 10, 'gamma': 1, 'kernel': 'rbf'} | 0.9000000 | 0.0000000 | 1.0000000 | 0.0000000 |
25 | 8 | {'C': 10, 'gamma': 0.01, 'kernel': 'rbf'} | 0.9000000 | 0.0000000 | 1.0000000 | 0.0000000 |
20 | 12 | {'C': 1, 'gamma': 0.1, 'kernel': 'rbf'} | 0.8888889 | 0.0000000 | 1.0000000 | 0.0000000 |
21 | 13 | {'C': 1, 'gamma': 1, 'kernel': 'rbf'} | 0.8777778 | 0.0000000 | 1.0000000 | 0.0000000 |
28 | 14 | {'C': 10, 'gamma': 10, 'kernel': 'rbf'} | 0.7333333 | 0.0000000 | 1.0000000 | 0.0000000 |
22 | 14 | {'C': 1, 'gamma': 10, 'kernel': 'rbf'} | 0.7333333 | 0.0000000 | 1.0000000 | 0.0000000 |
34 | 14 | {'C': 100, 'gamma': 10, 'kernel': 'rbf'} | 0.7333333 | 0.0000000 | 1.0000000 | 0.0000000 |
14 | 17 | {'C': 0.1, 'gamma': 0.1, 'kernel': 'rbf'} | 0.7000000 | 0.0000000 | 0.6818182 | 0.0000000 |
37 | 17 | {'C': 0.01, 'kernel': 'linear'} | 0.7000000 | 0.0000000 | 0.6818182 | 0.0000000 |
19 | 17 | {'C': 1, 'gamma': 0.01, 'kernel': 'rbf'} | 0.7000000 | 0.0000000 | 0.6818182 | 0.0000000 |
24 | 17 | {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'} | 0.7000000 | 0.0000000 | 0.6818182 | 0.0000000 |
15 | 21 | {'C': 0.1, 'gamma': 1, 'kernel': 'rbf'} | 0.5000000 | 0.0000000 | 0.5909091 | 0.0000000 |
35 | 22 | {'C': 100, 'gamma': 100, 'kernel': 'rbf'} | 0.4444444 | 0.0000000 | 1.0000000 | 0.0000000 |
29 | 22 | {'C': 10, 'gamma': 100, 'kernel': 'rbf'} | 0.4444444 | 0.0000000 | 1.0000000 | 0.0000000 |
23 | 24 | {'C': 1, 'gamma': 100, 'kernel': 'rbf'} | 0.4222222 | 0.0000000 | 1.0000000 | 0.0000000 |
4 | 25 | {'C': 0.001, 'gamma': 10, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
3 | 25 | {'C': 0.001, 'gamma': 1, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
12 | 25 | {'C': 0.1, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
2 | 25 | {'C': 0.001, 'gamma': 0.1, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
1 | 25 | {'C': 0.001, 'gamma': 0.01, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
36 | 25 | {'C': 0.001, 'kernel': 'linear'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
5 | 25 | {'C': 0.001, 'gamma': 100, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
8 | 25 | {'C': 0.01, 'gamma': 0.1, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
7 | 25 | {'C': 0.01, 'gamma': 0.01, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
11 | 25 | {'C': 0.01, 'gamma': 100, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
9 | 25 | {'C': 0.01, 'gamma': 1, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
10 | 25 | {'C': 0.01, 'gamma': 10, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
18 | 25 | {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
17 | 25 | {'C': 0.1, 'gamma': 100, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
16 | 25 | {'C': 0.1, 'gamma': 10, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
13 | 25 | {'C': 0.1, 'gamma': 0.01, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
6 | 25 | {'C': 0.01, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
0 | 25 | {'C': 0.001, 'gamma': 0.001, 'kernel': 'rbf'} | 0.3666667 | 0.0000000 | 0.3636364 | 0.0000000 |
param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100],
'gamma': [0.001, 0.01, 0.1, 1, 10, 100]}
grid_search = GridSearchCV(SVC(), param_grid, cv=5)
scores = cross_val_score(grid_search, iris.data, iris.target, n_jobs=-1, cv=5)
print("Cross-validation scores: ", scores)
print("Mean cross-validation score: ", scores.mean())
Cross-validation scores: [0.967 1. 0.967 0.967 1. ] Mean cross-validation score: 0.9800000000000001
def nested_cv(X, y, inner_cv, outer_cv, Classifier, parameter_grid):
outer_scores = []
outer_best_params = []
# for each split of the data in the outer cross-validation
# (split method returns indices of training and test part)
for training_samples, test_samples in outer_cv.split(X, y):
# find best parameter using inner cross-validation
best_parms = {}
best_score = -np.inf
# iterate over parameters
for parameters in parameter_grid:
# accumulate score over inner splits
cv_scores = []
# iterate over inner cross-validation
for inner_train, inner_test in inner_cv.split(X[training_samples], y[training_samples]):
# build classifier given parameters and training data
clf = Classifier(**parameters)
clf.fit(X[inner_train], y[inner_train])
# evaluate on inner test set
score = clf.score(X[inner_test], y[inner_test])
cv_scores.append(score)
# compute mean score over inner folds
mean_score = np.mean(cv_scores)
if mean_score > best_score:
# if better than so far, remember parameters
best_score = mean_score
best_params = parameters
# build classifier on best parameters using outer training set
clf = Classifier(**best_params)
clf.fit(X[training_samples], y[training_samples])
# evaluate
outer_scores.append(clf.score(X[test_samples], y[test_samples]))
outer_best_params.append(best_params)
return np.array(outer_scores), outer_best_params
from sklearn.model_selection import ParameterGrid, StratifiedKFold
scores, params = nested_cv(
iris.data,
iris.target,
StratifiedKFold(5),
StratifiedKFold(5),
SVC,
ParameterGrid(param_grid)
)
print("Cross-validation scores: {}".format(scores))
print("Mean cross-validation score: ", scores.mean())
print("best params: {}".format(params))
Cross-validation scores: [0.967 1. 0.967 0.967 1. ] Mean cross-validation score: 0.9800000000000001 best params: [{'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.01}, {'C': 100, 'gamma': 0.01}]
두 가지 분류 클래스
모델 적용 결과에 대한 분류
암의 조기 발견 어플리케이션
대부분의 경우 *거짓 음성이 거짓 양성*보다 더 치명적
거짓 음성 분류와 거짓 양성 분류 중 하나가 다른 것 보다 훨씬 많을 때 이 상황은 매우 중요한 상황으로 인식해야 함.
불균형 데이터셋(Imbalanced datasets)
따라서, '정확도'만으로 모델의 성능을 판별하는 것은 지양해야 함.
from sklearn.datasets import load_digits
digits = load_digits()
y = digits.target == 9
X_train, X_test, y_train, y_test = train_test_split(digits.data, y, random_state=0)
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)
print(y_test[:10])
print()
print(len(np.where(y_test == True)[0]))
print(len(np.where(y_test == False)[0]))
(1347, 64) (1347,) (450, 64) (450,) [False False False False False False False True False False] 47 403
DummyClassifier
DummyRegressor
from sklearn.dummy import DummyClassifier
dummy_majority = DummyClassifier(strategy='most_frequent').fit(X_train, y_train)
pred_most_frequent = dummy_majority.predict(X_test)
print("Unique predicted labels: {}".format(np.unique(pred_most_frequent)))
print("Test score: {:.2f}".format(dummy_majority.score(X_test, y_test)))
Unique predicted labels: [False] Test score: 0.90
from sklearn.tree import DecisionTreeClassifier
tree = DecisionTreeClassifier(max_depth=2).fit(X_train, y_train)
pred_tree = tree.predict(X_test)
print("Test score: {:.2f}".format(tree.score(X_test, y_test)))
Test score: 0.92
from sklearn.linear_model import LogisticRegression
dummy = DummyClassifier().fit(X_train, y_train)
pred_dummy = dummy.predict(X_test)
print("Unique predicted labels: {}".format(np.unique(pred_dummy)))
print("dummy score: {:.2f}".format(dummy.score(X_test, y_test)))
logreg = LogisticRegression(C=0.1).fit(X_train, y_train)
pred_logreg = logreg.predict(X_test)
print("logreg score: {:.2f}".format(logreg.score(X_test, y_test)))
Unique predicted labels: [False True] dummy score: 0.82 logreg score: 0.98
from sklearn.metrics import confusion_matrix
print(len(np.where(y_test == True)[0]))
print(len(np.where(y_test == False)[0]))
confusion = confusion_matrix(y_test, pred_logreg)
print("Confusion matrix:\n{}".format(confusion))
47 403 Confusion matrix: [[401 2] [ 8 39]]
[*음성* 정답] - 정답 '9가 아님'의 총 개수: 403
[*양성* 정답] - 정답 '9임'의 총 개수: 47
mglearn.plots.plot_confusion_matrix_illustration()
mglearn.plots.plot_binary_confusion_matrix()
print("Most frequent class:")
print(confusion_matrix(y_test, pred_most_frequent))
print("\nDummy model:")
print(confusion_matrix(y_test, pred_dummy))
print("\nDecision tree:")
print(confusion_matrix(y_test, pred_tree))
print("\nLogistic Regression")
print(confusion_matrix(y_test, pred_logreg))
Most frequent class: [[403 0] [ 47 0]] Dummy model: [[361 42] [ 41 6]] Decision tree: [[390 13] [ 24 23]] Logistic Regression [[401 2] [ 8 39]]
from sklearn.metrics import f1_score
print("f1 score most frequent: {:.2f}".format(f1_score(y_test, pred_most_frequent)))
print("f1 score dummy: {:.2f}".format(f1_score(y_test, pred_dummy)))
print("f1 score tree: {:.2f}".format(f1_score(y_test, pred_tree)))
print("f1 score logistic regression: {:.2f}".format(f1_score(y_test, pred_logreg)))
f1 score most frequent: 0.00 f1 score dummy: 0.13 f1 score tree: 0.55 f1 score logistic regression: 0.89
/Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples. 'precision', 'predicted', average, warn_for)
from sklearn.metrics import classification_report
print(classification_report(y_test, pred_most_frequent, target_names=["not nine", "nine"]))
precision recall f1-score support not nine 0.90 1.00 0.94 403 nine 0.00 0.00 0.00 47 avg / total 0.80 0.90 0.85 450
/Users/yhhan/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. 'precision', 'predicted', average, warn_for)
print(classification_report(y_test, pred_dummy, target_names=["not nine", "nine"]))
precision recall f1-score support not nine 0.90 0.90 0.90 403 nine 0.12 0.13 0.13 47 avg / total 0.82 0.82 0.82 450
print(classification_report(y_test, pred_logreg, target_names=["not nine", "nine"]))
precision recall f1-score support not nine 0.98 1.00 0.99 403 nine 0.95 0.83 0.89 47 avg / total 0.98 0.98 0.98 450
from mglearn.datasets import make_blobs
X, y = make_blobs(
n_samples=(400, 50), # 음성 클래스: 400개, 양성 클래스: 50개
centers=2,
cluster_std=[7.0, 2],
random_state=22
)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
print(X_train.shape)
print(y_train.shape)
print()
print(X_test.shape)
print(y_test.shape)
svc = SVC(gamma=.05).fit(X_train, y_train)
(337, 2) (337,) (113, 2) (113,)
mglearn.plots.plot_decision_threshold()
print(classification_report(y_test, svc.predict(X_test)))
precision recall f1-score support 0 0.97 0.89 0.93 104 1 0.35 0.67 0.46 9 avg / total 0.92 0.88 0.89 113
y_pred_lower_threshold = svc.decision_function(X_test) > -.8
print(y_pred_lower_threshold.shape)
(113,)
print(classification_report(y_test, y_pred_lower_threshold))
precision recall f1-score support 0 1.00 0.82 0.90 104 1 0.32 1.00 0.49 9 avg / total 0.95 0.83 0.87 113
from sklearn.metrics import precision_recall_curve
precision, recall, thresholds = precision_recall_curve(y_test, svc.decision_function(X_test))
print("precision: {}\n".format(precision))
print("recall: {}\n".format(recall))
print("thresholds: {}\n".format(thresholds))
close_zero = np.argmin(np.abs(thresholds))
print(close_zero)
print(thresholds[close_zero])
precision: [0.321 0.296 0.308 0.32 0.333 0.348 0.364 0.381 0.4 0.368 0.333 0.353 0.375 0.4 0.429 0.385 0.417 0.455 0.4 0.444 0.5 0.571 0.667 0.6 0.5 0.667 0.5 1. 1. ] recall: [1. 0.889 0.889 0.889 0.889 0.889 0.889 0.889 0.889 0.778 0.667 0.667 0.667 0.667 0.667 0.556 0.556 0.556 0.444 0.444 0.444 0.444 0.444 0.333 0.222 0.222 0.111 0.111 0. ] thresholds: [-0.751 -0.587 -0.487 -0.444 -0.404 -0.29 -0.242 -0.193 -0.179 -0.166 -0.16 0.086 0.146 0.192 0.37 0.52 0.523 0.532 0.632 0.744 0.872 0.88 0.884 0.978 1. 1.07 1.084 1.251] 11 0.08620483947417501
# create a similar dataset as before, but with more samples
# to get a smoother curve
X, y = make_blobs(
n_samples=(4000, 500),
centers=2,
cluster_std=[7.0, 2],
random_state=22
)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
svc = SVC(gamma=.05).fit(X_train, y_train)
precision, recall, thresholds = precision_recall_curve(y_test, svc.decision_function(X_test))
# find threshold closest to zero
close_zero = np.argmin(np.abs(thresholds))
plt.plot(
precision[close_zero],
recall[close_zero],
'o',
markersize=10,
label="threshold zero",
fillstyle="none",
c='k',
mew=2)
plt.plot(precision, recall, label="precision recall curve")
plt.xlabel("Precision")
plt.ylabel("Recall")
plt.legend(loc="best")
<matplotlib.legend.Legend at 0x1c14894f98>
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=100, random_state=0, max_features=2)
rf.fit(X_train, y_train)
# RandomForestClassifier has predict_proba, but not decision_function
precision_rf, recall_rf, thresholds_rf = precision_recall_curve(y_test, rf.predict_proba(X_test)[:, 1])
plt.plot(precision, recall, label="svc")
plt.plot(
precision[close_zero],
recall[close_zero],
'o',
markersize=10,
label="threshold zero svc",
fillstyle="none",
c='k',
mew=2)
plt.plot(precision_rf, recall_rf, label="rf")
close_default_rf = np.argmin(np.abs(thresholds_rf - 0.5))
plt.plot(
precision_rf[close_default_rf],
recall_rf[close_default_rf],
'^',
markersize=10,
label="threshold 0.5 rf",
fillstyle="none",
c='k',
mew=2)
plt.xlabel("Precision")
plt.ylabel("Recall")
plt.legend(loc="best")
<matplotlib.legend.Legend at 0x1c182e6be0>
from sklearn.metrics import f1_score
print("f1_score of random forest: {:.3f}".format(f1_score(y_test, rf.predict(X_test))))
print("f1_score of svc: {:.3f}".format(f1_score(y_test, svc.predict(X_test))))
f1_score of random forest: 0.610 f1_score of svc: 0.656
from sklearn.metrics import average_precision_score
ap_rf = average_precision_score(y_test, rf.predict_proba(X_test)[:, 1])
ap_svc = average_precision_score(y_test, svc.decision_function(X_test))
print("Average precision of random forest: {:.3f}".format(ap_rf))
print("Average precision of svc: {:.3f}".format(ap_svc))
Average precision of random forest: 0.660 Average precision of svc: 0.666
from sklearn.metrics import roc_curve
fpr, tpr, thresholds = roc_curve(y_test, svc.decision_function(X_test))
plt.plot(fpr, tpr, label="ROC Curve")
plt.xlabel("FPR")
plt.ylabel("TPR (recall)")
# find threshold closest to zero
close_zero = np.argmin(np.abs(thresholds))
plt.plot(fpr[close_zero], tpr[close_zero], 'o', markersize=10, label="threshold zero", fillstyle="none", c='k', mew=2)
plt.legend(loc=4)
<matplotlib.legend.Legend at 0x1c14847550>
fpr_rf, tpr_rf, thresholds_rf = roc_curve(y_test, rf.predict_proba(X_test)[:, 1])
plt.plot(fpr, tpr, label="ROC Curve SVC")
plt.plot(fpr_rf, tpr_rf, label="ROC Curve RF")
plt.xlabel("FPR")
plt.ylabel("TPR (recall)")
plt.plot(fpr[close_zero], tpr[close_zero], 'o', markersize=10, label="threshold zero SVC", fillstyle="none", c='k', mew=2)
close_default_rf = np.argmin(np.abs(thresholds_rf - 0.5))
plt.plot(fpr_rf[close_default_rf], tpr[close_default_rf], '^', markersize=10, label="threshold 0.5 RF", fillstyle="none", c='k', mew=2)
plt.legend(loc=4)
<matplotlib.legend.Legend at 0x1c183caf98>
from sklearn.metrics import roc_auc_score
rf_auc = roc_auc_score(y_test, rf.predict_proba(X_test)[:, 1])
svc_auc = roc_auc_score(y_test, svc.decision_function(X_test))
print("AUC for Random Forest: {:.3f}".format(rf_auc))
print("AUC for SVC: {:.3f}".format(svc_auc))
AUC for Random Forest: 0.937 AUC for SVC: 0.916
y = digits.target == 9
X_train, X_test, y_train, y_test = train_test_split(digits.data, y, random_state=0)
plt.figure()
for gamma in [1, 0.05, 0.01]:
svc = SVC(gamma=gamma).fit(X_train, y_train)
accuracy = svc.score(X_test, y_test)
auc = roc_auc_score(y_test, svc.decision_function(X_test))
fpr, tpr, _ = roc_curve(y_test , svc.decision_function(X_test))
print("gamma = {:.2f} accuracy = {:.2f} AUC = {:.2f}".format(gamma, accuracy, auc))
plt.plot(fpr, tpr, label="gamma={:.3f}".format(gamma))
plt.xlabel("FPR")
plt.ylabel("TPR")
plt.xlim(-0.01, 1)
plt.ylim(0, 1.02)
plt.legend(loc="best")
gamma = 1.00 accuracy = 0.90 AUC = 0.50 gamma = 0.05 accuracy = 0.90 AUC = 1.00 gamma = 0.01 accuracy = 0.90 AUC = 1.00
<matplotlib.legend.Legend at 0x1c14868be0>
from sklearn.metrics import accuracy_score
X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, random_state=0)
lr = LogisticRegression().fit(X_train, y_train)
pred = lr.predict(X_test)
print("Shape of Test data: {}".format(y_test.shape))
print("Accuracy: {:.3f}".format(accuracy_score(y_test, pred)))
print()
print("Confusion matrix:\n{}".format(confusion_matrix(y_test, pred)))
Shape of Test data: (450,) Accuracy: 0.953 Confusion matrix: [[37 0 0 0 0 0 0 0 0 0] [ 0 39 0 0 0 0 2 0 2 0] [ 0 0 41 3 0 0 0 0 0 0] [ 0 0 1 43 0 0 0 0 0 1] [ 0 0 0 0 38 0 0 0 0 0] [ 0 1 0 0 0 47 0 0 0 0] [ 0 0 0 0 0 0 52 0 0 0] [ 0 1 0 1 1 0 0 45 0 0] [ 0 3 1 0 0 0 0 0 43 1] [ 0 0 0 1 0 1 0 0 1 44]]
$ \begin{bmatrix} TN & FP \\ FN & TP \end{bmatrix} = \begin{bmatrix} 413 & 0 \\ 0 & 37 \end{bmatrix} $
= \begin{bmatrix} 402 & 0 \\ 3 & 45 \end{bmatrix} $
scores_image = mglearn.tools.heatmap(
confusion_matrix(y_test, pred), xlabel='Predicted label',
ylabel='True label', xticklabels=digits.target_names,
yticklabels=digits.target_names, cmap=plt.cm.gray_r, fmt="%d")
plt.title("Confusion matrix")
plt.gca().invert_yaxis()
print(classification_report(y_test, pred))
precision recall f1-score support 0 1.00 1.00 1.00 37 1 0.89 0.91 0.90 43 2 0.95 0.93 0.94 44 3 0.90 0.96 0.92 45 4 0.97 1.00 0.99 38 5 0.98 0.98 0.98 48 6 0.96 1.00 0.98 52 7 1.00 0.94 0.97 48 8 0.93 0.90 0.91 48 9 0.96 0.94 0.95 47 avg / total 0.95 0.95 0.95 450
print("Macro average f1 score: {:.3f}".format(f1_score(y_test, pred, average="macro")))
print("Weighted average f1 score: {:.3f}".format(f1_score(y_test, pred, average="weighted")))
print("Micro average f1 score: {:.3f}".format(f1_score(y_test, pred, average="micro")))
Macro average f1 score: 0.954 Weighted average f1 score: 0.953 Micro average f1 score: 0.953
from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import median_absolute_error
from sklearn.metrics import r2_score
y_test = [3, -0.5, 2, 7]
y_pred = [2.5, 0.0, 2, 8]
print("explained_variance_score:", explained_variance_score(y_test, y_pred))
print("mean_absolute_error:", mean_absolute_error(y_test, y_pred))
print("mean_squared_error:", mean_squared_error(y_test, y_pred))
print("mean_squared_log_error:", mean_squared_log_error(y_test, y_pred))
print("median_absolute_error:", median_absolute_error(y_test, y_pred))
print("r2_score:", r2_score(y_test, y_pred))
explained_variance_score: 0.9571734475374732 mean_absolute_error: 0.5 mean_squared_error: 0.375 mean_squared_log_error: 0.12803912255571967 median_absolute_error: 0.5 r2_score: 0.9486081370449679
import warnings
warnings.filterwarnings('ignore')
from sklearn.ensemble import GradientBoostingRegressor
X, y = mglearn.datasets.make_wave(n_samples=200)
print("X shape: {}".format(X.shape))
print("y shape: {}".format(y.shape))
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
print("X_train shape: {}".format(X_train.shape))
print("X_test shape: {}".format(X_test.shape))
print()
param_grid = {
'learning_rate': [0.001, 0.01, 0.1, 1, 10, 100],
'alpha': [0.1, 0.3, 0.5, 0.7, 0.9]
}
estimator = GradientBoostingRegressor()
grid_search = GridSearchCV(
estimator = estimator,
param_grid = param_grid,
n_jobs = -1,
cv = 5,
return_train_score = True
)
grid_search.fit(X_train, y_train)
print("Best cross-validation accuracy: {:.2f}".format(grid_search.best_score_))
print("Best parameters:\n{}".format(grid_search.best_params_))
print("Best estimator:\n{}".format(grid_search.best_estimator_))
print("Test set score: {:.2f}".format(grid_search.score(X_test, y_test)))
y_pred = gbr.predict(X_test)
print()
# Possible scoring
print("explained_variance_score:", explained_variance_score(y_test, y_pred))
print("mean_absolute_error:", mean_absolute_error(y_test, y_pred))
print("mean_squared_error:", mean_squared_error(y_test, y_pred))
#print("mean_squared_log_error:", mean_squared_log_error(y_test, y_pred))
print("median_absolute_error:", median_absolute_error(y_test, y_pred))
print("r2_score:", r2_score(y_test, y_pred))
X shape: (200, 1) y shape: (200,) X_train shape: (150, 1) X_test shape: (50, 1) Best cross-validation accuracy: 0.67 Best parameters: {'alpha': 0.1, 'learning_rate': 0.1} Best estimator: GradientBoostingRegressor(alpha=0.1, criterion='friedman_mse', init=None, learning_rate=0.1, loss='ls', max_depth=3, max_features=None, max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, min_samples_leaf=1, min_samples_split=2, min_weight_fraction_leaf=0.0, n_estimators=100, presort='auto', random_state=None, subsample=1.0, verbose=0, warm_start=False) Test set score: 0.62 explained_variance_score: 0.6333933570920071 mean_absolute_error: 0.4410440244803421 mean_squared_error: 0.2945365131871811 median_absolute_error: 0.37617245551529926 r2_score: 0.6195689836183305
# default scoring for classification is accuracy
scores = cross_val_score(SVC(), digits.data, digits.target == 9)
print("Default scoring: {}".format(scores))
# providing scoring="accuracy" doesn't change the results
scores2 = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="accuracy")
print("Explicit accuracy scoring: {}".format(scores2))
print()
# 곡선의 면적을 활용한 성능 측정 (Recommended)
roc_auc = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="roc_auc")
print("ROC_AUC scoring: {}".format(roc_auc))
average_precision = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="average_precision")
print("Average Precision scoring: {}".format(average_precision))
print()
# 다양한 성능 측정 (Not Recommended)
precision = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="precision_weighted")
print("Precision scoring: {}".format(precision))
recall = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="recall_weighted")
print("Precision scoring: {}".format(recall))
f1_score = cross_val_score(SVC(), digits.data, digits.target == 9, scoring="f1_weighted")
print("F1_score scoring: {}".format(f1_score))
Default scoring: [0.9 0.9 0.9] Explicit accuracy scoring: [0.9 0.9 0.9] ROC_AUC scoring: [0.994 0.99 0.996] Average Precision scoring: [0.96 0.953 0.978] Precision scoring: [0.81 0.81 0.81] Precision scoring: [0.9 0.9 0.9] F1_score scoring: [0.852 0.852 0.852]
scores = cross_val_score(SVC(), digits.data, digits.target, scoring="accuracy")
print("Explicit accuracy scoring: {}".format(scores))
f1_weighted = cross_val_score(SVC(), digits.data, digits.target, scoring="f1_weighted")
print("F1_weighted scoring: {}".format(f1_weighted))
Explicit accuracy scoring: [0.394 0.411 0.46 ] F1_weighted scoring: [0.439 0.463 0.524]
X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target == 9, random_state=0)
# we provide a somewhat bad grid to illustrate the point:
param_grid = {'gamma': [0.0001, 0.01, 0.1, 1, 10]}
# using the default scoring of accuracy:
grid = GridSearchCV(SVC(), param_grid=param_grid)
grid.fit(X_train, y_train)
print("Grid-Search with accuracy")
print("Best parameters:", grid.best_params_)
print("Best cross-validation score (accuracy)): {:.3f}".format(grid.best_score_))
print("Test set AUC: {:.3f}".format(roc_auc_score(y_test, grid.decision_function(X_test))))
print("Test set accuracy: {:.3f}".format(grid.score(X_test, y_test)))
print()
# using AUC scoring instead:
grid = GridSearchCV(SVC(), param_grid=param_grid, scoring="roc_auc")
grid.fit(X_train, y_train)
print("Grid-Search with AUC")
print("Best parameters:", grid.best_params_)
print("Best cross-validation score (AUC): {:.3f}".format(grid.best_score_))
print("Test set AUC: {:.3f}".format(roc_auc_score(y_test, grid.decision_function(X_test))))
print("Test set accuracy: {:.3f}".format(grid.score(X_test, y_test)))
Grid-Search with accuracy Best parameters: {'gamma': 0.0001} Best cross-validation score (accuracy)): 0.970 Test set AUC: 0.992 Test set accuracy: 0.973 Grid-Search with AUC Best parameters: {'gamma': 0.01} Best cross-validation score (AUC): 0.997 Test set AUC: 1.000 Test set accuracy: 1.000
from sklearn.metrics.scorer import SCORERS
print("Available scorers:\n{}".format(sorted(SCORERS.keys())))
Available scorers: ['accuracy', 'adjusted_mutual_info_score', 'adjusted_rand_score', 'average_precision', 'completeness_score', 'explained_variance', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'fowlkes_mallows_score', 'homogeneity_score', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'mutual_info_score', 'neg_log_loss', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'normalized_mutual_info_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc', 'v_measure_score']