import numpy as np
from scipy.special import expit as sigmoid
from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin
from sklearn.tree import DecisionTreeRegressor
from sklearn.datasets import make_regression
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
class BaseGradientBoosting(BaseEstimator):
def __init__(self, n_iter, learning_rate, loss):
self.loss = loss
self.learning_rate = learning_rate
self.n_iter = n_iter
def fit(self, X, y):
rng = np.random.RandomState(seed=0)
n_samples = X.shape[0]
y_pred_train = rng.normal(size=n_samples)
self.predictors = list()
for m in range(self.n_iter): # Gradient Descent
negative_gradient = -self.loss.compute_gradients(y, y_pred_train)
new_predictor = DecisionTreeRegressor(max_depth=3)
new_predictor.fit(X, y=self.learning_rate * negative_gradient)
y_pred_train += new_predictor.predict(X)
self.predictors.append(new_predictor) # save for predict()
def predict(self, X):
return sum(predictor.predict(X) for predictor in self.predictors)
# GBDT for regression
class LeastSquaresLoss:
def compute_gradients(self, y_true, y_pred):
return -2 * (y_true - y_pred)
class GradientBoostingRegressor(BaseGradientBoosting, RegressorMixin):
def __init__(self, n_iter=100, learning_rate=.1):
super().__init__(n_iter, learning_rate, loss=LeastSquaresLoss())
X, y = make_regression(n_samples=1000, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
gb = GradientBoostingRegressor()
gb.fit(X_train, y_train)
print('r squared on training data: {:.3f}'.format(gb.score(X_train, y_train)))
print('r squared on test data: {:.3f}'.format(gb.score(X_test, y_test)))
r squared on training data: 0.997 r squared on test data: 0.937
# GBDT for classification
class BinaryCrossEntropy:
def compute_gradients(self, y_true, y_pred):
return sigmoid(y_pred) - y_true
def raw_predictions_to_proba(self, raw_predictions):
return sigmoid(raw_predictions)
class GradientBoostingClassifier(BaseGradientBoosting, ClassifierMixin):
def __init__(self, n_iter=100, learning_rate=.1):
super().__init__(n_iter, learning_rate, loss=BinaryCrossEntropy())
def predict(self, X):
raw_predictions = super().predict(X)
proba_positive_class = self.loss.raw_predictions_to_proba(raw_predictions)
return proba_positive_class > .5
X, y = make_classification(n_samples=1000, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
gb = GradientBoostingClassifier()
gb.fit(X_train, y_train)
print('accuracy on training data: {:.3f}'.format(gb.score(X_train, y_train)))
print('accuracy on test data: {:.3f}'.format(gb.score(X_test, y_test)))
accuracy on training data: 0.981 accuracy on test data: 0.956