import math
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.datasets import fetch_california_housing
import tensorflow_addons as tfa
from tensorflow.keras.callbacks import EarlyStopping
from tabtransformertf.models.fttransformer import FTTransformerEncoder, FTTransformer
from tabtransformertf.utils.preprocessing import df_to_dataset
import catboost as cb
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
import seaborn as sns
/Users/antonsruberts/miniconda/envs/blog/lib/python3.9/site-packages/tensorflow_addons/utils/ensure_tf_install.py:53: UserWarning: Tensorflow Addons supports using Python ops for all Tensorflow versions above or equal to 2.4.0 and strictly below 2.7.0 (nightly versions are not supported). The versions of TensorFlow you are currently using is 2.7.0 and is not supported. Some things might work, some things might not. If you were to encounter a bug, do not file an issue. If you want to make sure you're using a tested and supported configuration, either change the TensorFlow version or the TensorFlow Addons's version. You can find the compatibility matrix in TensorFlow Addon's readme: https://github.com/tensorflow/addons warnings.warn(
%matplotlib inline
plt.rcParams["figure.figsize"] = (20,10)
plt.rcParams.update({'font.size': 15})
import random
random.seed(42)
dset = fetch_california_housing()
data = dset['data']
y = dset['target']
LABEL = dset['target_names'][0]
NUMERIC_FEATURES = ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup', 'Longitude', 'Latitude']
data = pd.DataFrame(data, columns=dset['feature_names'])
data[LABEL] = y
train_data, test_data = train_test_split(data, test_size=0.2)
print(f"Train dataset shape: {train_data.shape}")
print(f"Test dataset shape: {test_data.shape}")
Train dataset shape: (16512, 9) Test dataset shape: (4128, 9)
X_train, X_val = train_test_split(train_data, test_size=0.2)
sc = StandardScaler()
X_train.loc[:, NUMERIC_FEATURES] = sc.fit_transform(X_train[NUMERIC_FEATURES])
X_val.loc[:, NUMERIC_FEATURES] = sc.transform(X_val[NUMERIC_FEATURES])
test_data.loc[:, NUMERIC_FEATURES] = sc.transform(test_data[NUMERIC_FEATURES])
FEATURES = NUMERIC_FEATURES
sns.distplot(X_train[LABEL])
sns.distplot(X_val[LABEL])
sns.distplot(test_data[LABEL])
/var/folders/66/1klxbkpn5vdgpvqwt_hmtn5c0000gn/T/ipykernel_65933/141569203.py:1: UserWarning: `distplot` is a deprecated function and will be removed in seaborn v0.14.0. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms). For a guide to updating your code to use the new functions, please see https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751 sns.distplot(X_train[LABEL]) /var/folders/66/1klxbkpn5vdgpvqwt_hmtn5c0000gn/T/ipykernel_65933/141569203.py:2: UserWarning: `distplot` is a deprecated function and will be removed in seaborn v0.14.0. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms). For a guide to updating your code to use the new functions, please see https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751 sns.distplot(X_val[LABEL]) /var/folders/66/1klxbkpn5vdgpvqwt_hmtn5c0000gn/T/ipykernel_65933/141569203.py:3: UserWarning: `distplot` is a deprecated function and will be removed in seaborn v0.14.0. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms). For a guide to updating your code to use the new functions, please see https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751 sns.distplot(test_data[LABEL])
<AxesSubplot:xlabel='MedHouseVal', ylabel='Density'>
rf = RandomForestRegressor(n_estimators=100, max_depth=20)
rf.fit(X_train[FEATURES], X_train[LABEL])
RandomForestRegressor(max_depth=20)
rf_preds = rf.predict(test_data[FEATURES])
rf_rms = mean_squared_error(test_data[LABEL], rf_preds, squared=False)
print(rf_rms)
0.5111468321095857
catb_train_dataset = cb.Pool(X_train[FEATURES], X_train[LABEL])
catb_val_dataset = cb.Pool(X_val[FEATURES], X_val[LABEL])
catb_test_dataset = cb.Pool(test_data[FEATURES], test_data[LABEL])
tuned_catb = cb.CatBoostRegressor()
tuned_catb.fit(catb_train_dataset, eval_set=catb_val_dataset, early_stopping_rounds=50)
Learning rate set to 0.076361 0: learn: 1.1095583 test: 1.0881120 best: 1.0881120 (0) total: 12.5ms remaining: 12.5s 1: learn: 1.0665468 test: 1.0453537 best: 1.0453537 (1) total: 14.4ms remaining: 7.19s 2: learn: 1.0256526 test: 1.0062932 best: 1.0062932 (2) total: 16.9ms remaining: 5.61s 3: learn: 0.9892363 test: 0.9709483 best: 0.9709483 (3) total: 18.4ms remaining: 4.58s 4: learn: 0.9547325 test: 0.9376612 best: 0.9376612 (4) total: 19.9ms remaining: 3.96s 5: learn: 0.9241432 test: 0.9081350 best: 0.9081350 (5) total: 22.5ms remaining: 3.72s 6: learn: 0.8956798 test: 0.8803798 best: 0.8803798 (6) total: 25ms remaining: 3.54s 7: learn: 0.8700884 test: 0.8555606 best: 0.8555606 (7) total: 26.8ms remaining: 3.32s 8: learn: 0.8476829 test: 0.8335607 best: 0.8335607 (8) total: 28.3ms remaining: 3.12s 9: learn: 0.8265173 test: 0.8135861 best: 0.8135861 (9) total: 29.9ms remaining: 2.96s 10: learn: 0.8076136 test: 0.7953894 best: 0.7953894 (10) total: 31.4ms remaining: 2.83s 11: learn: 0.7892390 test: 0.7776602 best: 0.7776602 (11) total: 32.9ms remaining: 2.71s 12: learn: 0.7722510 test: 0.7618619 best: 0.7618619 (12) total: 34.4ms remaining: 2.61s 13: learn: 0.7576047 test: 0.7481415 best: 0.7481415 (13) total: 36ms remaining: 2.53s 14: learn: 0.7452769 test: 0.7364133 best: 0.7364133 (14) total: 38ms remaining: 2.49s 15: learn: 0.7327560 test: 0.7245772 best: 0.7245772 (15) total: 39.6ms remaining: 2.44s 16: learn: 0.7197274 test: 0.7123585 best: 0.7123585 (16) total: 41.3ms remaining: 2.39s 17: learn: 0.7069121 test: 0.7004626 best: 0.7004626 (17) total: 42.9ms remaining: 2.34s 18: learn: 0.6976305 test: 0.6914005 best: 0.6914005 (18) total: 45.7ms remaining: 2.36s 19: learn: 0.6867908 test: 0.6811619 best: 0.6811619 (19) total: 47.4ms remaining: 2.32s 20: learn: 0.6790106 test: 0.6740170 best: 0.6740170 (20) total: 48.9ms remaining: 2.28s 21: learn: 0.6718886 test: 0.6673920 best: 0.6673920 (21) total: 50.6ms remaining: 2.25s 22: learn: 0.6658073 test: 0.6618017 best: 0.6618017 (22) total: 52.3ms remaining: 2.22s 23: learn: 0.6584829 test: 0.6554214 best: 0.6554214 (23) total: 54.8ms remaining: 2.23s 24: learn: 0.6492555 test: 0.6468283 best: 0.6468283 (24) total: 56.5ms remaining: 2.21s 25: learn: 0.6434679 test: 0.6417883 best: 0.6417883 (25) total: 58.1ms remaining: 2.18s 26: learn: 0.6385035 test: 0.6377783 best: 0.6377783 (26) total: 59.9ms remaining: 2.16s 27: learn: 0.6329988 test: 0.6324359 best: 0.6324359 (27) total: 61.7ms remaining: 2.14s 28: learn: 0.6269642 test: 0.6268466 best: 0.6268466 (28) total: 63.4ms remaining: 2.12s 29: learn: 0.6234592 test: 0.6239189 best: 0.6239189 (29) total: 65.4ms remaining: 2.11s 30: learn: 0.6179250 test: 0.6181557 best: 0.6181557 (30) total: 67.9ms remaining: 2.12s 31: learn: 0.6144073 test: 0.6144330 best: 0.6144330 (31) total: 70.6ms remaining: 2.13s 32: learn: 0.6086309 test: 0.6090642 best: 0.6090642 (32) total: 72.4ms remaining: 2.12s 33: learn: 0.6037458 test: 0.6043272 best: 0.6043272 (33) total: 75.2ms remaining: 2.14s 34: learn: 0.6004579 test: 0.6011762 best: 0.6011762 (34) total: 77.3ms remaining: 2.13s 35: learn: 0.5981243 test: 0.5992781 best: 0.5992781 (35) total: 79.3ms remaining: 2.12s 36: learn: 0.5931739 test: 0.5944431 best: 0.5944431 (36) total: 81.3ms remaining: 2.12s 37: learn: 0.5898248 test: 0.5908918 best: 0.5908918 (37) total: 84.3ms remaining: 2.13s 38: learn: 0.5871147 test: 0.5883600 best: 0.5883600 (38) total: 86.5ms remaining: 2.13s 39: learn: 0.5846173 test: 0.5860292 best: 0.5860292 (39) total: 89.1ms remaining: 2.14s 40: learn: 0.5822084 test: 0.5838252 best: 0.5838252 (40) total: 91.3ms remaining: 2.13s 41: learn: 0.5803170 test: 0.5823210 best: 0.5823210 (41) total: 93.6ms remaining: 2.13s 42: learn: 0.5774009 test: 0.5795503 best: 0.5795503 (42) total: 95.7ms remaining: 2.13s 43: learn: 0.5735119 test: 0.5760510 best: 0.5760510 (43) total: 97.8ms remaining: 2.13s 44: learn: 0.5717143 test: 0.5746729 best: 0.5746729 (44) total: 99.9ms remaining: 2.12s 45: learn: 0.5686860 test: 0.5720414 best: 0.5720414 (45) total: 102ms remaining: 2.12s 46: learn: 0.5658872 test: 0.5696752 best: 0.5696752 (46) total: 104ms remaining: 2.11s 47: learn: 0.5638058 test: 0.5678917 best: 0.5678917 (47) total: 107ms remaining: 2.12s 48: learn: 0.5619500 test: 0.5659927 best: 0.5659927 (48) total: 109ms remaining: 2.11s 49: learn: 0.5605352 test: 0.5648541 best: 0.5648541 (49) total: 111ms remaining: 2.11s 50: learn: 0.5589743 test: 0.5632843 best: 0.5632843 (50) total: 113ms remaining: 2.11s 51: learn: 0.5576368 test: 0.5620282 best: 0.5620282 (51) total: 115ms remaining: 2.1s 52: learn: 0.5552292 test: 0.5598264 best: 0.5598264 (52) total: 117ms remaining: 2.09s 53: learn: 0.5527650 test: 0.5575684 best: 0.5575684 (53) total: 120ms remaining: 2.09s 54: learn: 0.5511119 test: 0.5560437 best: 0.5560437 (54) total: 121ms remaining: 2.09s 55: learn: 0.5495379 test: 0.5547161 best: 0.5547161 (55) total: 124ms remaining: 2.08s 56: learn: 0.5481851 test: 0.5537525 best: 0.5537525 (56) total: 126ms remaining: 2.08s 57: learn: 0.5473546 test: 0.5528840 best: 0.5528840 (57) total: 128ms remaining: 2.08s 58: learn: 0.5459717 test: 0.5515309 best: 0.5515309 (58) total: 130ms remaining: 2.07s 59: learn: 0.5447503 test: 0.5503841 best: 0.5503841 (59) total: 132ms remaining: 2.07s 60: learn: 0.5427674 test: 0.5483932 best: 0.5483932 (60) total: 134ms remaining: 2.07s 61: learn: 0.5411051 test: 0.5466629 best: 0.5466629 (61) total: 136ms remaining: 2.06s 62: learn: 0.5396007 test: 0.5454116 best: 0.5454116 (62) total: 138ms remaining: 2.06s 63: learn: 0.5384043 test: 0.5445576 best: 0.5445576 (63) total: 140ms remaining: 2.05s 64: learn: 0.5375733 test: 0.5440341 best: 0.5440341 (64) total: 142ms remaining: 2.04s 65: learn: 0.5364740 test: 0.5433318 best: 0.5433318 (65) total: 144ms remaining: 2.04s 66: learn: 0.5342059 test: 0.5410842 best: 0.5410842 (66) total: 146ms remaining: 2.03s 67: learn: 0.5335266 test: 0.5406767 best: 0.5406767 (67) total: 148ms remaining: 2.03s 68: learn: 0.5324534 test: 0.5396904 best: 0.5396904 (68) total: 149ms remaining: 2.02s 69: learn: 0.5316308 test: 0.5388161 best: 0.5388161 (69) total: 152ms remaining: 2.01s 70: learn: 0.5308245 test: 0.5382193 best: 0.5382193 (70) total: 153ms remaining: 2s 71: learn: 0.5296715 test: 0.5371941 best: 0.5371941 (71) total: 156ms remaining: 2.01s 72: learn: 0.5288267 test: 0.5363812 best: 0.5363812 (72) total: 158ms remaining: 2s 73: learn: 0.5281700 test: 0.5357871 best: 0.5357871 (73) total: 160ms remaining: 2s 74: learn: 0.5272767 test: 0.5350212 best: 0.5350212 (74) total: 162ms remaining: 1.99s 75: learn: 0.5263206 test: 0.5345923 best: 0.5345923 (75) total: 164ms remaining: 1.99s 76: learn: 0.5249515 test: 0.5331182 best: 0.5331182 (76) total: 165ms remaining: 1.98s 77: learn: 0.5240734 test: 0.5324422 best: 0.5324422 (77) total: 167ms remaining: 1.98s 78: learn: 0.5233650 test: 0.5321581 best: 0.5321581 (78) total: 169ms remaining: 1.97s 79: learn: 0.5224577 test: 0.5316648 best: 0.5316648 (79) total: 171ms remaining: 1.97s 80: learn: 0.5205762 test: 0.5299709 best: 0.5299709 (80) total: 174ms remaining: 1.97s 81: learn: 0.5198431 test: 0.5294422 best: 0.5294422 (81) total: 177ms remaining: 1.98s 82: learn: 0.5183171 test: 0.5280369 best: 0.5280369 (82) total: 179ms remaining: 1.97s 83: learn: 0.5176717 test: 0.5276445 best: 0.5276445 (83) total: 181ms remaining: 1.97s 84: learn: 0.5168389 test: 0.5268163 best: 0.5268163 (84) total: 184ms remaining: 1.98s 85: learn: 0.5159815 test: 0.5264867 best: 0.5264867 (85) total: 186ms remaining: 1.98s 86: learn: 0.5152221 test: 0.5261003 best: 0.5261003 (86) total: 188ms remaining: 1.98s 87: learn: 0.5142495 test: 0.5254863 best: 0.5254863 (87) total: 191ms remaining: 1.98s 88: learn: 0.5130888 test: 0.5244914 best: 0.5244914 (88) total: 193ms remaining: 1.98s 89: learn: 0.5123750 test: 0.5241766 best: 0.5241766 (89) total: 195ms remaining: 1.98s 90: learn: 0.5118165 test: 0.5237088 best: 0.5237088 (90) total: 198ms remaining: 1.97s 91: learn: 0.5110588 test: 0.5232339 best: 0.5232339 (91) total: 200ms remaining: 1.97s 92: learn: 0.5103424 test: 0.5227407 best: 0.5227407 (92) total: 202ms remaining: 1.97s 93: learn: 0.5095282 test: 0.5223773 best: 0.5223773 (93) total: 204ms remaining: 1.97s 94: learn: 0.5088656 test: 0.5220076 best: 0.5220076 (94) total: 206ms remaining: 1.96s 95: learn: 0.5068319 test: 0.5200034 best: 0.5200034 (95) total: 209ms remaining: 1.96s 96: learn: 0.5063532 test: 0.5195774 best: 0.5195774 (96) total: 210ms remaining: 1.96s 97: learn: 0.5057246 test: 0.5193373 best: 0.5193373 (97) total: 212ms remaining: 1.95s 98: learn: 0.5050172 test: 0.5187724 best: 0.5187724 (98) total: 214ms remaining: 1.95s 99: learn: 0.5045517 test: 0.5184133 best: 0.5184133 (99) total: 216ms remaining: 1.94s 100: learn: 0.5040171 test: 0.5181372 best: 0.5181372 (100) total: 218ms remaining: 1.94s 101: learn: 0.5035286 test: 0.5179268 best: 0.5179268 (101) total: 220ms remaining: 1.93s 102: learn: 0.5028391 test: 0.5172142 best: 0.5172142 (102) total: 222ms remaining: 1.93s 103: learn: 0.5022377 test: 0.5170015 best: 0.5170015 (103) total: 224ms remaining: 1.93s 104: learn: 0.5009955 test: 0.5157114 best: 0.5157114 (104) total: 225ms remaining: 1.92s 105: learn: 0.5003875 test: 0.5155258 best: 0.5155258 (105) total: 227ms remaining: 1.92s 106: learn: 0.4995588 test: 0.5149240 best: 0.5149240 (106) total: 229ms remaining: 1.91s 107: learn: 0.4991062 test: 0.5147603 best: 0.5147603 (107) total: 231ms remaining: 1.91s 108: learn: 0.4982047 test: 0.5140554 best: 0.5140554 (108) total: 233ms remaining: 1.9s 109: learn: 0.4972622 test: 0.5136570 best: 0.5136570 (109) total: 234ms remaining: 1.9s 110: learn: 0.4965052 test: 0.5130658 best: 0.5130658 (110) total: 236ms remaining: 1.89s 111: learn: 0.4957817 test: 0.5123801 best: 0.5123801 (111) total: 238ms remaining: 1.89s 112: learn: 0.4948535 test: 0.5115987 best: 0.5115987 (112) total: 240ms remaining: 1.88s 113: learn: 0.4942940 test: 0.5110781 best: 0.5110781 (113) total: 241ms remaining: 1.88s 114: learn: 0.4937026 test: 0.5107385 best: 0.5107385 (114) total: 243ms remaining: 1.87s 115: learn: 0.4932004 test: 0.5104221 best: 0.5104221 (115) total: 245ms remaining: 1.87s 116: learn: 0.4927203 test: 0.5102999 best: 0.5102999 (116) total: 247ms remaining: 1.86s 117: learn: 0.4922763 test: 0.5099347 best: 0.5099347 (117) total: 249ms remaining: 1.86s 118: learn: 0.4916442 test: 0.5095815 best: 0.5095815 (118) total: 251ms remaining: 1.86s 119: learn: 0.4911869 test: 0.5093852 best: 0.5093852 (119) total: 252ms remaining: 1.85s 120: learn: 0.4898788 test: 0.5080086 best: 0.5080086 (120) total: 254ms remaining: 1.85s 121: learn: 0.4890990 test: 0.5075641 best: 0.5075641 (121) total: 256ms remaining: 1.84s 122: learn: 0.4882052 test: 0.5065343 best: 0.5065343 (122) total: 258ms remaining: 1.84s 123: learn: 0.4876528 test: 0.5061295 best: 0.5061295 (123) total: 260ms remaining: 1.83s 124: learn: 0.4869966 test: 0.5058176 best: 0.5058176 (124) total: 262ms remaining: 1.83s 125: learn: 0.4866247 test: 0.5056009 best: 0.5056009 (125) total: 263ms remaining: 1.82s 126: learn: 0.4861119 test: 0.5050987 best: 0.5050987 (126) total: 265ms remaining: 1.82s 127: learn: 0.4856722 test: 0.5048975 best: 0.5048975 (127) total: 266ms remaining: 1.81s 128: learn: 0.4848439 test: 0.5045729 best: 0.5045729 (128) total: 269ms remaining: 1.82s 129: learn: 0.4845216 test: 0.5044496 best: 0.5044496 (129) total: 271ms remaining: 1.81s 130: learn: 0.4839588 test: 0.5043427 best: 0.5043427 (130) total: 273ms remaining: 1.81s 131: learn: 0.4834653 test: 0.5040548 best: 0.5040548 (131) total: 274ms remaining: 1.8s 132: learn: 0.4829768 test: 0.5037523 best: 0.5037523 (132) total: 276ms remaining: 1.8s 133: learn: 0.4822573 test: 0.5033178 best: 0.5033178 (133) total: 278ms remaining: 1.8s 134: learn: 0.4816677 test: 0.5030359 best: 0.5030359 (134) total: 280ms remaining: 1.79s 135: learn: 0.4809819 test: 0.5025658 best: 0.5025658 (135) total: 281ms remaining: 1.79s 136: learn: 0.4806162 test: 0.5022973 best: 0.5022973 (136) total: 283ms remaining: 1.78s 137: learn: 0.4801806 test: 0.5020857 best: 0.5020857 (137) total: 285ms remaining: 1.78s 138: learn: 0.4796855 test: 0.5018488 best: 0.5018488 (138) total: 286ms remaining: 1.77s 139: learn: 0.4791549 test: 0.5018299 best: 0.5018299 (139) total: 288ms remaining: 1.77s 140: learn: 0.4781387 test: 0.5010955 best: 0.5010955 (140) total: 290ms remaining: 1.77s 141: learn: 0.4775505 test: 0.5009320 best: 0.5009320 (141) total: 292ms remaining: 1.76s 142: learn: 0.4770279 test: 0.5005132 best: 0.5005132 (142) total: 293ms remaining: 1.76s 143: learn: 0.4765113 test: 0.5001188 best: 0.5001188 (143) total: 295ms remaining: 1.75s 144: learn: 0.4760068 test: 0.4999983 best: 0.4999983 (144) total: 297ms remaining: 1.75s 145: learn: 0.4751624 test: 0.4992649 best: 0.4992649 (145) total: 299ms remaining: 1.75s 146: learn: 0.4747860 test: 0.4990964 best: 0.4990964 (146) total: 301ms remaining: 1.74s 147: learn: 0.4740333 test: 0.4985572 best: 0.4985572 (147) total: 303ms remaining: 1.74s 148: learn: 0.4737011 test: 0.4984917 best: 0.4984917 (148) total: 304ms remaining: 1.74s 149: learn: 0.4732045 test: 0.4982251 best: 0.4982251 (149) total: 306ms remaining: 1.73s 150: learn: 0.4728080 test: 0.4980420 best: 0.4980420 (150) total: 308ms remaining: 1.73s 151: learn: 0.4723490 test: 0.4976653 best: 0.4976653 (151) total: 309ms remaining: 1.73s 152: learn: 0.4711559 test: 0.4967602 best: 0.4967602 (152) total: 311ms remaining: 1.72s 153: learn: 0.4706986 test: 0.4967116 best: 0.4967116 (153) total: 313ms remaining: 1.72s 154: learn: 0.4698777 test: 0.4961506 best: 0.4961506 (154) total: 314ms remaining: 1.71s 155: learn: 0.4693788 test: 0.4957685 best: 0.4957685 (155) total: 316ms remaining: 1.71s 156: learn: 0.4689852 test: 0.4956002 best: 0.4956002 (156) total: 318ms remaining: 1.71s 157: learn: 0.4684932 test: 0.4953504 best: 0.4953504 (157) total: 319ms remaining: 1.7s 158: learn: 0.4681757 test: 0.4951478 best: 0.4951478 (158) total: 321ms remaining: 1.7s 159: learn: 0.4677574 test: 0.4950387 best: 0.4950387 (159) total: 322ms remaining: 1.69s 160: learn: 0.4674240 test: 0.4948524 best: 0.4948524 (160) total: 324ms remaining: 1.69s 161: learn: 0.4666871 test: 0.4940861 best: 0.4940861 (161) total: 326ms remaining: 1.69s 162: learn: 0.4661525 test: 0.4938116 best: 0.4938116 (162) total: 328ms remaining: 1.68s 163: learn: 0.4655983 test: 0.4934390 best: 0.4934390 (163) total: 329ms remaining: 1.68s 164: learn: 0.4645857 test: 0.4926002 best: 0.4926002 (164) total: 331ms remaining: 1.67s 165: learn: 0.4641644 test: 0.4926371 best: 0.4926002 (164) total: 333ms remaining: 1.67s 166: learn: 0.4635668 test: 0.4923227 best: 0.4923227 (166) total: 335ms remaining: 1.67s 167: learn: 0.4629516 test: 0.4918334 best: 0.4918334 (167) total: 337ms remaining: 1.67s 168: learn: 0.4624099 test: 0.4916429 best: 0.4916429 (168) total: 338ms remaining: 1.66s 169: learn: 0.4618782 test: 0.4916166 best: 0.4916166 (169) total: 340ms remaining: 1.66s 170: learn: 0.4609966 test: 0.4909287 best: 0.4909287 (170) total: 342ms remaining: 1.66s 171: learn: 0.4606705 test: 0.4906489 best: 0.4906489 (171) total: 344ms remaining: 1.66s 172: learn: 0.4602082 test: 0.4906122 best: 0.4906122 (172) total: 346ms remaining: 1.65s 173: learn: 0.4598999 test: 0.4904549 best: 0.4904549 (173) total: 348ms remaining: 1.65s 174: learn: 0.4594232 test: 0.4901642 best: 0.4901642 (174) total: 349ms remaining: 1.65s 175: learn: 0.4590624 test: 0.4899771 best: 0.4899771 (175) total: 351ms remaining: 1.64s 176: learn: 0.4587828 test: 0.4899488 best: 0.4899488 (176) total: 353ms remaining: 1.64s 177: learn: 0.4583735 test: 0.4898755 best: 0.4898755 (177) total: 354ms remaining: 1.63s 178: learn: 0.4578922 test: 0.4896325 best: 0.4896325 (178) total: 356ms remaining: 1.63s 179: learn: 0.4573663 test: 0.4892950 best: 0.4892950 (179) total: 358ms remaining: 1.63s 180: learn: 0.4569801 test: 0.4890520 best: 0.4890520 (180) total: 360ms remaining: 1.63s 181: learn: 0.4566363 test: 0.4890679 best: 0.4890520 (180) total: 362ms remaining: 1.63s 182: learn: 0.4563096 test: 0.4889217 best: 0.4889217 (182) total: 364ms remaining: 1.63s 183: learn: 0.4558515 test: 0.4887227 best: 0.4887227 (183) total: 366ms remaining: 1.62s 184: learn: 0.4553731 test: 0.4883675 best: 0.4883675 (184) total: 368ms remaining: 1.62s 185: learn: 0.4547885 test: 0.4879452 best: 0.4879452 (185) total: 371ms remaining: 1.62s 186: learn: 0.4543931 test: 0.4877658 best: 0.4877658 (186) total: 372ms remaining: 1.62s 187: learn: 0.4540999 test: 0.4877788 best: 0.4877658 (186) total: 374ms remaining: 1.62s 188: learn: 0.4537886 test: 0.4877047 best: 0.4877047 (188) total: 377ms remaining: 1.61s 189: learn: 0.4531642 test: 0.4874126 best: 0.4874126 (189) total: 379ms remaining: 1.61s 190: learn: 0.4528835 test: 0.4871667 best: 0.4871667 (190) total: 381ms remaining: 1.61s 191: learn: 0.4523681 test: 0.4867513 best: 0.4867513 (191) total: 385ms remaining: 1.62s 192: learn: 0.4519749 test: 0.4868370 best: 0.4867513 (191) total: 388ms remaining: 1.62s 193: learn: 0.4514928 test: 0.4863694 best: 0.4863694 (193) total: 390ms remaining: 1.62s 194: learn: 0.4511564 test: 0.4863953 best: 0.4863694 (193) total: 391ms remaining: 1.61s 195: learn: 0.4506815 test: 0.4862452 best: 0.4862452 (195) total: 393ms remaining: 1.61s 196: learn: 0.4504594 test: 0.4862511 best: 0.4862452 (195) total: 395ms remaining: 1.61s 197: learn: 0.4500881 test: 0.4861145 best: 0.4861145 (197) total: 397ms remaining: 1.61s 198: learn: 0.4496473 test: 0.4859879 best: 0.4859879 (198) total: 399ms remaining: 1.6s 199: learn: 0.4492486 test: 0.4857914 best: 0.4857914 (199) total: 400ms remaining: 1.6s 200: learn: 0.4486601 test: 0.4855720 best: 0.4855720 (200) total: 402ms remaining: 1.6s 201: learn: 0.4483826 test: 0.4855942 best: 0.4855720 (200) total: 404ms remaining: 1.59s 202: learn: 0.4474747 test: 0.4847118 best: 0.4847118 (202) total: 406ms remaining: 1.59s 203: learn: 0.4472811 test: 0.4845968 best: 0.4845968 (203) total: 407ms remaining: 1.59s 204: learn: 0.4469377 test: 0.4846119 best: 0.4845968 (203) total: 409ms remaining: 1.59s 205: learn: 0.4466374 test: 0.4845243 best: 0.4845243 (205) total: 411ms remaining: 1.58s 206: learn: 0.4462238 test: 0.4843668 best: 0.4843668 (206) total: 413ms remaining: 1.58s 207: learn: 0.4459811 test: 0.4842313 best: 0.4842313 (207) total: 414ms remaining: 1.58s 208: learn: 0.4453497 test: 0.4837868 best: 0.4837868 (208) total: 416ms remaining: 1.57s 209: learn: 0.4449249 test: 0.4836808 best: 0.4836808 (209) total: 418ms remaining: 1.57s 210: learn: 0.4444954 test: 0.4834605 best: 0.4834605 (210) total: 420ms remaining: 1.57s 211: learn: 0.4439879 test: 0.4829294 best: 0.4829294 (211) total: 422ms remaining: 1.57s 212: learn: 0.4435927 test: 0.4827132 best: 0.4827132 (212) total: 424ms remaining: 1.56s 213: learn: 0.4431466 test: 0.4823570 best: 0.4823570 (213) total: 425ms remaining: 1.56s 214: learn: 0.4426976 test: 0.4820976 best: 0.4820976 (214) total: 427ms remaining: 1.56s 215: learn: 0.4423695 test: 0.4818988 best: 0.4818988 (215) total: 429ms remaining: 1.56s 216: learn: 0.4419610 test: 0.4816709 best: 0.4816709 (216) total: 430ms remaining: 1.55s 217: learn: 0.4416160 test: 0.4815078 best: 0.4815078 (217) total: 432ms remaining: 1.55s 218: learn: 0.4412127 test: 0.4813667 best: 0.4813667 (218) total: 434ms remaining: 1.55s 219: learn: 0.4409583 test: 0.4812617 best: 0.4812617 (219) total: 435ms remaining: 1.54s 220: learn: 0.4406064 test: 0.4812920 best: 0.4812617 (219) total: 437ms remaining: 1.54s 221: learn: 0.4398048 test: 0.4808083 best: 0.4808083 (221) total: 439ms remaining: 1.54s 222: learn: 0.4395637 test: 0.4807484 best: 0.4807484 (222) total: 441ms remaining: 1.54s 223: learn: 0.4393006 test: 0.4805079 best: 0.4805079 (223) total: 443ms remaining: 1.53s 224: learn: 0.4390188 test: 0.4805481 best: 0.4805079 (223) total: 444ms remaining: 1.53s 225: learn: 0.4385645 test: 0.4805161 best: 0.4805079 (223) total: 446ms remaining: 1.53s 226: learn: 0.4383164 test: 0.4804533 best: 0.4804533 (226) total: 448ms remaining: 1.52s 227: learn: 0.4377002 test: 0.4800820 best: 0.4800820 (227) total: 450ms remaining: 1.52s 228: learn: 0.4373380 test: 0.4798304 best: 0.4798304 (228) total: 451ms remaining: 1.52s 229: learn: 0.4368268 test: 0.4794870 best: 0.4794870 (229) total: 453ms remaining: 1.52s 230: learn: 0.4363041 test: 0.4790164 best: 0.4790164 (230) total: 455ms remaining: 1.51s 231: learn: 0.4358161 test: 0.4789560 best: 0.4789560 (231) total: 456ms remaining: 1.51s 232: learn: 0.4354018 test: 0.4788120 best: 0.4788120 (232) total: 458ms remaining: 1.51s 233: learn: 0.4350956 test: 0.4787588 best: 0.4787588 (233) total: 460ms remaining: 1.5s 234: learn: 0.4347140 test: 0.4786686 best: 0.4786686 (234) total: 462ms remaining: 1.5s 235: learn: 0.4343750 test: 0.4784357 best: 0.4784357 (235) total: 464ms remaining: 1.5s 236: learn: 0.4340305 test: 0.4782305 best: 0.4782305 (236) total: 465ms remaining: 1.5s 237: learn: 0.4338106 test: 0.4781313 best: 0.4781313 (237) total: 467ms remaining: 1.5s 238: learn: 0.4335269 test: 0.4780149 best: 0.4780149 (238) total: 469ms remaining: 1.49s 239: learn: 0.4332576 test: 0.4779955 best: 0.4779955 (239) total: 471ms remaining: 1.49s 240: learn: 0.4328172 test: 0.4777712 best: 0.4777712 (240) total: 472ms remaining: 1.49s 241: learn: 0.4325275 test: 0.4776458 best: 0.4776458 (241) total: 474ms remaining: 1.48s 242: learn: 0.4320691 test: 0.4774999 best: 0.4774999 (242) total: 476ms remaining: 1.48s 243: learn: 0.4318501 test: 0.4774783 best: 0.4774783 (243) total: 477ms remaining: 1.48s 244: learn: 0.4312636 test: 0.4770291 best: 0.4770291 (244) total: 479ms remaining: 1.48s 245: learn: 0.4309035 test: 0.4768353 best: 0.4768353 (245) total: 481ms remaining: 1.47s 246: learn: 0.4306592 test: 0.4766473 best: 0.4766473 (246) total: 483ms remaining: 1.47s 247: learn: 0.4302798 test: 0.4765306 best: 0.4765306 (247) total: 488ms remaining: 1.48s 248: learn: 0.4300155 test: 0.4763662 best: 0.4763662 (248) total: 491ms remaining: 1.48s 249: learn: 0.4297146 test: 0.4762287 best: 0.4762287 (249) total: 493ms remaining: 1.48s 250: learn: 0.4294499 test: 0.4760763 best: 0.4760763 (250) total: 495ms remaining: 1.48s 251: learn: 0.4291395 test: 0.4759902 best: 0.4759902 (251) total: 496ms remaining: 1.47s 252: learn: 0.4287019 test: 0.4756735 best: 0.4756735 (252) total: 498ms remaining: 1.47s 253: learn: 0.4282793 test: 0.4755237 best: 0.4755237 (253) total: 500ms remaining: 1.47s 254: learn: 0.4279875 test: 0.4754137 best: 0.4754137 (254) total: 501ms remaining: 1.47s 255: learn: 0.4277887 test: 0.4754504 best: 0.4754137 (254) total: 503ms remaining: 1.46s 256: learn: 0.4275117 test: 0.4755141 best: 0.4754137 (254) total: 506ms remaining: 1.46s 257: learn: 0.4268716 test: 0.4750086 best: 0.4750086 (257) total: 508ms remaining: 1.46s 258: learn: 0.4264238 test: 0.4746981 best: 0.4746981 (258) total: 509ms remaining: 1.46s 259: learn: 0.4260910 test: 0.4747560 best: 0.4746981 (258) total: 511ms remaining: 1.45s 260: learn: 0.4257544 test: 0.4746210 best: 0.4746210 (260) total: 513ms remaining: 1.45s 261: learn: 0.4254834 test: 0.4744198 best: 0.4744198 (261) total: 515ms remaining: 1.45s 262: learn: 0.4252464 test: 0.4744585 best: 0.4744198 (261) total: 516ms remaining: 1.45s 263: learn: 0.4249110 test: 0.4745085 best: 0.4744198 (261) total: 518ms remaining: 1.44s 264: learn: 0.4246863 test: 0.4744738 best: 0.4744198 (261) total: 520ms remaining: 1.44s 265: learn: 0.4244930 test: 0.4744139 best: 0.4744139 (265) total: 522ms remaining: 1.44s 266: learn: 0.4242463 test: 0.4743476 best: 0.4743476 (266) total: 524ms remaining: 1.44s 267: learn: 0.4240063 test: 0.4744264 best: 0.4743476 (266) total: 525ms remaining: 1.43s 268: learn: 0.4236866 test: 0.4743115 best: 0.4743115 (268) total: 527ms remaining: 1.43s 269: learn: 0.4234283 test: 0.4742923 best: 0.4742923 (269) total: 529ms remaining: 1.43s 270: learn: 0.4231463 test: 0.4742903 best: 0.4742903 (270) total: 531ms remaining: 1.43s 271: learn: 0.4227358 test: 0.4740890 best: 0.4740890 (271) total: 533ms remaining: 1.43s 272: learn: 0.4224702 test: 0.4738246 best: 0.4738246 (272) total: 534ms remaining: 1.42s 273: learn: 0.4220762 test: 0.4736398 best: 0.4736398 (273) total: 536ms remaining: 1.42s 274: learn: 0.4218335 test: 0.4737953 best: 0.4736398 (273) total: 538ms remaining: 1.42s 275: learn: 0.4216472 test: 0.4738377 best: 0.4736398 (273) total: 540ms remaining: 1.42s 276: learn: 0.4213290 test: 0.4736880 best: 0.4736398 (273) total: 541ms remaining: 1.41s 277: learn: 0.4211214 test: 0.4736644 best: 0.4736398 (273) total: 543ms remaining: 1.41s 278: learn: 0.4205820 test: 0.4732220 best: 0.4732220 (278) total: 545ms remaining: 1.41s 279: learn: 0.4202530 test: 0.4728266 best: 0.4728266 (279) total: 547ms remaining: 1.41s 280: learn: 0.4200279 test: 0.4728685 best: 0.4728266 (279) total: 549ms remaining: 1.4s 281: learn: 0.4198252 test: 0.4729484 best: 0.4728266 (279) total: 552ms remaining: 1.4s 282: learn: 0.4195928 test: 0.4729503 best: 0.4728266 (279) total: 554ms remaining: 1.4s 283: learn: 0.4193946 test: 0.4729573 best: 0.4728266 (279) total: 556ms remaining: 1.4s 284: learn: 0.4190762 test: 0.4728307 best: 0.4728266 (279) total: 558ms remaining: 1.4s 285: learn: 0.4187869 test: 0.4724905 best: 0.4724905 (285) total: 560ms remaining: 1.4s 286: learn: 0.4184980 test: 0.4723907 best: 0.4723907 (286) total: 562ms remaining: 1.4s 287: learn: 0.4182807 test: 0.4722431 best: 0.4722431 (287) total: 564ms remaining: 1.39s 288: learn: 0.4179935 test: 0.4721274 best: 0.4721274 (288) total: 567ms remaining: 1.39s 289: learn: 0.4177648 test: 0.4719542 best: 0.4719542 (289) total: 569ms remaining: 1.39s 290: learn: 0.4175624 test: 0.4717834 best: 0.4717834 (290) total: 571ms remaining: 1.39s 291: learn: 0.4172253 test: 0.4716238 best: 0.4716238 (291) total: 574ms remaining: 1.39s 292: learn: 0.4170039 test: 0.4715688 best: 0.4715688 (292) total: 577ms remaining: 1.39s 293: learn: 0.4167356 test: 0.4713133 best: 0.4713133 (293) total: 580ms remaining: 1.39s 294: learn: 0.4164855 test: 0.4711828 best: 0.4711828 (294) total: 583ms remaining: 1.39s 295: learn: 0.4161872 test: 0.4710400 best: 0.4710400 (295) total: 584ms remaining: 1.39s 296: learn: 0.4159124 test: 0.4709237 best: 0.4709237 (296) total: 586ms remaining: 1.39s 297: learn: 0.4157120 test: 0.4708394 best: 0.4708394 (297) total: 588ms remaining: 1.39s 298: learn: 0.4153938 test: 0.4707661 best: 0.4707661 (298) total: 590ms remaining: 1.38s 299: learn: 0.4151147 test: 0.4706647 best: 0.4706647 (299) total: 592ms remaining: 1.38s 300: learn: 0.4147062 test: 0.4702998 best: 0.4702998 (300) total: 594ms remaining: 1.38s 301: learn: 0.4145502 test: 0.4702750 best: 0.4702750 (301) total: 597ms remaining: 1.38s 302: learn: 0.4142955 test: 0.4700363 best: 0.4700363 (302) total: 599ms remaining: 1.38s 303: learn: 0.4140141 test: 0.4698707 best: 0.4698707 (303) total: 600ms remaining: 1.37s 304: learn: 0.4137539 test: 0.4698926 best: 0.4698707 (303) total: 602ms remaining: 1.37s 305: learn: 0.4135894 test: 0.4699054 best: 0.4698707 (303) total: 604ms remaining: 1.37s 306: learn: 0.4133285 test: 0.4697868 best: 0.4697868 (306) total: 606ms remaining: 1.37s 307: learn: 0.4131704 test: 0.4696910 best: 0.4696910 (307) total: 608ms remaining: 1.36s 308: learn: 0.4130150 test: 0.4696316 best: 0.4696316 (308) total: 610ms remaining: 1.36s 309: learn: 0.4128083 test: 0.4695344 best: 0.4695344 (309) total: 612ms remaining: 1.36s 310: learn: 0.4124519 test: 0.4694964 best: 0.4694964 (310) total: 614ms remaining: 1.36s 311: learn: 0.4120934 test: 0.4691967 best: 0.4691967 (311) total: 615ms remaining: 1.36s 312: learn: 0.4117543 test: 0.4692920 best: 0.4691967 (311) total: 618ms remaining: 1.35s 313: learn: 0.4115881 test: 0.4692497 best: 0.4691967 (311) total: 619ms remaining: 1.35s 314: learn: 0.4113500 test: 0.4691328 best: 0.4691328 (314) total: 621ms remaining: 1.35s 315: learn: 0.4112205 test: 0.4691441 best: 0.4691328 (314) total: 622ms remaining: 1.35s 316: learn: 0.4108105 test: 0.4686581 best: 0.4686581 (316) total: 625ms remaining: 1.35s 317: learn: 0.4104059 test: 0.4683776 best: 0.4683776 (317) total: 627ms remaining: 1.34s 318: learn: 0.4099906 test: 0.4682019 best: 0.4682019 (318) total: 628ms remaining: 1.34s 319: learn: 0.4095554 test: 0.4678216 best: 0.4678216 (319) total: 630ms remaining: 1.34s 320: learn: 0.4092459 test: 0.4677190 best: 0.4677190 (320) total: 633ms remaining: 1.34s 321: learn: 0.4089935 test: 0.4676817 best: 0.4676817 (321) total: 634ms remaining: 1.33s 322: learn: 0.4087872 test: 0.4676981 best: 0.4676817 (321) total: 636ms remaining: 1.33s 323: learn: 0.4086636 test: 0.4677253 best: 0.4676817 (321) total: 638ms remaining: 1.33s 324: learn: 0.4084079 test: 0.4676140 best: 0.4676140 (324) total: 640ms remaining: 1.33s 325: learn: 0.4080422 test: 0.4675048 best: 0.4675048 (325) total: 642ms remaining: 1.33s 326: learn: 0.4077018 test: 0.4673581 best: 0.4673581 (326) total: 644ms remaining: 1.32s 327: learn: 0.4074540 test: 0.4671777 best: 0.4671777 (327) total: 645ms remaining: 1.32s 328: learn: 0.4071613 test: 0.4668262 best: 0.4668262 (328) total: 648ms remaining: 1.32s 329: learn: 0.4067793 test: 0.4666068 best: 0.4666068 (329) total: 649ms remaining: 1.32s 330: learn: 0.4065947 test: 0.4665650 best: 0.4665650 (330) total: 651ms remaining: 1.31s 331: learn: 0.4063854 test: 0.4664972 best: 0.4664972 (331) total: 652ms remaining: 1.31s 332: learn: 0.4062471 test: 0.4664662 best: 0.4664662 (332) total: 655ms remaining: 1.31s 333: learn: 0.4059458 test: 0.4661934 best: 0.4661934 (333) total: 657ms remaining: 1.31s 334: learn: 0.4056861 test: 0.4661362 best: 0.4661362 (334) total: 658ms remaining: 1.31s 335: learn: 0.4054306 test: 0.4661320 best: 0.4661320 (335) total: 660ms remaining: 1.3s 336: learn: 0.4051850 test: 0.4660401 best: 0.4660401 (336) total: 662ms remaining: 1.3s 337: learn: 0.4049020 test: 0.4658359 best: 0.4658359 (337) total: 664ms remaining: 1.3s 338: learn: 0.4046978 test: 0.4656967 best: 0.4656967 (338) total: 666ms remaining: 1.3s 339: learn: 0.4043855 test: 0.4656253 best: 0.4656253 (339) total: 667ms remaining: 1.29s 340: learn: 0.4041783 test: 0.4656035 best: 0.4656035 (340) total: 669ms remaining: 1.29s 341: learn: 0.4039756 test: 0.4657138 best: 0.4656035 (340) total: 671ms remaining: 1.29s 342: learn: 0.4037496 test: 0.4656271 best: 0.4656035 (340) total: 673ms remaining: 1.29s 343: learn: 0.4035459 test: 0.4654073 best: 0.4654073 (343) total: 674ms remaining: 1.28s 344: learn: 0.4033576 test: 0.4653693 best: 0.4653693 (344) total: 676ms remaining: 1.28s 345: learn: 0.4030596 test: 0.4651069 best: 0.4651069 (345) total: 678ms remaining: 1.28s 346: learn: 0.4028326 test: 0.4649950 best: 0.4649950 (346) total: 680ms remaining: 1.28s 347: learn: 0.4026203 test: 0.4649868 best: 0.4649868 (347) total: 681ms remaining: 1.28s 348: learn: 0.4023212 test: 0.4649225 best: 0.4649225 (348) total: 684ms remaining: 1.27s 349: learn: 0.4020260 test: 0.4648777 best: 0.4648777 (349) total: 686ms remaining: 1.27s 350: learn: 0.4017638 test: 0.4650331 best: 0.4648777 (349) total: 687ms remaining: 1.27s 351: learn: 0.4015640 test: 0.4648929 best: 0.4648777 (349) total: 689ms remaining: 1.27s 352: learn: 0.4013439 test: 0.4650321 best: 0.4648777 (349) total: 691ms remaining: 1.27s 353: learn: 0.4012224 test: 0.4651262 best: 0.4648777 (349) total: 693ms remaining: 1.26s 354: learn: 0.4009741 test: 0.4649815 best: 0.4648777 (349) total: 694ms remaining: 1.26s 355: learn: 0.4008433 test: 0.4648659 best: 0.4648659 (355) total: 696ms remaining: 1.26s 356: learn: 0.4006373 test: 0.4647753 best: 0.4647753 (356) total: 698ms remaining: 1.26s 357: learn: 0.4004349 test: 0.4646394 best: 0.4646394 (357) total: 700ms remaining: 1.25s 358: learn: 0.4000786 test: 0.4645259 best: 0.4645259 (358) total: 702ms remaining: 1.25s 359: learn: 0.3999053 test: 0.4644311 best: 0.4644311 (359) total: 704ms remaining: 1.25s 360: learn: 0.3997862 test: 0.4644178 best: 0.4644178 (360) total: 706ms remaining: 1.25s 361: learn: 0.3995012 test: 0.4643126 best: 0.4643126 (361) total: 708ms remaining: 1.25s 362: learn: 0.3993160 test: 0.4642327 best: 0.4642327 (362) total: 710ms remaining: 1.25s 363: learn: 0.3989545 test: 0.4640093 best: 0.4640093 (363) total: 711ms remaining: 1.24s 364: learn: 0.3988178 test: 0.4640168 best: 0.4640093 (363) total: 714ms remaining: 1.24s 365: learn: 0.3985280 test: 0.4639499 best: 0.4639499 (365) total: 715ms remaining: 1.24s 366: learn: 0.3982459 test: 0.4638520 best: 0.4638520 (366) total: 717ms remaining: 1.24s 367: learn: 0.3980047 test: 0.4638432 best: 0.4638432 (367) total: 719ms remaining: 1.23s 368: learn: 0.3977876 test: 0.4638098 best: 0.4638098 (368) total: 721ms remaining: 1.23s 369: learn: 0.3976526 test: 0.4637376 best: 0.4637376 (369) total: 723ms remaining: 1.23s 370: learn: 0.3974839 test: 0.4636538 best: 0.4636538 (370) total: 724ms remaining: 1.23s 371: learn: 0.3972668 test: 0.4635931 best: 0.4635931 (371) total: 726ms remaining: 1.23s 372: learn: 0.3971393 test: 0.4636377 best: 0.4635931 (371) total: 728ms remaining: 1.22s 373: learn: 0.3969227 test: 0.4635259 best: 0.4635259 (373) total: 731ms remaining: 1.22s 374: learn: 0.3967487 test: 0.4634886 best: 0.4634886 (374) total: 733ms remaining: 1.22s 375: learn: 0.3965598 test: 0.4633903 best: 0.4633903 (375) total: 736ms remaining: 1.22s 376: learn: 0.3962595 test: 0.4633580 best: 0.4633580 (376) total: 737ms remaining: 1.22s 377: learn: 0.3959874 test: 0.4633210 best: 0.4633210 (377) total: 739ms remaining: 1.22s 378: learn: 0.3958285 test: 0.4631592 best: 0.4631592 (378) total: 741ms remaining: 1.21s 379: learn: 0.3956846 test: 0.4630112 best: 0.4630112 (379) total: 743ms remaining: 1.21s 380: learn: 0.3954136 test: 0.4630881 best: 0.4630112 (379) total: 745ms remaining: 1.21s 381: learn: 0.3950980 test: 0.4628841 best: 0.4628841 (381) total: 747ms remaining: 1.21s 382: learn: 0.3948906 test: 0.4628020 best: 0.4628020 (382) total: 749ms remaining: 1.21s 383: learn: 0.3946293 test: 0.4626498 best: 0.4626498 (383) total: 751ms remaining: 1.2s 384: learn: 0.3944728 test: 0.4626887 best: 0.4626498 (383) total: 753ms remaining: 1.2s 385: learn: 0.3943120 test: 0.4627094 best: 0.4626498 (383) total: 754ms remaining: 1.2s 386: learn: 0.3941253 test: 0.4627980 best: 0.4626498 (383) total: 756ms remaining: 1.2s 387: learn: 0.3938836 test: 0.4629803 best: 0.4626498 (383) total: 758ms remaining: 1.2s 388: learn: 0.3936447 test: 0.4627845 best: 0.4626498 (383) total: 760ms remaining: 1.19s 389: learn: 0.3934151 test: 0.4627236 best: 0.4626498 (383) total: 762ms remaining: 1.19s 390: learn: 0.3931970 test: 0.4626682 best: 0.4626498 (383) total: 764ms remaining: 1.19s 391: learn: 0.3930434 test: 0.4625839 best: 0.4625839 (391) total: 766ms remaining: 1.19s 392: learn: 0.3928506 test: 0.4625670 best: 0.4625670 (392) total: 768ms remaining: 1.19s 393: learn: 0.3926837 test: 0.4624799 best: 0.4624799 (393) total: 771ms remaining: 1.19s 394: learn: 0.3924173 test: 0.4624548 best: 0.4624548 (394) total: 774ms remaining: 1.19s 395: learn: 0.3922466 test: 0.4624784 best: 0.4624548 (394) total: 776ms remaining: 1.18s 396: learn: 0.3919847 test: 0.4623537 best: 0.4623537 (396) total: 778ms remaining: 1.18s 397: learn: 0.3918224 test: 0.4624311 best: 0.4623537 (396) total: 780ms remaining: 1.18s 398: learn: 0.3916190 test: 0.4623019 best: 0.4623019 (398) total: 782ms remaining: 1.18s 399: learn: 0.3913157 test: 0.4623466 best: 0.4623019 (398) total: 784ms remaining: 1.18s 400: learn: 0.3911075 test: 0.4622400 best: 0.4622400 (400) total: 786ms remaining: 1.17s 401: learn: 0.3909517 test: 0.4622102 best: 0.4622102 (401) total: 788ms remaining: 1.17s 402: learn: 0.3907373 test: 0.4621944 best: 0.4621944 (402) total: 790ms remaining: 1.17s 403: learn: 0.3905042 test: 0.4621022 best: 0.4621022 (403) total: 792ms remaining: 1.17s 404: learn: 0.3903988 test: 0.4620897 best: 0.4620897 (404) total: 795ms remaining: 1.17s 405: learn: 0.3901936 test: 0.4620538 best: 0.4620538 (405) total: 796ms remaining: 1.16s 406: learn: 0.3899798 test: 0.4619438 best: 0.4619438 (406) total: 798ms remaining: 1.16s 407: learn: 0.3897263 test: 0.4620154 best: 0.4619438 (406) total: 800ms remaining: 1.16s 408: learn: 0.3894834 test: 0.4617614 best: 0.4617614 (408) total: 802ms remaining: 1.16s 409: learn: 0.3892403 test: 0.4617804 best: 0.4617614 (408) total: 804ms remaining: 1.16s 410: learn: 0.3890856 test: 0.4617770 best: 0.4617614 (408) total: 805ms remaining: 1.15s 411: learn: 0.3889069 test: 0.4617123 best: 0.4617123 (411) total: 808ms remaining: 1.15s 412: learn: 0.3887944 test: 0.4617345 best: 0.4617123 (411) total: 809ms remaining: 1.15s 413: learn: 0.3885406 test: 0.4617309 best: 0.4617123 (411) total: 811ms remaining: 1.15s 414: learn: 0.3883321 test: 0.4616899 best: 0.4616899 (414) total: 813ms remaining: 1.15s 415: learn: 0.3881393 test: 0.4616392 best: 0.4616392 (415) total: 815ms remaining: 1.14s 416: learn: 0.3879969 test: 0.4616921 best: 0.4616392 (415) total: 817ms remaining: 1.14s 417: learn: 0.3879150 test: 0.4616461 best: 0.4616392 (415) total: 819ms remaining: 1.14s 418: learn: 0.3876392 test: 0.4616471 best: 0.4616392 (415) total: 820ms remaining: 1.14s 419: learn: 0.3874439 test: 0.4615402 best: 0.4615402 (419) total: 823ms remaining: 1.14s 420: learn: 0.3870439 test: 0.4613441 best: 0.4613441 (420) total: 825ms remaining: 1.13s 421: learn: 0.3868033 test: 0.4611799 best: 0.4611799 (421) total: 826ms remaining: 1.13s 422: learn: 0.3865067 test: 0.4610684 best: 0.4610684 (422) total: 828ms remaining: 1.13s 423: learn: 0.3863048 test: 0.4610296 best: 0.4610296 (423) total: 830ms remaining: 1.13s 424: learn: 0.3861001 test: 0.4608806 best: 0.4608806 (424) total: 832ms remaining: 1.13s 425: learn: 0.3859036 test: 0.4607350 best: 0.4607350 (425) total: 834ms remaining: 1.12s 426: learn: 0.3857714 test: 0.4606924 best: 0.4606924 (426) total: 835ms remaining: 1.12s 427: learn: 0.3855804 test: 0.4606313 best: 0.4606313 (427) total: 838ms remaining: 1.12s 428: learn: 0.3853068 test: 0.4605384 best: 0.4605384 (428) total: 840ms remaining: 1.12s 429: learn: 0.3851964 test: 0.4605498 best: 0.4605384 (428) total: 841ms remaining: 1.11s 430: learn: 0.3849383 test: 0.4604930 best: 0.4604930 (430) total: 843ms remaining: 1.11s 431: learn: 0.3847079 test: 0.4604588 best: 0.4604588 (431) total: 845ms remaining: 1.11s 432: learn: 0.3845392 test: 0.4602797 best: 0.4602797 (432) total: 847ms remaining: 1.11s 433: learn: 0.3843821 test: 0.4602275 best: 0.4602275 (433) total: 848ms remaining: 1.11s 434: learn: 0.3841973 test: 0.4602417 best: 0.4602275 (433) total: 850ms remaining: 1.1s 435: learn: 0.3840661 test: 0.4602228 best: 0.4602228 (435) total: 852ms remaining: 1.1s 436: learn: 0.3839446 test: 0.4602341 best: 0.4602228 (435) total: 854ms remaining: 1.1s 437: learn: 0.3838096 test: 0.4602069 best: 0.4602069 (437) total: 856ms remaining: 1.1s 438: learn: 0.3835484 test: 0.4599853 best: 0.4599853 (438) total: 857ms remaining: 1.09s 439: learn: 0.3831853 test: 0.4597335 best: 0.4597335 (439) total: 859ms remaining: 1.09s 440: learn: 0.3828886 test: 0.4596162 best: 0.4596162 (440) total: 862ms remaining: 1.09s 441: learn: 0.3827541 test: 0.4596029 best: 0.4596029 (441) total: 863ms remaining: 1.09s 442: learn: 0.3823087 test: 0.4594325 best: 0.4594325 (442) total: 865ms remaining: 1.09s 443: learn: 0.3820628 test: 0.4592120 best: 0.4592120 (443) total: 867ms remaining: 1.09s 444: learn: 0.3818454 test: 0.4591605 best: 0.4591605 (444) total: 869ms remaining: 1.08s 445: learn: 0.3816425 test: 0.4591494 best: 0.4591494 (445) total: 871ms remaining: 1.08s 446: learn: 0.3815131 test: 0.4591791 best: 0.4591494 (445) total: 873ms remaining: 1.08s 447: learn: 0.3813370 test: 0.4590624 best: 0.4590624 (447) total: 875ms remaining: 1.08s 448: learn: 0.3810517 test: 0.4591419 best: 0.4590624 (447) total: 877ms remaining: 1.07s 449: learn: 0.3808885 test: 0.4590666 best: 0.4590624 (447) total: 878ms remaining: 1.07s 450: learn: 0.3807437 test: 0.4590183 best: 0.4590183 (450) total: 880ms remaining: 1.07s 451: learn: 0.3804929 test: 0.4589770 best: 0.4589770 (451) total: 882ms remaining: 1.07s 452: learn: 0.3803460 test: 0.4589075 best: 0.4589075 (452) total: 884ms remaining: 1.07s 453: learn: 0.3801472 test: 0.4588455 best: 0.4588455 (453) total: 886ms remaining: 1.06s 454: learn: 0.3799855 test: 0.4588497 best: 0.4588455 (453) total: 888ms remaining: 1.06s 455: learn: 0.3798194 test: 0.4587796 best: 0.4587796 (455) total: 890ms remaining: 1.06s 456: learn: 0.3796637 test: 0.4587287 best: 0.4587287 (456) total: 891ms remaining: 1.06s 457: learn: 0.3794175 test: 0.4587313 best: 0.4587287 (456) total: 893ms remaining: 1.06s 458: learn: 0.3792524 test: 0.4586370 best: 0.4586370 (458) total: 895ms remaining: 1.05s 459: learn: 0.3791000 test: 0.4585296 best: 0.4585296 (459) total: 897ms remaining: 1.05s 460: learn: 0.3789550 test: 0.4584329 best: 0.4584329 (460) total: 899ms remaining: 1.05s 461: learn: 0.3787978 test: 0.4585463 best: 0.4584329 (460) total: 901ms remaining: 1.05s 462: learn: 0.3787007 test: 0.4585236 best: 0.4584329 (460) total: 903ms remaining: 1.05s 463: learn: 0.3783253 test: 0.4582361 best: 0.4582361 (463) total: 906ms remaining: 1.05s 464: learn: 0.3781044 test: 0.4581071 best: 0.4581071 (464) total: 908ms remaining: 1.04s 465: learn: 0.3779858 test: 0.4580826 best: 0.4580826 (465) total: 911ms remaining: 1.04s 466: learn: 0.3777721 test: 0.4578111 best: 0.4578111 (466) total: 914ms remaining: 1.04s 467: learn: 0.3776498 test: 0.4579274 best: 0.4578111 (466) total: 916ms remaining: 1.04s 468: learn: 0.3774224 test: 0.4578371 best: 0.4578111 (466) total: 918ms remaining: 1.04s 469: learn: 0.3772571 test: 0.4579221 best: 0.4578111 (466) total: 920ms remaining: 1.04s 470: learn: 0.3770893 test: 0.4577775 best: 0.4577775 (470) total: 922ms remaining: 1.03s 471: learn: 0.3768787 test: 0.4578221 best: 0.4577775 (470) total: 925ms remaining: 1.03s 472: learn: 0.3766489 test: 0.4578429 best: 0.4577775 (470) total: 927ms remaining: 1.03s 473: learn: 0.3765176 test: 0.4578355 best: 0.4577775 (470) total: 930ms remaining: 1.03s 474: learn: 0.3763470 test: 0.4577627 best: 0.4577627 (474) total: 934ms remaining: 1.03s 475: learn: 0.3761178 test: 0.4576394 best: 0.4576394 (475) total: 936ms remaining: 1.03s 476: learn: 0.3759527 test: 0.4575430 best: 0.4575430 (476) total: 941ms remaining: 1.03s 477: learn: 0.3757806 test: 0.4575766 best: 0.4575430 (476) total: 943ms remaining: 1.03s 478: learn: 0.3755504 test: 0.4575121 best: 0.4575121 (478) total: 946ms remaining: 1.03s 479: learn: 0.3754086 test: 0.4574586 best: 0.4574586 (479) total: 949ms remaining: 1.03s 480: learn: 0.3751915 test: 0.4574086 best: 0.4574086 (480) total: 952ms remaining: 1.03s 481: learn: 0.3750591 test: 0.4572873 best: 0.4572873 (481) total: 954ms remaining: 1.02s 482: learn: 0.3749035 test: 0.4572108 best: 0.4572108 (482) total: 958ms remaining: 1.02s 483: learn: 0.3747040 test: 0.4570471 best: 0.4570471 (483) total: 964ms remaining: 1.03s 484: learn: 0.3745406 test: 0.4569358 best: 0.4569358 (484) total: 969ms remaining: 1.03s 485: learn: 0.3742219 test: 0.4566797 best: 0.4566797 (485) total: 972ms remaining: 1.03s 486: learn: 0.3740671 test: 0.4566683 best: 0.4566683 (486) total: 976ms remaining: 1.03s 487: learn: 0.3738948 test: 0.4566546 best: 0.4566546 (487) total: 979ms remaining: 1.03s 488: learn: 0.3736109 test: 0.4564869 best: 0.4564869 (488) total: 981ms remaining: 1.02s 489: learn: 0.3734664 test: 0.4563811 best: 0.4563811 (489) total: 984ms remaining: 1.02s 490: learn: 0.3733291 test: 0.4563624 best: 0.4563624 (490) total: 988ms remaining: 1.02s 491: learn: 0.3730900 test: 0.4563834 best: 0.4563624 (490) total: 990ms remaining: 1.02s 492: learn: 0.3729676 test: 0.4564172 best: 0.4563624 (490) total: 993ms remaining: 1.02s 493: learn: 0.3728750 test: 0.4564134 best: 0.4563624 (490) total: 996ms remaining: 1.02s 494: learn: 0.3726553 test: 0.4564480 best: 0.4563624 (490) total: 999ms remaining: 1.02s 495: learn: 0.3724661 test: 0.4564110 best: 0.4563624 (490) total: 1s remaining: 1.02s 496: learn: 0.3723685 test: 0.4564254 best: 0.4563624 (490) total: 1s remaining: 1.02s 497: learn: 0.3723064 test: 0.4564183 best: 0.4563624 (490) total: 1.01s remaining: 1.01s 498: learn: 0.3720263 test: 0.4562899 best: 0.4562899 (498) total: 1.01s remaining: 1.01s 499: learn: 0.3718582 test: 0.4562228 best: 0.4562228 (499) total: 1.01s remaining: 1.01s 500: learn: 0.3716931 test: 0.4560420 best: 0.4560420 (500) total: 1.01s remaining: 1.01s 501: learn: 0.3715441 test: 0.4560777 best: 0.4560420 (500) total: 1.02s remaining: 1.01s 502: learn: 0.3714461 test: 0.4559992 best: 0.4559992 (502) total: 1.02s remaining: 1.01s 503: learn: 0.3712422 test: 0.4560265 best: 0.4559992 (502) total: 1.02s remaining: 1.01s 504: learn: 0.3710768 test: 0.4559903 best: 0.4559903 (504) total: 1.02s remaining: 1s 505: learn: 0.3709068 test: 0.4559693 best: 0.4559693 (505) total: 1.03s remaining: 1s 506: learn: 0.3708091 test: 0.4559973 best: 0.4559693 (505) total: 1.03s remaining: 1s 507: learn: 0.3707095 test: 0.4560061 best: 0.4559693 (505) total: 1.03s remaining: 999ms 508: learn: 0.3705384 test: 0.4559879 best: 0.4559693 (505) total: 1.03s remaining: 997ms 509: learn: 0.3703723 test: 0.4559505 best: 0.4559505 (509) total: 1.03s remaining: 995ms 510: learn: 0.3702438 test: 0.4559421 best: 0.4559421 (510) total: 1.04s remaining: 994ms 511: learn: 0.3700600 test: 0.4558247 best: 0.4558247 (511) total: 1.04s remaining: 992ms 512: learn: 0.3698845 test: 0.4557016 best: 0.4557016 (512) total: 1.04s remaining: 990ms 513: learn: 0.3696780 test: 0.4557063 best: 0.4557016 (512) total: 1.04s remaining: 988ms 514: learn: 0.3694622 test: 0.4555759 best: 0.4555759 (514) total: 1.05s remaining: 986ms 515: learn: 0.3692288 test: 0.4554281 best: 0.4554281 (515) total: 1.05s remaining: 984ms 516: learn: 0.3689731 test: 0.4553527 best: 0.4553527 (516) total: 1.05s remaining: 982ms 517: learn: 0.3688603 test: 0.4553357 best: 0.4553357 (517) total: 1.05s remaining: 980ms 518: learn: 0.3687584 test: 0.4553931 best: 0.4553357 (517) total: 1.05s remaining: 978ms 519: learn: 0.3685514 test: 0.4552799 best: 0.4552799 (519) total: 1.06s remaining: 975ms 520: learn: 0.3684423 test: 0.4552735 best: 0.4552735 (520) total: 1.06s remaining: 974ms 521: learn: 0.3683104 test: 0.4552393 best: 0.4552393 (521) total: 1.06s remaining: 971ms 522: learn: 0.3681854 test: 0.4552526 best: 0.4552393 (521) total: 1.06s remaining: 969ms 523: learn: 0.3681200 test: 0.4552407 best: 0.4552393 (521) total: 1.06s remaining: 967ms 524: learn: 0.3679412 test: 0.4551991 best: 0.4551991 (524) total: 1.07s remaining: 965ms 525: learn: 0.3677101 test: 0.4550751 best: 0.4550751 (525) total: 1.07s remaining: 963ms 526: learn: 0.3674994 test: 0.4550415 best: 0.4550415 (526) total: 1.07s remaining: 961ms 527: learn: 0.3673062 test: 0.4549686 best: 0.4549686 (527) total: 1.07s remaining: 959ms 528: learn: 0.3671697 test: 0.4549244 best: 0.4549244 (528) total: 1.07s remaining: 957ms 529: learn: 0.3670087 test: 0.4547984 best: 0.4547984 (529) total: 1.08s remaining: 954ms 530: learn: 0.3666946 test: 0.4545358 best: 0.4545358 (530) total: 1.08s remaining: 952ms 531: learn: 0.3665931 test: 0.4545701 best: 0.4545358 (530) total: 1.08s remaining: 950ms 532: learn: 0.3664337 test: 0.4546688 best: 0.4545358 (530) total: 1.08s remaining: 948ms 533: learn: 0.3662919 test: 0.4546133 best: 0.4545358 (530) total: 1.08s remaining: 946ms 534: learn: 0.3661269 test: 0.4545574 best: 0.4545358 (530) total: 1.08s remaining: 944ms 535: learn: 0.3659185 test: 0.4544481 best: 0.4544481 (535) total: 1.09s remaining: 942ms 536: learn: 0.3658256 test: 0.4544465 best: 0.4544465 (536) total: 1.09s remaining: 940ms 537: learn: 0.3656806 test: 0.4544206 best: 0.4544206 (537) total: 1.09s remaining: 937ms 538: learn: 0.3654344 test: 0.4542178 best: 0.4542178 (538) total: 1.09s remaining: 935ms 539: learn: 0.3652770 test: 0.4541737 best: 0.4541737 (539) total: 1.09s remaining: 933ms 540: learn: 0.3651505 test: 0.4542643 best: 0.4541737 (539) total: 1.1s remaining: 931ms 541: learn: 0.3650816 test: 0.4542361 best: 0.4541737 (539) total: 1.1s remaining: 928ms 542: learn: 0.3649708 test: 0.4542362 best: 0.4541737 (539) total: 1.1s remaining: 927ms 543: learn: 0.3648119 test: 0.4543527 best: 0.4541737 (539) total: 1.1s remaining: 926ms 544: learn: 0.3646678 test: 0.4542986 best: 0.4541737 (539) total: 1.11s remaining: 923ms 545: learn: 0.3645568 test: 0.4542985 best: 0.4541737 (539) total: 1.11s remaining: 921ms 546: learn: 0.3643196 test: 0.4540870 best: 0.4540870 (546) total: 1.11s remaining: 919ms 547: learn: 0.3640933 test: 0.4539569 best: 0.4539569 (547) total: 1.11s remaining: 917ms 548: learn: 0.3637642 test: 0.4537709 best: 0.4537709 (548) total: 1.11s remaining: 915ms 549: learn: 0.3636192 test: 0.4538008 best: 0.4537709 (548) total: 1.11s remaining: 913ms 550: learn: 0.3635000 test: 0.4537476 best: 0.4537476 (550) total: 1.12s remaining: 911ms 551: learn: 0.3633704 test: 0.4536799 best: 0.4536799 (551) total: 1.12s remaining: 909ms 552: learn: 0.3632059 test: 0.4535276 best: 0.4535276 (552) total: 1.12s remaining: 906ms 553: learn: 0.3629570 test: 0.4535240 best: 0.4535240 (553) total: 1.12s remaining: 904ms 554: learn: 0.3628264 test: 0.4534812 best: 0.4534812 (554) total: 1.13s remaining: 903ms 555: learn: 0.3627569 test: 0.4534668 best: 0.4534668 (555) total: 1.13s remaining: 900ms 556: learn: 0.3626255 test: 0.4536026 best: 0.4534668 (555) total: 1.13s remaining: 898ms 557: learn: 0.3623966 test: 0.4534064 best: 0.4534064 (557) total: 1.13s remaining: 896ms 558: learn: 0.3622472 test: 0.4533681 best: 0.4533681 (558) total: 1.13s remaining: 894ms 559: learn: 0.3621752 test: 0.4533328 best: 0.4533328 (559) total: 1.14s remaining: 892ms 560: learn: 0.3619818 test: 0.4532139 best: 0.4532139 (560) total: 1.14s remaining: 890ms 561: learn: 0.3618326 test: 0.4531008 best: 0.4531008 (561) total: 1.14s remaining: 888ms 562: learn: 0.3616758 test: 0.4530587 best: 0.4530587 (562) total: 1.14s remaining: 886ms 563: learn: 0.3614447 test: 0.4530438 best: 0.4530438 (563) total: 1.14s remaining: 884ms 564: learn: 0.3613002 test: 0.4530353 best: 0.4530353 (564) total: 1.15s remaining: 882ms 565: learn: 0.3611861 test: 0.4529558 best: 0.4529558 (565) total: 1.15s remaining: 880ms 566: learn: 0.3610359 test: 0.4528964 best: 0.4528964 (566) total: 1.15s remaining: 878ms 567: learn: 0.3608907 test: 0.4529287 best: 0.4528964 (566) total: 1.15s remaining: 876ms 568: learn: 0.3607032 test: 0.4528937 best: 0.4528937 (568) total: 1.15s remaining: 874ms 569: learn: 0.3605887 test: 0.4527526 best: 0.4527526 (569) total: 1.16s remaining: 872ms 570: learn: 0.3604270 test: 0.4526739 best: 0.4526739 (570) total: 1.16s remaining: 869ms 571: learn: 0.3602503 test: 0.4526871 best: 0.4526739 (570) total: 1.16s remaining: 867ms 572: learn: 0.3600745 test: 0.4526388 best: 0.4526388 (572) total: 1.16s remaining: 865ms 573: learn: 0.3598905 test: 0.4526175 best: 0.4526175 (573) total: 1.16s remaining: 863ms 574: learn: 0.3596701 test: 0.4526206 best: 0.4526175 (573) total: 1.16s remaining: 860ms 575: learn: 0.3595452 test: 0.4525539 best: 0.4525539 (575) total: 1.17s remaining: 858ms 576: learn: 0.3594343 test: 0.4526179 best: 0.4525539 (575) total: 1.17s remaining: 856ms 577: learn: 0.3592430 test: 0.4526135 best: 0.4525539 (575) total: 1.17s remaining: 854ms 578: learn: 0.3590777 test: 0.4526206 best: 0.4525539 (575) total: 1.17s remaining: 852ms 579: learn: 0.3589057 test: 0.4525578 best: 0.4525539 (575) total: 1.17s remaining: 849ms 580: learn: 0.3587969 test: 0.4525692 best: 0.4525539 (575) total: 1.17s remaining: 847ms 581: learn: 0.3586741 test: 0.4525287 best: 0.4525287 (581) total: 1.18s remaining: 845ms 582: learn: 0.3585389 test: 0.4524736 best: 0.4524736 (582) total: 1.18s remaining: 843ms 583: learn: 0.3583276 test: 0.4524619 best: 0.4524619 (583) total: 1.18s remaining: 841ms 584: learn: 0.3582043 test: 0.4523919 best: 0.4523919 (584) total: 1.18s remaining: 839ms 585: learn: 0.3580040 test: 0.4522215 best: 0.4522215 (585) total: 1.18s remaining: 837ms 586: learn: 0.3578602 test: 0.4522179 best: 0.4522179 (586) total: 1.19s remaining: 834ms 587: learn: 0.3577584 test: 0.4521785 best: 0.4521785 (587) total: 1.19s remaining: 832ms 588: learn: 0.3576225 test: 0.4521613 best: 0.4521613 (588) total: 1.19s remaining: 830ms 589: learn: 0.3574660 test: 0.4522085 best: 0.4521613 (588) total: 1.19s remaining: 828ms 590: learn: 0.3573013 test: 0.4521938 best: 0.4521613 (588) total: 1.19s remaining: 826ms 591: learn: 0.3571465 test: 0.4521626 best: 0.4521613 (588) total: 1.19s remaining: 823ms 592: learn: 0.3570434 test: 0.4521739 best: 0.4521613 (588) total: 1.2s remaining: 821ms 593: learn: 0.3569294 test: 0.4522541 best: 0.4521613 (588) total: 1.2s remaining: 819ms 594: learn: 0.3567138 test: 0.4523330 best: 0.4521613 (588) total: 1.2s remaining: 817ms 595: learn: 0.3565939 test: 0.4523812 best: 0.4521613 (588) total: 1.2s remaining: 814ms 596: learn: 0.3564835 test: 0.4523393 best: 0.4521613 (588) total: 1.2s remaining: 812ms 597: learn: 0.3563931 test: 0.4523082 best: 0.4521613 (588) total: 1.2s remaining: 810ms 598: learn: 0.3563169 test: 0.4523311 best: 0.4521613 (588) total: 1.21s remaining: 808ms 599: learn: 0.3562045 test: 0.4523020 best: 0.4521613 (588) total: 1.21s remaining: 805ms 600: learn: 0.3559486 test: 0.4522038 best: 0.4521613 (588) total: 1.21s remaining: 803ms 601: learn: 0.3557929 test: 0.4521696 best: 0.4521613 (588) total: 1.21s remaining: 801ms 602: learn: 0.3556311 test: 0.4521975 best: 0.4521613 (588) total: 1.21s remaining: 799ms 603: learn: 0.3555172 test: 0.4522001 best: 0.4521613 (588) total: 1.22s remaining: 797ms 604: learn: 0.3552883 test: 0.4520314 best: 0.4520314 (604) total: 1.22s remaining: 794ms 605: learn: 0.3551670 test: 0.4520707 best: 0.4520314 (604) total: 1.22s remaining: 792ms 606: learn: 0.3550347 test: 0.4520391 best: 0.4520314 (604) total: 1.22s remaining: 790ms 607: learn: 0.3548418 test: 0.4519827 best: 0.4519827 (607) total: 1.22s remaining: 788ms 608: learn: 0.3546102 test: 0.4519486 best: 0.4519486 (608) total: 1.22s remaining: 786ms 609: learn: 0.3544652 test: 0.4519662 best: 0.4519486 (608) total: 1.23s remaining: 783ms 610: learn: 0.3543448 test: 0.4519505 best: 0.4519486 (608) total: 1.23s remaining: 781ms 611: learn: 0.3540972 test: 0.4516767 best: 0.4516767 (611) total: 1.23s remaining: 779ms 612: learn: 0.3539381 test: 0.4516293 best: 0.4516293 (612) total: 1.23s remaining: 776ms 613: learn: 0.3537930 test: 0.4516225 best: 0.4516225 (613) total: 1.23s remaining: 774ms 614: learn: 0.3536655 test: 0.4516223 best: 0.4516223 (614) total: 1.23s remaining: 772ms 615: learn: 0.3535237 test: 0.4515574 best: 0.4515574 (615) total: 1.23s remaining: 769ms 616: learn: 0.3534395 test: 0.4515008 best: 0.4515008 (616) total: 1.24s remaining: 767ms 617: learn: 0.3532703 test: 0.4513767 best: 0.4513767 (617) total: 1.24s remaining: 765ms 618: learn: 0.3531507 test: 0.4513808 best: 0.4513767 (617) total: 1.24s remaining: 763ms 619: learn: 0.3529990 test: 0.4513338 best: 0.4513338 (619) total: 1.24s remaining: 761ms 620: learn: 0.3528324 test: 0.4512358 best: 0.4512358 (620) total: 1.24s remaining: 758ms 621: learn: 0.3527236 test: 0.4511975 best: 0.4511975 (621) total: 1.24s remaining: 756ms 622: learn: 0.3526426 test: 0.4512292 best: 0.4511975 (621) total: 1.25s remaining: 754ms 623: learn: 0.3524959 test: 0.4513646 best: 0.4511975 (621) total: 1.25s remaining: 752ms 624: learn: 0.3524341 test: 0.4513227 best: 0.4511975 (621) total: 1.25s remaining: 750ms 625: learn: 0.3522439 test: 0.4513181 best: 0.4511975 (621) total: 1.25s remaining: 748ms 626: learn: 0.3520710 test: 0.4513923 best: 0.4511975 (621) total: 1.25s remaining: 746ms 627: learn: 0.3519070 test: 0.4514177 best: 0.4511975 (621) total: 1.25s remaining: 744ms 628: learn: 0.3518303 test: 0.4514094 best: 0.4511975 (621) total: 1.26s remaining: 742ms 629: learn: 0.3516313 test: 0.4514009 best: 0.4511975 (621) total: 1.26s remaining: 740ms 630: learn: 0.3514781 test: 0.4513576 best: 0.4511975 (621) total: 1.26s remaining: 737ms 631: learn: 0.3513106 test: 0.4513620 best: 0.4511975 (621) total: 1.26s remaining: 735ms 632: learn: 0.3511917 test: 0.4513382 best: 0.4511975 (621) total: 1.26s remaining: 733ms 633: learn: 0.3510870 test: 0.4512828 best: 0.4511975 (621) total: 1.27s remaining: 731ms 634: learn: 0.3509999 test: 0.4512698 best: 0.4511975 (621) total: 1.27s remaining: 729ms 635: learn: 0.3508614 test: 0.4512558 best: 0.4511975 (621) total: 1.27s remaining: 727ms 636: learn: 0.3507796 test: 0.4512235 best: 0.4511975 (621) total: 1.27s remaining: 724ms 637: learn: 0.3506667 test: 0.4512186 best: 0.4511975 (621) total: 1.27s remaining: 722ms 638: learn: 0.3505759 test: 0.4511842 best: 0.4511842 (638) total: 1.27s remaining: 720ms 639: learn: 0.3504699 test: 0.4511717 best: 0.4511717 (639) total: 1.28s remaining: 718ms 640: learn: 0.3503966 test: 0.4511167 best: 0.4511167 (640) total: 1.28s remaining: 716ms 641: learn: 0.3502779 test: 0.4510674 best: 0.4510674 (641) total: 1.28s remaining: 714ms 642: learn: 0.3501999 test: 0.4510700 best: 0.4510674 (641) total: 1.28s remaining: 712ms 643: learn: 0.3500558 test: 0.4510462 best: 0.4510462 (643) total: 1.28s remaining: 710ms 644: learn: 0.3498809 test: 0.4508607 best: 0.4508607 (644) total: 1.28s remaining: 707ms 645: learn: 0.3498088 test: 0.4508257 best: 0.4508257 (645) total: 1.29s remaining: 706ms 646: learn: 0.3496633 test: 0.4507757 best: 0.4507757 (646) total: 1.29s remaining: 705ms 647: learn: 0.3494837 test: 0.4507397 best: 0.4507397 (647) total: 1.29s remaining: 703ms 648: learn: 0.3493792 test: 0.4507343 best: 0.4507343 (648) total: 1.3s remaining: 702ms 649: learn: 0.3492303 test: 0.4507165 best: 0.4507165 (649) total: 1.3s remaining: 700ms 650: learn: 0.3491264 test: 0.4506973 best: 0.4506973 (650) total: 1.3s remaining: 698ms 651: learn: 0.3489795 test: 0.4507093 best: 0.4506973 (650) total: 1.3s remaining: 696ms 652: learn: 0.3487483 test: 0.4505893 best: 0.4505893 (652) total: 1.31s remaining: 694ms 653: learn: 0.3485988 test: 0.4505608 best: 0.4505608 (653) total: 1.31s remaining: 692ms 654: learn: 0.3484733 test: 0.4505742 best: 0.4505608 (653) total: 1.31s remaining: 690ms 655: learn: 0.3483063 test: 0.4504212 best: 0.4504212 (655) total: 1.31s remaining: 688ms 656: learn: 0.3481913 test: 0.4504479 best: 0.4504212 (655) total: 1.31s remaining: 686ms 657: learn: 0.3480648 test: 0.4504311 best: 0.4504212 (655) total: 1.32s remaining: 684ms 658: learn: 0.3478703 test: 0.4502807 best: 0.4502807 (658) total: 1.32s remaining: 682ms 659: learn: 0.3476640 test: 0.4502985 best: 0.4502807 (658) total: 1.32s remaining: 680ms 660: learn: 0.3475148 test: 0.4502375 best: 0.4502375 (660) total: 1.32s remaining: 678ms 661: learn: 0.3474307 test: 0.4502277 best: 0.4502277 (661) total: 1.32s remaining: 676ms 662: learn: 0.3472480 test: 0.4502618 best: 0.4502277 (661) total: 1.33s remaining: 674ms 663: learn: 0.3471248 test: 0.4501787 best: 0.4501787 (663) total: 1.33s remaining: 673ms 664: learn: 0.3469478 test: 0.4501878 best: 0.4501787 (663) total: 1.33s remaining: 671ms 665: learn: 0.3467468 test: 0.4501752 best: 0.4501752 (665) total: 1.33s remaining: 669ms 666: learn: 0.3466099 test: 0.4501964 best: 0.4501752 (665) total: 1.33s remaining: 667ms 667: learn: 0.3464609 test: 0.4501244 best: 0.4501244 (667) total: 1.34s remaining: 664ms 668: learn: 0.3463551 test: 0.4500687 best: 0.4500687 (668) total: 1.34s remaining: 662ms 669: learn: 0.3462343 test: 0.4500241 best: 0.4500241 (669) total: 1.34s remaining: 660ms 670: learn: 0.3460802 test: 0.4499758 best: 0.4499758 (670) total: 1.34s remaining: 658ms 671: learn: 0.3457832 test: 0.4499942 best: 0.4499758 (670) total: 1.34s remaining: 656ms 672: learn: 0.3456550 test: 0.4499581 best: 0.4499581 (672) total: 1.35s remaining: 654ms 673: learn: 0.3455471 test: 0.4499217 best: 0.4499217 (673) total: 1.35s remaining: 652ms 674: learn: 0.3453801 test: 0.4499283 best: 0.4499217 (673) total: 1.35s remaining: 650ms 675: learn: 0.3452653 test: 0.4499382 best: 0.4499217 (673) total: 1.35s remaining: 648ms 676: learn: 0.3451395 test: 0.4499279 best: 0.4499217 (673) total: 1.35s remaining: 646ms 677: learn: 0.3449938 test: 0.4498734 best: 0.4498734 (677) total: 1.35s remaining: 644ms 678: learn: 0.3448599 test: 0.4498323 best: 0.4498323 (678) total: 1.36s remaining: 642ms 679: learn: 0.3447214 test: 0.4498490 best: 0.4498323 (678) total: 1.36s remaining: 639ms 680: learn: 0.3445565 test: 0.4497034 best: 0.4497034 (680) total: 1.36s remaining: 637ms 681: learn: 0.3444302 test: 0.4497114 best: 0.4497034 (680) total: 1.36s remaining: 635ms 682: learn: 0.3443149 test: 0.4497263 best: 0.4497034 (680) total: 1.36s remaining: 633ms 683: learn: 0.3442483 test: 0.4497423 best: 0.4497034 (680) total: 1.37s remaining: 631ms 684: learn: 0.3441722 test: 0.4497507 best: 0.4497034 (680) total: 1.37s remaining: 629ms 685: learn: 0.3440606 test: 0.4497421 best: 0.4497034 (680) total: 1.37s remaining: 627ms 686: learn: 0.3439843 test: 0.4497868 best: 0.4497034 (680) total: 1.37s remaining: 625ms 687: learn: 0.3438549 test: 0.4498327 best: 0.4497034 (680) total: 1.37s remaining: 623ms 688: learn: 0.3436875 test: 0.4498460 best: 0.4497034 (680) total: 1.38s remaining: 621ms 689: learn: 0.3435309 test: 0.4498920 best: 0.4497034 (680) total: 1.38s remaining: 619ms 690: learn: 0.3433689 test: 0.4498614 best: 0.4497034 (680) total: 1.38s remaining: 617ms 691: learn: 0.3432276 test: 0.4498571 best: 0.4497034 (680) total: 1.38s remaining: 615ms 692: learn: 0.3430815 test: 0.4498003 best: 0.4497034 (680) total: 1.38s remaining: 613ms 693: learn: 0.3429931 test: 0.4498154 best: 0.4497034 (680) total: 1.39s remaining: 611ms 694: learn: 0.3428547 test: 0.4497594 best: 0.4497034 (680) total: 1.39s remaining: 609ms 695: learn: 0.3427864 test: 0.4496403 best: 0.4496403 (695) total: 1.39s remaining: 607ms 696: learn: 0.3426784 test: 0.4496015 best: 0.4496015 (696) total: 1.39s remaining: 605ms 697: learn: 0.3424128 test: 0.4493935 best: 0.4493935 (697) total: 1.39s remaining: 603ms 698: learn: 0.3422978 test: 0.4493813 best: 0.4493813 (698) total: 1.39s remaining: 601ms 699: learn: 0.3422281 test: 0.4493704 best: 0.4493704 (699) total: 1.4s remaining: 598ms 700: learn: 0.3421115 test: 0.4494301 best: 0.4493704 (699) total: 1.4s remaining: 596ms 701: learn: 0.3420280 test: 0.4493929 best: 0.4493704 (699) total: 1.4s remaining: 594ms 702: learn: 0.3419194 test: 0.4493825 best: 0.4493704 (699) total: 1.4s remaining: 592ms 703: learn: 0.3418499 test: 0.4494706 best: 0.4493704 (699) total: 1.4s remaining: 590ms 704: learn: 0.3417394 test: 0.4495496 best: 0.4493704 (699) total: 1.41s remaining: 588ms 705: learn: 0.3415784 test: 0.4494536 best: 0.4493704 (699) total: 1.41s remaining: 586ms 706: learn: 0.3414503 test: 0.4494528 best: 0.4493704 (699) total: 1.41s remaining: 584ms 707: learn: 0.3412782 test: 0.4494707 best: 0.4493704 (699) total: 1.41s remaining: 582ms 708: learn: 0.3411702 test: 0.4494190 best: 0.4493704 (699) total: 1.41s remaining: 580ms 709: learn: 0.3409134 test: 0.4493317 best: 0.4493317 (709) total: 1.42s remaining: 578ms 710: learn: 0.3408406 test: 0.4493277 best: 0.4493277 (710) total: 1.42s remaining: 576ms 711: learn: 0.3407736 test: 0.4492972 best: 0.4492972 (711) total: 1.42s remaining: 574ms 712: learn: 0.3406474 test: 0.4492530 best: 0.4492530 (712) total: 1.42s remaining: 572ms 713: learn: 0.3405421 test: 0.4492533 best: 0.4492530 (712) total: 1.42s remaining: 570ms 714: learn: 0.3404309 test: 0.4493359 best: 0.4492530 (712) total: 1.42s remaining: 568ms 715: learn: 0.3403119 test: 0.4492935 best: 0.4492530 (712) total: 1.43s remaining: 566ms 716: learn: 0.3402084 test: 0.4492541 best: 0.4492530 (712) total: 1.43s remaining: 564ms 717: learn: 0.3400999 test: 0.4492278 best: 0.4492278 (717) total: 1.43s remaining: 562ms 718: learn: 0.3399238 test: 0.4492508 best: 0.4492278 (717) total: 1.43s remaining: 560ms 719: learn: 0.3398334 test: 0.4492470 best: 0.4492278 (717) total: 1.43s remaining: 557ms 720: learn: 0.3397041 test: 0.4492391 best: 0.4492278 (717) total: 1.44s remaining: 556ms 721: learn: 0.3396423 test: 0.4492348 best: 0.4492278 (717) total: 1.44s remaining: 553ms 722: learn: 0.3394952 test: 0.4492906 best: 0.4492278 (717) total: 1.44s remaining: 551ms 723: learn: 0.3393941 test: 0.4493074 best: 0.4492278 (717) total: 1.44s remaining: 549ms 724: learn: 0.3391323 test: 0.4491894 best: 0.4491894 (724) total: 1.44s remaining: 547ms 725: learn: 0.3390695 test: 0.4492249 best: 0.4491894 (724) total: 1.44s remaining: 545ms 726: learn: 0.3389961 test: 0.4492457 best: 0.4491894 (724) total: 1.45s remaining: 543ms 727: learn: 0.3388534 test: 0.4492629 best: 0.4491894 (724) total: 1.45s remaining: 541ms 728: learn: 0.3387811 test: 0.4492749 best: 0.4491894 (724) total: 1.45s remaining: 539ms 729: learn: 0.3387023 test: 0.4492508 best: 0.4491894 (724) total: 1.45s remaining: 537ms 730: learn: 0.3386125 test: 0.4493191 best: 0.4491894 (724) total: 1.45s remaining: 535ms 731: learn: 0.3384358 test: 0.4492011 best: 0.4491894 (724) total: 1.46s remaining: 533ms 732: learn: 0.3383206 test: 0.4491927 best: 0.4491894 (724) total: 1.46s remaining: 531ms 733: learn: 0.3382188 test: 0.4491330 best: 0.4491330 (733) total: 1.46s remaining: 529ms 734: learn: 0.3381224 test: 0.4490965 best: 0.4490965 (734) total: 1.46s remaining: 527ms 735: learn: 0.3380513 test: 0.4490976 best: 0.4490965 (734) total: 1.46s remaining: 525ms 736: learn: 0.3379321 test: 0.4490311 best: 0.4490311 (736) total: 1.46s remaining: 523ms 737: learn: 0.3378468 test: 0.4490440 best: 0.4490311 (736) total: 1.47s remaining: 520ms 738: learn: 0.3377314 test: 0.4490296 best: 0.4490296 (738) total: 1.47s remaining: 518ms 739: learn: 0.3376514 test: 0.4490403 best: 0.4490296 (738) total: 1.47s remaining: 516ms 740: learn: 0.3375279 test: 0.4489787 best: 0.4489787 (740) total: 1.47s remaining: 515ms 741: learn: 0.3373556 test: 0.4489523 best: 0.4489523 (741) total: 1.48s remaining: 513ms 742: learn: 0.3372855 test: 0.4489170 best: 0.4489170 (742) total: 1.48s remaining: 511ms 743: learn: 0.3370805 test: 0.4489074 best: 0.4489074 (743) total: 1.48s remaining: 509ms 744: learn: 0.3369286 test: 0.4489332 best: 0.4489074 (743) total: 1.48s remaining: 507ms 745: learn: 0.3368073 test: 0.4488808 best: 0.4488808 (745) total: 1.48s remaining: 505ms 746: learn: 0.3367102 test: 0.4488287 best: 0.4488287 (746) total: 1.49s remaining: 503ms 747: learn: 0.3365695 test: 0.4488994 best: 0.4488287 (746) total: 1.49s remaining: 501ms 748: learn: 0.3364098 test: 0.4489626 best: 0.4488287 (746) total: 1.49s remaining: 499ms 749: learn: 0.3363047 test: 0.4489390 best: 0.4488287 (746) total: 1.49s remaining: 497ms 750: learn: 0.3362020 test: 0.4488302 best: 0.4488287 (746) total: 1.49s remaining: 495ms 751: learn: 0.3360682 test: 0.4488184 best: 0.4488184 (751) total: 1.5s remaining: 494ms 752: learn: 0.3359355 test: 0.4488114 best: 0.4488114 (752) total: 1.5s remaining: 492ms 753: learn: 0.3357649 test: 0.4487194 best: 0.4487194 (753) total: 1.5s remaining: 490ms 754: learn: 0.3356758 test: 0.4486939 best: 0.4486939 (754) total: 1.5s remaining: 488ms 755: learn: 0.3354174 test: 0.4484674 best: 0.4484674 (755) total: 1.51s remaining: 486ms 756: learn: 0.3352449 test: 0.4484659 best: 0.4484659 (756) total: 1.51s remaining: 484ms 757: learn: 0.3351779 test: 0.4484698 best: 0.4484659 (756) total: 1.51s remaining: 482ms 758: learn: 0.3350295 test: 0.4484769 best: 0.4484659 (756) total: 1.51s remaining: 480ms 759: learn: 0.3349258 test: 0.4484045 best: 0.4484045 (759) total: 1.51s remaining: 478ms 760: learn: 0.3346965 test: 0.4483521 best: 0.4483521 (760) total: 1.52s remaining: 477ms 761: learn: 0.3345443 test: 0.4483616 best: 0.4483521 (760) total: 1.52s remaining: 474ms 762: learn: 0.3343984 test: 0.4483255 best: 0.4483255 (762) total: 1.52s remaining: 472ms 763: learn: 0.3342487 test: 0.4483490 best: 0.4483255 (762) total: 1.52s remaining: 470ms 764: learn: 0.3341746 test: 0.4483406 best: 0.4483255 (762) total: 1.52s remaining: 468ms 765: learn: 0.3339596 test: 0.4482022 best: 0.4482022 (765) total: 1.53s remaining: 466ms 766: learn: 0.3338799 test: 0.4482792 best: 0.4482022 (765) total: 1.53s remaining: 464ms 767: learn: 0.3335880 test: 0.4480480 best: 0.4480480 (767) total: 1.53s remaining: 463ms 768: learn: 0.3334890 test: 0.4480059 best: 0.4480059 (768) total: 1.53s remaining: 461ms 769: learn: 0.3334158 test: 0.4480274 best: 0.4480059 (768) total: 1.53s remaining: 459ms 770: learn: 0.3333199 test: 0.4480452 best: 0.4480059 (768) total: 1.54s remaining: 457ms 771: learn: 0.3331947 test: 0.4480729 best: 0.4480059 (768) total: 1.54s remaining: 455ms 772: learn: 0.3330361 test: 0.4479817 best: 0.4479817 (772) total: 1.54s remaining: 453ms 773: learn: 0.3329505 test: 0.4479530 best: 0.4479530 (773) total: 1.54s remaining: 451ms 774: learn: 0.3326915 test: 0.4477351 best: 0.4477351 (774) total: 1.54s remaining: 449ms 775: learn: 0.3326152 test: 0.4477006 best: 0.4477006 (775) total: 1.55s remaining: 447ms 776: learn: 0.3324882 test: 0.4475646 best: 0.4475646 (776) total: 1.55s remaining: 445ms 777: learn: 0.3323860 test: 0.4475196 best: 0.4475196 (777) total: 1.55s remaining: 443ms 778: learn: 0.3322565 test: 0.4475023 best: 0.4475023 (778) total: 1.55s remaining: 441ms 779: learn: 0.3321066 test: 0.4474318 best: 0.4474318 (779) total: 1.55s remaining: 439ms 780: learn: 0.3319431 test: 0.4473443 best: 0.4473443 (780) total: 1.56s remaining: 437ms 781: learn: 0.3318780 test: 0.4473519 best: 0.4473443 (780) total: 1.56s remaining: 435ms 782: learn: 0.3317741 test: 0.4473095 best: 0.4473095 (782) total: 1.56s remaining: 433ms 783: learn: 0.3316827 test: 0.4473089 best: 0.4473089 (783) total: 1.56s remaining: 431ms 784: learn: 0.3315570 test: 0.4472885 best: 0.4472885 (784) total: 1.56s remaining: 429ms 785: learn: 0.3314190 test: 0.4472668 best: 0.4472668 (785) total: 1.57s remaining: 427ms 786: learn: 0.3313254 test: 0.4472447 best: 0.4472447 (786) total: 1.57s remaining: 425ms 787: learn: 0.3311721 test: 0.4472506 best: 0.4472447 (786) total: 1.57s remaining: 423ms 788: learn: 0.3311030 test: 0.4472479 best: 0.4472447 (786) total: 1.57s remaining: 421ms 789: learn: 0.3309282 test: 0.4471860 best: 0.4471860 (789) total: 1.57s remaining: 419ms 790: learn: 0.3308158 test: 0.4471247 best: 0.4471247 (790) total: 1.58s remaining: 417ms 791: learn: 0.3307378 test: 0.4471671 best: 0.4471247 (790) total: 1.58s remaining: 415ms 792: learn: 0.3306531 test: 0.4471757 best: 0.4471247 (790) total: 1.58s remaining: 413ms 793: learn: 0.3305515 test: 0.4471828 best: 0.4471247 (790) total: 1.58s remaining: 411ms 794: learn: 0.3304838 test: 0.4471427 best: 0.4471247 (790) total: 1.58s remaining: 409ms 795: learn: 0.3303588 test: 0.4471033 best: 0.4471033 (795) total: 1.59s remaining: 407ms 796: learn: 0.3302404 test: 0.4471146 best: 0.4471033 (795) total: 1.59s remaining: 405ms 797: learn: 0.3300981 test: 0.4471381 best: 0.4471033 (795) total: 1.59s remaining: 403ms 798: learn: 0.3300096 test: 0.4470811 best: 0.4470811 (798) total: 1.59s remaining: 401ms 799: learn: 0.3299127 test: 0.4470412 best: 0.4470412 (799) total: 1.6s remaining: 399ms 800: learn: 0.3297780 test: 0.4470290 best: 0.4470290 (800) total: 1.6s remaining: 397ms 801: learn: 0.3297001 test: 0.4469901 best: 0.4469901 (801) total: 1.6s remaining: 395ms 802: learn: 0.3296007 test: 0.4469510 best: 0.4469510 (802) total: 1.6s remaining: 393ms 803: learn: 0.3294954 test: 0.4469343 best: 0.4469343 (803) total: 1.6s remaining: 391ms 804: learn: 0.3293730 test: 0.4468544 best: 0.4468544 (804) total: 1.6s remaining: 389ms 805: learn: 0.3293055 test: 0.4468859 best: 0.4468544 (804) total: 1.61s remaining: 387ms 806: learn: 0.3291587 test: 0.4468946 best: 0.4468544 (804) total: 1.61s remaining: 385ms 807: learn: 0.3290208 test: 0.4469143 best: 0.4468544 (804) total: 1.61s remaining: 383ms 808: learn: 0.3289118 test: 0.4468827 best: 0.4468544 (804) total: 1.61s remaining: 381ms 809: learn: 0.3287980 test: 0.4469598 best: 0.4468544 (804) total: 1.61s remaining: 379ms 810: learn: 0.3285994 test: 0.4469964 best: 0.4468544 (804) total: 1.61s remaining: 377ms 811: learn: 0.3284315 test: 0.4470848 best: 0.4468544 (804) total: 1.62s remaining: 374ms 812: learn: 0.3283003 test: 0.4471345 best: 0.4468544 (804) total: 1.62s remaining: 372ms 813: learn: 0.3282166 test: 0.4471151 best: 0.4468544 (804) total: 1.62s remaining: 370ms 814: learn: 0.3281250 test: 0.4471026 best: 0.4468544 (804) total: 1.62s remaining: 368ms 815: learn: 0.3279921 test: 0.4471559 best: 0.4468544 (804) total: 1.62s remaining: 366ms 816: learn: 0.3278370 test: 0.4471519 best: 0.4468544 (804) total: 1.63s remaining: 364ms 817: learn: 0.3277616 test: 0.4471235 best: 0.4468544 (804) total: 1.63s remaining: 362ms 818: learn: 0.3276950 test: 0.4471024 best: 0.4468544 (804) total: 1.63s remaining: 360ms 819: learn: 0.3276018 test: 0.4470298 best: 0.4468544 (804) total: 1.63s remaining: 358ms 820: learn: 0.3274646 test: 0.4469632 best: 0.4468544 (804) total: 1.63s remaining: 356ms 821: learn: 0.3273249 test: 0.4469973 best: 0.4468544 (804) total: 1.63s remaining: 354ms 822: learn: 0.3272356 test: 0.4469486 best: 0.4468544 (804) total: 1.64s remaining: 352ms 823: learn: 0.3271246 test: 0.4469178 best: 0.4468544 (804) total: 1.64s remaining: 350ms 824: learn: 0.3270272 test: 0.4469025 best: 0.4468544 (804) total: 1.64s remaining: 348ms 825: learn: 0.3268826 test: 0.4468332 best: 0.4468332 (825) total: 1.64s remaining: 346ms 826: learn: 0.3267683 test: 0.4467710 best: 0.4467710 (826) total: 1.64s remaining: 344ms 827: learn: 0.3267233 test: 0.4468028 best: 0.4467710 (826) total: 1.64s remaining: 342ms 828: learn: 0.3266394 test: 0.4467998 best: 0.4467710 (826) total: 1.65s remaining: 340ms 829: learn: 0.3265011 test: 0.4468246 best: 0.4467710 (826) total: 1.65s remaining: 338ms 830: learn: 0.3264659 test: 0.4468123 best: 0.4467710 (826) total: 1.65s remaining: 336ms 831: learn: 0.3263024 test: 0.4467532 best: 0.4467532 (831) total: 1.65s remaining: 333ms 832: learn: 0.3262003 test: 0.4466895 best: 0.4466895 (832) total: 1.65s remaining: 331ms 833: learn: 0.3260685 test: 0.4467169 best: 0.4466895 (832) total: 1.66s remaining: 330ms 834: learn: 0.3259548 test: 0.4466210 best: 0.4466210 (834) total: 1.66s remaining: 328ms 835: learn: 0.3258127 test: 0.4465855 best: 0.4465855 (835) total: 1.66s remaining: 326ms 836: learn: 0.3257575 test: 0.4465748 best: 0.4465748 (836) total: 1.66s remaining: 324ms 837: learn: 0.3256862 test: 0.4466397 best: 0.4465748 (836) total: 1.66s remaining: 322ms 838: learn: 0.3255857 test: 0.4466350 best: 0.4465748 (836) total: 1.67s remaining: 320ms 839: learn: 0.3254993 test: 0.4466350 best: 0.4465748 (836) total: 1.67s remaining: 318ms 840: learn: 0.3253725 test: 0.4465757 best: 0.4465748 (836) total: 1.67s remaining: 316ms 841: learn: 0.3252333 test: 0.4465062 best: 0.4465062 (841) total: 1.67s remaining: 313ms 842: learn: 0.3251191 test: 0.4465301 best: 0.4465062 (841) total: 1.67s remaining: 311ms 843: learn: 0.3250078 test: 0.4464478 best: 0.4464478 (843) total: 1.67s remaining: 309ms 844: learn: 0.3248804 test: 0.4464304 best: 0.4464304 (844) total: 1.68s remaining: 307ms 845: learn: 0.3247585 test: 0.4464804 best: 0.4464304 (844) total: 1.68s remaining: 305ms 846: learn: 0.3247042 test: 0.4464924 best: 0.4464304 (844) total: 1.68s remaining: 303ms 847: learn: 0.3246154 test: 0.4464493 best: 0.4464304 (844) total: 1.68s remaining: 301ms 848: learn: 0.3245394 test: 0.4464756 best: 0.4464304 (844) total: 1.68s remaining: 299ms 849: learn: 0.3244311 test: 0.4465001 best: 0.4464304 (844) total: 1.68s remaining: 297ms 850: learn: 0.3243775 test: 0.4464683 best: 0.4464304 (844) total: 1.69s remaining: 295ms 851: learn: 0.3242137 test: 0.4464668 best: 0.4464304 (844) total: 1.69s remaining: 293ms 852: learn: 0.3241368 test: 0.4464426 best: 0.4464304 (844) total: 1.69s remaining: 291ms 853: learn: 0.3240492 test: 0.4464308 best: 0.4464304 (844) total: 1.69s remaining: 289ms 854: learn: 0.3238082 test: 0.4463066 best: 0.4463066 (854) total: 1.69s remaining: 287ms 855: learn: 0.3237006 test: 0.4462505 best: 0.4462505 (855) total: 1.69s remaining: 285ms 856: learn: 0.3235326 test: 0.4461668 best: 0.4461668 (856) total: 1.7s remaining: 283ms 857: learn: 0.3234675 test: 0.4461273 best: 0.4461273 (857) total: 1.7s remaining: 281ms 858: learn: 0.3233841 test: 0.4461301 best: 0.4461273 (857) total: 1.7s remaining: 279ms 859: learn: 0.3232962 test: 0.4461676 best: 0.4461273 (857) total: 1.7s remaining: 277ms 860: learn: 0.3232065 test: 0.4462015 best: 0.4461273 (857) total: 1.7s remaining: 275ms 861: learn: 0.3230829 test: 0.4461734 best: 0.4461273 (857) total: 1.7s remaining: 273ms 862: learn: 0.3229138 test: 0.4461380 best: 0.4461273 (857) total: 1.71s remaining: 271ms 863: learn: 0.3228337 test: 0.4461187 best: 0.4461187 (863) total: 1.71s remaining: 269ms 864: learn: 0.3226794 test: 0.4460380 best: 0.4460380 (864) total: 1.71s remaining: 267ms 865: learn: 0.3225733 test: 0.4460392 best: 0.4460380 (864) total: 1.71s remaining: 265ms 866: learn: 0.3224867 test: 0.4460528 best: 0.4460380 (864) total: 1.72s remaining: 263ms 867: learn: 0.3224175 test: 0.4460712 best: 0.4460380 (864) total: 1.72s remaining: 261ms 868: learn: 0.3223132 test: 0.4460394 best: 0.4460380 (864) total: 1.72s remaining: 259ms 869: learn: 0.3222811 test: 0.4460182 best: 0.4460182 (869) total: 1.72s remaining: 257ms 870: learn: 0.3221397 test: 0.4459820 best: 0.4459820 (870) total: 1.72s remaining: 255ms 871: learn: 0.3219710 test: 0.4459557 best: 0.4459557 (871) total: 1.73s remaining: 253ms 872: learn: 0.3218084 test: 0.4460052 best: 0.4459557 (871) total: 1.73s remaining: 251ms 873: learn: 0.3217288 test: 0.4459796 best: 0.4459557 (871) total: 1.73s remaining: 249ms 874: learn: 0.3216271 test: 0.4459911 best: 0.4459557 (871) total: 1.73s remaining: 247ms 875: learn: 0.3215182 test: 0.4459739 best: 0.4459557 (871) total: 1.73s remaining: 245ms 876: learn: 0.3214295 test: 0.4458947 best: 0.4458947 (876) total: 1.73s remaining: 243ms 877: learn: 0.3213508 test: 0.4459046 best: 0.4458947 (876) total: 1.74s remaining: 241ms 878: learn: 0.3212303 test: 0.4458228 best: 0.4458228 (878) total: 1.74s remaining: 239ms 879: learn: 0.3210909 test: 0.4457500 best: 0.4457500 (879) total: 1.74s remaining: 237ms 880: learn: 0.3209811 test: 0.4457643 best: 0.4457500 (879) total: 1.74s remaining: 235ms 881: learn: 0.3208519 test: 0.4457309 best: 0.4457309 (881) total: 1.74s remaining: 233ms 882: learn: 0.3206554 test: 0.4455677 best: 0.4455677 (882) total: 1.75s remaining: 231ms 883: learn: 0.3205867 test: 0.4456083 best: 0.4455677 (882) total: 1.75s remaining: 229ms 884: learn: 0.3205395 test: 0.4455860 best: 0.4455677 (882) total: 1.75s remaining: 227ms 885: learn: 0.3204429 test: 0.4455597 best: 0.4455597 (885) total: 1.75s remaining: 225ms 886: learn: 0.3203529 test: 0.4456243 best: 0.4455597 (885) total: 1.75s remaining: 223ms 887: learn: 0.3202450 test: 0.4455517 best: 0.4455517 (887) total: 1.75s remaining: 221ms 888: learn: 0.3200878 test: 0.4455225 best: 0.4455225 (888) total: 1.75s remaining: 219ms 889: learn: 0.3200013 test: 0.4455375 best: 0.4455225 (888) total: 1.76s remaining: 217ms 890: learn: 0.3199157 test: 0.4455208 best: 0.4455208 (890) total: 1.76s remaining: 215ms 891: learn: 0.3198167 test: 0.4455510 best: 0.4455208 (890) total: 1.76s remaining: 213ms 892: learn: 0.3197023 test: 0.4455647 best: 0.4455208 (890) total: 1.76s remaining: 211ms 893: learn: 0.3196266 test: 0.4455524 best: 0.4455208 (890) total: 1.76s remaining: 209ms 894: learn: 0.3195202 test: 0.4455694 best: 0.4455208 (890) total: 1.77s remaining: 207ms 895: learn: 0.3194360 test: 0.4455543 best: 0.4455208 (890) total: 1.77s remaining: 205ms 896: learn: 0.3193204 test: 0.4456157 best: 0.4455208 (890) total: 1.77s remaining: 203ms 897: learn: 0.3192434 test: 0.4456211 best: 0.4455208 (890) total: 1.77s remaining: 201ms 898: learn: 0.3191823 test: 0.4456473 best: 0.4455208 (890) total: 1.77s remaining: 199ms 899: learn: 0.3191074 test: 0.4455302 best: 0.4455208 (890) total: 1.78s remaining: 197ms 900: learn: 0.3190035 test: 0.4455214 best: 0.4455208 (890) total: 1.78s remaining: 195ms 901: learn: 0.3188846 test: 0.4455469 best: 0.4455208 (890) total: 1.78s remaining: 193ms 902: learn: 0.3188104 test: 0.4455472 best: 0.4455208 (890) total: 1.78s remaining: 191ms 903: learn: 0.3186982 test: 0.4455060 best: 0.4455060 (903) total: 1.78s remaining: 189ms 904: learn: 0.3186027 test: 0.4454978 best: 0.4454978 (904) total: 1.78s remaining: 187ms 905: learn: 0.3185235 test: 0.4454817 best: 0.4454817 (905) total: 1.79s remaining: 185ms 906: learn: 0.3184428 test: 0.4455162 best: 0.4454817 (905) total: 1.79s remaining: 183ms 907: learn: 0.3183649 test: 0.4455216 best: 0.4454817 (905) total: 1.79s remaining: 181ms 908: learn: 0.3182977 test: 0.4455300 best: 0.4454817 (905) total: 1.79s remaining: 179ms 909: learn: 0.3180378 test: 0.4452906 best: 0.4452906 (909) total: 1.79s remaining: 177ms 910: learn: 0.3179224 test: 0.4453036 best: 0.4452906 (909) total: 1.8s remaining: 176ms 911: learn: 0.3178433 test: 0.4452591 best: 0.4452591 (911) total: 1.8s remaining: 174ms 912: learn: 0.3177341 test: 0.4452377 best: 0.4452377 (912) total: 1.8s remaining: 172ms 913: learn: 0.3176509 test: 0.4452572 best: 0.4452377 (912) total: 1.8s remaining: 170ms 914: learn: 0.3175585 test: 0.4452775 best: 0.4452377 (912) total: 1.8s remaining: 168ms 915: learn: 0.3173851 test: 0.4452791 best: 0.4452377 (912) total: 1.81s remaining: 166ms 916: learn: 0.3172763 test: 0.4452820 best: 0.4452377 (912) total: 1.81s remaining: 164ms 917: learn: 0.3171603 test: 0.4452286 best: 0.4452286 (917) total: 1.81s remaining: 162ms 918: learn: 0.3170917 test: 0.4451987 best: 0.4451987 (918) total: 1.81s remaining: 160ms 919: learn: 0.3170155 test: 0.4451553 best: 0.4451553 (919) total: 1.81s remaining: 158ms 920: learn: 0.3169429 test: 0.4452317 best: 0.4451553 (919) total: 1.82s remaining: 156ms 921: learn: 0.3168505 test: 0.4452106 best: 0.4451553 (919) total: 1.82s remaining: 154ms 922: learn: 0.3167600 test: 0.4451950 best: 0.4451553 (919) total: 1.82s remaining: 152ms 923: learn: 0.3166795 test: 0.4451300 best: 0.4451300 (923) total: 1.82s remaining: 150ms 924: learn: 0.3165725 test: 0.4451125 best: 0.4451125 (924) total: 1.82s remaining: 148ms 925: learn: 0.3164745 test: 0.4451228 best: 0.4451125 (924) total: 1.83s remaining: 146ms 926: learn: 0.3163607 test: 0.4450804 best: 0.4450804 (926) total: 1.83s remaining: 144ms 927: learn: 0.3162424 test: 0.4450846 best: 0.4450804 (926) total: 1.83s remaining: 142ms 928: learn: 0.3161570 test: 0.4450157 best: 0.4450157 (928) total: 1.83s remaining: 140ms 929: learn: 0.3161097 test: 0.4450263 best: 0.4450157 (928) total: 1.84s remaining: 138ms 930: learn: 0.3160296 test: 0.4450104 best: 0.4450104 (930) total: 1.84s remaining: 136ms 931: learn: 0.3159126 test: 0.4449210 best: 0.4449210 (931) total: 1.84s remaining: 134ms 932: learn: 0.3157717 test: 0.4448210 best: 0.4448210 (932) total: 1.84s remaining: 132ms 933: learn: 0.3156881 test: 0.4448241 best: 0.4448210 (932) total: 1.84s remaining: 130ms 934: learn: 0.3155896 test: 0.4448618 best: 0.4448210 (932) total: 1.85s remaining: 128ms 935: learn: 0.3154436 test: 0.4448043 best: 0.4448043 (935) total: 1.85s remaining: 126ms 936: learn: 0.3153250 test: 0.4447768 best: 0.4447768 (936) total: 1.85s remaining: 125ms 937: learn: 0.3152313 test: 0.4447839 best: 0.4447768 (936) total: 1.85s remaining: 123ms 938: learn: 0.3151321 test: 0.4447147 best: 0.4447147 (938) total: 1.86s remaining: 121ms 939: learn: 0.3149995 test: 0.4446968 best: 0.4446968 (939) total: 1.86s remaining: 119ms 940: learn: 0.3149027 test: 0.4447079 best: 0.4446968 (939) total: 1.86s remaining: 117ms 941: learn: 0.3147923 test: 0.4446599 best: 0.4446599 (941) total: 1.86s remaining: 115ms 942: learn: 0.3146760 test: 0.4446541 best: 0.4446541 (942) total: 1.86s remaining: 113ms 943: learn: 0.3145463 test: 0.4446697 best: 0.4446541 (942) total: 1.87s remaining: 111ms 944: learn: 0.3144103 test: 0.4446938 best: 0.4446541 (942) total: 1.87s remaining: 109ms 945: learn: 0.3143523 test: 0.4446536 best: 0.4446536 (945) total: 1.87s remaining: 107ms 946: learn: 0.3142948 test: 0.4446604 best: 0.4446536 (945) total: 1.87s remaining: 105ms 947: learn: 0.3142091 test: 0.4446790 best: 0.4446536 (945) total: 1.88s remaining: 103ms 948: learn: 0.3140833 test: 0.4446572 best: 0.4446536 (945) total: 1.88s remaining: 101ms 949: learn: 0.3140115 test: 0.4446729 best: 0.4446536 (945) total: 1.88s remaining: 98.9ms 950: learn: 0.3139034 test: 0.4446259 best: 0.4446259 (950) total: 1.88s remaining: 97ms 951: learn: 0.3137919 test: 0.4445681 best: 0.4445681 (951) total: 1.88s remaining: 95ms 952: learn: 0.3137028 test: 0.4446243 best: 0.4445681 (951) total: 1.89s remaining: 93ms 953: learn: 0.3136455 test: 0.4446189 best: 0.4445681 (951) total: 1.89s remaining: 91ms 954: learn: 0.3135709 test: 0.4446423 best: 0.4445681 (951) total: 1.89s remaining: 89ms 955: learn: 0.3134693 test: 0.4445852 best: 0.4445681 (951) total: 1.89s remaining: 87ms 956: learn: 0.3133453 test: 0.4445811 best: 0.4445681 (951) total: 1.89s remaining: 85ms 957: learn: 0.3132688 test: 0.4445970 best: 0.4445681 (951) total: 1.89s remaining: 83ms 958: learn: 0.3131294 test: 0.4445063 best: 0.4445063 (958) total: 1.9s remaining: 81.1ms 959: learn: 0.3130266 test: 0.4445248 best: 0.4445063 (958) total: 1.9s remaining: 79.1ms 960: learn: 0.3129239 test: 0.4445024 best: 0.4445024 (960) total: 1.9s remaining: 77.1ms 961: learn: 0.3128550 test: 0.4445128 best: 0.4445024 (960) total: 1.9s remaining: 75.1ms 962: learn: 0.3127257 test: 0.4445139 best: 0.4445024 (960) total: 1.9s remaining: 73.1ms 963: learn: 0.3126102 test: 0.4444677 best: 0.4444677 (963) total: 1.91s remaining: 71.2ms 964: learn: 0.3125010 test: 0.4444442 best: 0.4444442 (964) total: 1.91s remaining: 69.2ms 965: learn: 0.3124048 test: 0.4444565 best: 0.4444442 (964) total: 1.91s remaining: 67.2ms 966: learn: 0.3123129 test: 0.4443938 best: 0.4443938 (966) total: 1.91s remaining: 65.2ms 967: learn: 0.3122108 test: 0.4444015 best: 0.4443938 (966) total: 1.91s remaining: 63.2ms 968: learn: 0.3121302 test: 0.4444035 best: 0.4443938 (966) total: 1.92s remaining: 61.3ms 969: learn: 0.3120825 test: 0.4444132 best: 0.4443938 (966) total: 1.92s remaining: 59.3ms 970: learn: 0.3119909 test: 0.4444389 best: 0.4443938 (966) total: 1.92s remaining: 57.3ms 971: learn: 0.3119071 test: 0.4444478 best: 0.4443938 (966) total: 1.92s remaining: 55.3ms 972: learn: 0.3118359 test: 0.4444822 best: 0.4443938 (966) total: 1.92s remaining: 53.3ms 973: learn: 0.3117334 test: 0.4444749 best: 0.4443938 (966) total: 1.92s remaining: 51.4ms 974: learn: 0.3116460 test: 0.4444866 best: 0.4443938 (966) total: 1.93s remaining: 49.4ms 975: learn: 0.3115741 test: 0.4444930 best: 0.4443938 (966) total: 1.93s remaining: 47.4ms 976: learn: 0.3115064 test: 0.4444900 best: 0.4443938 (966) total: 1.93s remaining: 45.5ms 977: learn: 0.3114402 test: 0.4445040 best: 0.4443938 (966) total: 1.93s remaining: 43.5ms 978: learn: 0.3113510 test: 0.4445167 best: 0.4443938 (966) total: 1.93s remaining: 41.5ms 979: learn: 0.3112352 test: 0.4444321 best: 0.4443938 (966) total: 1.94s remaining: 39.5ms 980: learn: 0.3111575 test: 0.4444457 best: 0.4443938 (966) total: 1.94s remaining: 37.5ms 981: learn: 0.3110623 test: 0.4444896 best: 0.4443938 (966) total: 1.94s remaining: 35.6ms 982: learn: 0.3109767 test: 0.4444648 best: 0.4443938 (966) total: 1.94s remaining: 33.6ms 983: learn: 0.3109170 test: 0.4444549 best: 0.4443938 (966) total: 1.94s remaining: 31.6ms 984: learn: 0.3108043 test: 0.4444339 best: 0.4443938 (966) total: 1.95s remaining: 29.6ms 985: learn: 0.3106993 test: 0.4444209 best: 0.4443938 (966) total: 1.95s remaining: 27.6ms 986: learn: 0.3105823 test: 0.4443757 best: 0.4443757 (986) total: 1.95s remaining: 25.7ms 987: learn: 0.3104023 test: 0.4442578 best: 0.4442578 (987) total: 1.95s remaining: 23.7ms 988: learn: 0.3102693 test: 0.4442628 best: 0.4442578 (987) total: 1.95s remaining: 21.7ms 989: learn: 0.3101750 test: 0.4443024 best: 0.4442578 (987) total: 1.95s remaining: 19.7ms 990: learn: 0.3100911 test: 0.4443136 best: 0.4442578 (987) total: 1.96s remaining: 17.8ms 991: learn: 0.3100110 test: 0.4443248 best: 0.4442578 (987) total: 1.96s remaining: 15.8ms 992: learn: 0.3098664 test: 0.4442948 best: 0.4442578 (987) total: 1.96s remaining: 13.8ms 993: learn: 0.3097941 test: 0.4442559 best: 0.4442559 (993) total: 1.96s remaining: 11.8ms 994: learn: 0.3096655 test: 0.4441859 best: 0.4441859 (994) total: 1.96s remaining: 9.87ms 995: learn: 0.3095558 test: 0.4442223 best: 0.4441859 (994) total: 1.97s remaining: 7.89ms 996: learn: 0.3094865 test: 0.4442231 best: 0.4441859 (994) total: 1.97s remaining: 5.92ms 997: learn: 0.3094134 test: 0.4442271 best: 0.4441859 (994) total: 1.97s remaining: 3.95ms 998: learn: 0.3093425 test: 0.4442830 best: 0.4441859 (994) total: 1.97s remaining: 1.97ms 999: learn: 0.3092760 test: 0.4442812 best: 0.4441859 (994) total: 1.97s remaining: 0us bestTest = 0.4441859469 bestIteration = 994 Shrink model to first 995 iterations.
<catboost.core.CatBoostRegressor at 0x7f9b71ddbfa0>
catb_preds = tuned_catb.predict(catb_test_dataset)
catb_rms = mean_squared_error(test_data[LABEL], catb_preds, squared=False)
# To TF Dataset
train_dataset = df_to_dataset(X_train[FEATURES + [LABEL]], LABEL, shuffle=True)
val_dataset = df_to_dataset(X_val[FEATURES + [LABEL]], LABEL, shuffle=False) # No shuffle
test_dataset = df_to_dataset(test_data[FEATURES + [LABEL]], shuffle=False) # No target, no shuffle
/Users/antonsruberts/personal/TabTransformerTF/tabtransformertf/utils/preprocessing.py:21: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version. Convert to a numpy array before indexing instead. dataset[key] = value[:, tf.newaxis] /Users/antonsruberts/personal/TabTransformerTF/tabtransformertf/utils/preprocessing.py:21: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version. Convert to a numpy array before indexing instead. dataset[key] = value[:, tf.newaxis] /Users/antonsruberts/personal/TabTransformerTF/tabtransformertf/utils/preprocessing.py:27: FutureWarning: Support for multi-dimensional indexing (e.g. `obj[:, None]`) is deprecated and will be removed in a future version. Convert to a numpy array before indexing instead. dataset[key] = value[:, tf.newaxis]
ft_linear_encoder = FTTransformerEncoder(
numerical_features = NUMERIC_FEATURES,
categorical_features = [],
numerical_data = X_train[NUMERIC_FEATURES].values,
categorical_data =None, # No categorical data
y = None,
numerical_embedding_type='linear',
embedding_dim=64,
depth=3,
heads=6,
attn_dropout=0.3,
ff_dropout=0.3,
explainable=True
)
# Pass th encoder to the model
ft_linear_transformer = FTTransformer(
encoder=ft_linear_encoder,
out_dim=1,
out_activation="relu",
)
LEARNING_RATE = 0.001
WEIGHT_DECAY = 0.0001
NUM_EPOCHS = 1000
optimizer = tfa.optimizers.AdamW(
learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
ft_linear_transformer.compile(
optimizer = optimizer,
loss = {"output": tf.keras.losses.MeanSquaredError(name='mse'), "importances": None},
metrics= {"output": [tf.keras.metrics.RootMeanSquaredError(name='rmse')], "importances": None},
)
early = EarlyStopping(monitor="val_output_loss", mode="min", patience=16, restore_best_weights=True)
callback_list = [early]
ft_linear_history = ft_linear_transformer.fit(
train_dataset,
epochs=NUM_EPOCHS,
validation_data=val_dataset,
callbacks=callback_list
)
Epoch 1/1000 26/26 [==============================] - 10s 230ms/step - loss: 1.2817 - output_loss: 1.2817 - output_rmse: 1.1321 - val_loss: 0.7904 - val_output_loss: 0.7904 - val_output_rmse: 0.8890 Epoch 2/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.7574 - output_loss: 0.7574 - output_rmse: 0.8703 - val_loss: 0.6370 - val_output_loss: 0.6370 - val_output_rmse: 0.7981 Epoch 3/1000 26/26 [==============================] - 5s 190ms/step - loss: 0.6220 - output_loss: 0.6220 - output_rmse: 0.7887 - val_loss: 0.5082 - val_output_loss: 0.5082 - val_output_rmse: 0.7129 Epoch 4/1000 26/26 [==============================] - 5s 192ms/step - loss: 0.5493 - output_loss: 0.5493 - output_rmse: 0.7412 - val_loss: 0.4677 - val_output_loss: 0.4677 - val_output_rmse: 0.6839 Epoch 5/1000 26/26 [==============================] - 5s 205ms/step - loss: 0.5040 - output_loss: 0.5040 - output_rmse: 0.7099 - val_loss: 0.4803 - val_output_loss: 0.4803 - val_output_rmse: 0.6931 Epoch 6/1000 26/26 [==============================] - 6s 219ms/step - loss: 0.4777 - output_loss: 0.4777 - output_rmse: 0.6912 - val_loss: 0.3875 - val_output_loss: 0.3875 - val_output_rmse: 0.6225 Epoch 7/1000 26/26 [==============================] - 6s 215ms/step - loss: 0.4410 - output_loss: 0.4410 - output_rmse: 0.6641 - val_loss: 0.4442 - val_output_loss: 0.4442 - val_output_rmse: 0.6665 Epoch 8/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.4280 - output_loss: 0.4280 - output_rmse: 0.6542 - val_loss: 0.3708 - val_output_loss: 0.3708 - val_output_rmse: 0.6089 Epoch 9/1000 26/26 [==============================] - 6s 217ms/step - loss: 0.4045 - output_loss: 0.4045 - output_rmse: 0.6360 - val_loss: 0.3603 - val_output_loss: 0.3603 - val_output_rmse: 0.6002 Epoch 10/1000 26/26 [==============================] - 6s 220ms/step - loss: 0.4109 - output_loss: 0.4109 - output_rmse: 0.6410 - val_loss: 0.3524 - val_output_loss: 0.3524 - val_output_rmse: 0.5937 Epoch 11/1000 26/26 [==============================] - 6s 219ms/step - loss: 0.3795 - output_loss: 0.3795 - output_rmse: 0.6160 - val_loss: 0.3377 - val_output_loss: 0.3377 - val_output_rmse: 0.5811 Epoch 12/1000 26/26 [==============================] - 6s 213ms/step - loss: 0.3790 - output_loss: 0.3790 - output_rmse: 0.6156 - val_loss: 0.3414 - val_output_loss: 0.3414 - val_output_rmse: 0.5843 Epoch 13/1000 26/26 [==============================] - 6s 234ms/step - loss: 0.3726 - output_loss: 0.3726 - output_rmse: 0.6104 - val_loss: 0.3403 - val_output_loss: 0.3403 - val_output_rmse: 0.5834 Epoch 14/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.3663 - output_loss: 0.3663 - output_rmse: 0.6053 - val_loss: 0.3329 - val_output_loss: 0.3329 - val_output_rmse: 0.5770 Epoch 15/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.3619 - output_loss: 0.3619 - output_rmse: 0.6015 - val_loss: 0.3834 - val_output_loss: 0.3834 - val_output_rmse: 0.6192 Epoch 16/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.3679 - output_loss: 0.3679 - output_rmse: 0.6065 - val_loss: 0.3405 - val_output_loss: 0.3405 - val_output_rmse: 0.5835 Epoch 17/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.3493 - output_loss: 0.3493 - output_rmse: 0.5910 - val_loss: 0.3257 - val_output_loss: 0.3257 - val_output_rmse: 0.5707 Epoch 18/1000 26/26 [==============================] - 6s 222ms/step - loss: 0.3610 - output_loss: 0.3610 - output_rmse: 0.6008 - val_loss: 0.3143 - val_output_loss: 0.3143 - val_output_rmse: 0.5606 Epoch 19/1000 26/26 [==============================] - 6s 229ms/step - loss: 0.3433 - output_loss: 0.3433 - output_rmse: 0.5859 - val_loss: 0.3179 - val_output_loss: 0.3179 - val_output_rmse: 0.5638 Epoch 20/1000 26/26 [==============================] - 6s 215ms/step - loss: 0.3510 - output_loss: 0.3510 - output_rmse: 0.5924 - val_loss: 0.3436 - val_output_loss: 0.3436 - val_output_rmse: 0.5862 Epoch 21/1000 26/26 [==============================] - 6s 211ms/step - loss: 0.3435 - output_loss: 0.3435 - output_rmse: 0.5861 - val_loss: 0.3177 - val_output_loss: 0.3177 - val_output_rmse: 0.5636 Epoch 22/1000 26/26 [==============================] - 6s 217ms/step - loss: 0.3351 - output_loss: 0.3351 - output_rmse: 0.5789 - val_loss: 0.3282 - val_output_loss: 0.3282 - val_output_rmse: 0.5729 Epoch 23/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.3405 - output_loss: 0.3405 - output_rmse: 0.5835 - val_loss: 0.3122 - val_output_loss: 0.3122 - val_output_rmse: 0.5588 Epoch 24/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.3441 - output_loss: 0.3441 - output_rmse: 0.5866 - val_loss: 0.3124 - val_output_loss: 0.3124 - val_output_rmse: 0.5589 Epoch 25/1000 26/26 [==============================] - 6s 221ms/step - loss: 0.3348 - output_loss: 0.3348 - output_rmse: 0.5786 - val_loss: 0.3270 - val_output_loss: 0.3270 - val_output_rmse: 0.5718 Epoch 26/1000 26/26 [==============================] - 6s 242ms/step - loss: 0.3371 - output_loss: 0.3371 - output_rmse: 0.5806 - val_loss: 0.3164 - val_output_loss: 0.3164 - val_output_rmse: 0.5625 Epoch 27/1000 26/26 [==============================] - 6s 227ms/step - loss: 0.3281 - output_loss: 0.3281 - output_rmse: 0.5728 - val_loss: 0.3086 - val_output_loss: 0.3086 - val_output_rmse: 0.5555 Epoch 28/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.3348 - output_loss: 0.3348 - output_rmse: 0.5786 - val_loss: 0.3193 - val_output_loss: 0.3193 - val_output_rmse: 0.5651 Epoch 29/1000 26/26 [==============================] - 6s 211ms/step - loss: 0.3365 - output_loss: 0.3365 - output_rmse: 0.5801 - val_loss: 0.3190 - val_output_loss: 0.3190 - val_output_rmse: 0.5648 Epoch 30/1000 26/26 [==============================] - 6s 216ms/step - loss: 0.3348 - output_loss: 0.3348 - output_rmse: 0.5786 - val_loss: 0.3177 - val_output_loss: 0.3177 - val_output_rmse: 0.5636 Epoch 31/1000 26/26 [==============================] - 6s 224ms/step - loss: 0.3304 - output_loss: 0.3304 - output_rmse: 0.5748 - val_loss: 0.3098 - val_output_loss: 0.3098 - val_output_rmse: 0.5566 Epoch 32/1000 26/26 [==============================] - 6s 225ms/step - loss: 0.3479 - output_loss: 0.3479 - output_rmse: 0.5898 - val_loss: 0.3129 - val_output_loss: 0.3129 - val_output_rmse: 0.5594 Epoch 33/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.3259 - output_loss: 0.3259 - output_rmse: 0.5709 - val_loss: 0.3039 - val_output_loss: 0.3039 - val_output_rmse: 0.5513 Epoch 34/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.3330 - output_loss: 0.3330 - output_rmse: 0.5770 - val_loss: 0.3092 - val_output_loss: 0.3092 - val_output_rmse: 0.5561 Epoch 35/1000 26/26 [==============================] - 6s 222ms/step - loss: 0.3190 - output_loss: 0.3190 - output_rmse: 0.5648 - val_loss: 0.3019 - val_output_loss: 0.3019 - val_output_rmse: 0.5495 Epoch 36/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.3228 - output_loss: 0.3228 - output_rmse: 0.5682 - val_loss: 0.3086 - val_output_loss: 0.3086 - val_output_rmse: 0.5555 Epoch 37/1000 26/26 [==============================] - 6s 217ms/step - loss: 0.3171 - output_loss: 0.3171 - output_rmse: 0.5631 - val_loss: 0.2924 - val_output_loss: 0.2924 - val_output_rmse: 0.5408 Epoch 38/1000 26/26 [==============================] - 6s 219ms/step - loss: 0.3190 - output_loss: 0.3190 - output_rmse: 0.5648 - val_loss: 0.3220 - val_output_loss: 0.3220 - val_output_rmse: 0.5674 Epoch 39/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.3208 - output_loss: 0.3208 - output_rmse: 0.5664 - val_loss: 0.3051 - val_output_loss: 0.3051 - val_output_rmse: 0.5523 Epoch 40/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.3145 - output_loss: 0.3145 - output_rmse: 0.5608 - val_loss: 0.2910 - val_output_loss: 0.2910 - val_output_rmse: 0.5395 Epoch 41/1000 26/26 [==============================] - 6s 216ms/step - loss: 0.3074 - output_loss: 0.3074 - output_rmse: 0.5544 - val_loss: 0.3110 - val_output_loss: 0.3110 - val_output_rmse: 0.5577 Epoch 42/1000 26/26 [==============================] - 6s 224ms/step - loss: 0.3123 - output_loss: 0.3123 - output_rmse: 0.5589 - val_loss: 0.2943 - val_output_loss: 0.2943 - val_output_rmse: 0.5425 Epoch 43/1000 26/26 [==============================] - 6s 218ms/step - loss: 0.3201 - output_loss: 0.3201 - output_rmse: 0.5657 - val_loss: 0.3230 - val_output_loss: 0.3230 - val_output_rmse: 0.5683 Epoch 44/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.3139 - output_loss: 0.3139 - output_rmse: 0.5602 - val_loss: 0.3747 - val_output_loss: 0.3747 - val_output_rmse: 0.6121 Epoch 45/1000 26/26 [==============================] - 6s 218ms/step - loss: 0.3220 - output_loss: 0.3220 - output_rmse: 0.5674 - val_loss: 0.2979 - val_output_loss: 0.2979 - val_output_rmse: 0.5458 Epoch 46/1000 26/26 [==============================] - 6s 221ms/step - loss: 0.3058 - output_loss: 0.3058 - output_rmse: 0.5530 - val_loss: 0.3060 - val_output_loss: 0.3060 - val_output_rmse: 0.5531 Epoch 47/1000 26/26 [==============================] - 6s 217ms/step - loss: 0.3079 - output_loss: 0.3079 - output_rmse: 0.5549 - val_loss: 0.2886 - val_output_loss: 0.2886 - val_output_rmse: 0.5372 Epoch 48/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.3170 - output_loss: 0.3170 - output_rmse: 0.5630 - val_loss: 0.3127 - val_output_loss: 0.3127 - val_output_rmse: 0.5592 Epoch 49/1000 26/26 [==============================] - 6s 217ms/step - loss: 0.3095 - output_loss: 0.3095 - output_rmse: 0.5563 - val_loss: 0.2861 - val_output_loss: 0.2861 - val_output_rmse: 0.5349 Epoch 50/1000 26/26 [==============================] - 6s 224ms/step - loss: 0.3108 - output_loss: 0.3108 - output_rmse: 0.5575 - val_loss: 0.3061 - val_output_loss: 0.3061 - val_output_rmse: 0.5533 Epoch 51/1000 26/26 [==============================] - 6s 239ms/step - loss: 0.3160 - output_loss: 0.3160 - output_rmse: 0.5622 - val_loss: 0.2884 - val_output_loss: 0.2884 - val_output_rmse: 0.5370 Epoch 52/1000 26/26 [==============================] - 6s 239ms/step - loss: 0.2997 - output_loss: 0.2997 - output_rmse: 0.5474 - val_loss: 0.2876 - val_output_loss: 0.2876 - val_output_rmse: 0.5363 Epoch 53/1000 26/26 [==============================] - 6s 234ms/step - loss: 0.3173 - output_loss: 0.3173 - output_rmse: 0.5633 - val_loss: 0.2901 - val_output_loss: 0.2901 - val_output_rmse: 0.5386 Epoch 54/1000 26/26 [==============================] - 6s 224ms/step - loss: 0.3011 - output_loss: 0.3011 - output_rmse: 0.5487 - val_loss: 0.2853 - val_output_loss: 0.2853 - val_output_rmse: 0.5342 Epoch 55/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.3008 - output_loss: 0.3008 - output_rmse: 0.5485 - val_loss: 0.2865 - val_output_loss: 0.2865 - val_output_rmse: 0.5352 Epoch 56/1000 26/26 [==============================] - 6s 236ms/step - loss: 0.3011 - output_loss: 0.3011 - output_rmse: 0.5487 - val_loss: 0.2912 - val_output_loss: 0.2912 - val_output_rmse: 0.5396 Epoch 57/1000 26/26 [==============================] - 6s 233ms/step - loss: 0.3042 - output_loss: 0.3042 - output_rmse: 0.5515 - val_loss: 0.3066 - val_output_loss: 0.3066 - val_output_rmse: 0.5537 Epoch 58/1000 26/26 [==============================] - 6s 245ms/step - loss: 0.3006 - output_loss: 0.3006 - output_rmse: 0.5482 - val_loss: 0.2879 - val_output_loss: 0.2879 - val_output_rmse: 0.5366 Epoch 59/1000 26/26 [==============================] - 7s 253ms/step - loss: 0.2987 - output_loss: 0.2987 - output_rmse: 0.5465 - val_loss: 0.2935 - val_output_loss: 0.2935 - val_output_rmse: 0.5418 Epoch 60/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.2940 - output_loss: 0.2940 - output_rmse: 0.5422 - val_loss: 0.2889 - val_output_loss: 0.2889 - val_output_rmse: 0.5375 Epoch 61/1000 26/26 [==============================] - 6s 210ms/step - loss: 0.2978 - output_loss: 0.2978 - output_rmse: 0.5457 - val_loss: 0.2999 - val_output_loss: 0.2999 - val_output_rmse: 0.5476 Epoch 62/1000 26/26 [==============================] - 6s 210ms/step - loss: 0.2953 - output_loss: 0.2953 - output_rmse: 0.5434 - val_loss: 0.2903 - val_output_loss: 0.2903 - val_output_rmse: 0.5388 Epoch 63/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2953 - output_loss: 0.2953 - output_rmse: 0.5434 - val_loss: 0.2846 - val_output_loss: 0.2846 - val_output_rmse: 0.5335 Epoch 64/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.3032 - output_loss: 0.3032 - output_rmse: 0.5506 - val_loss: 0.3062 - val_output_loss: 0.3062 - val_output_rmse: 0.5534 Epoch 65/1000 26/26 [==============================] - 6s 210ms/step - loss: 0.2966 - output_loss: 0.2966 - output_rmse: 0.5446 - val_loss: 0.3080 - val_output_loss: 0.3080 - val_output_rmse: 0.5550 Epoch 66/1000 26/26 [==============================] - 5s 209ms/step - loss: 0.2928 - output_loss: 0.2928 - output_rmse: 0.5411 - val_loss: 0.2776 - val_output_loss: 0.2776 - val_output_rmse: 0.5268 Epoch 67/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2995 - output_loss: 0.2995 - output_rmse: 0.5473 - val_loss: 0.2823 - val_output_loss: 0.2823 - val_output_rmse: 0.5313 Epoch 68/1000 26/26 [==============================] - 5s 209ms/step - loss: 0.2863 - output_loss: 0.2863 - output_rmse: 0.5351 - val_loss: 0.2817 - val_output_loss: 0.2817 - val_output_rmse: 0.5307 Epoch 69/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2896 - output_loss: 0.2896 - output_rmse: 0.5381 - val_loss: 0.2852 - val_output_loss: 0.2852 - val_output_rmse: 0.5340 Epoch 70/1000 26/26 [==============================] - 6s 211ms/step - loss: 0.2837 - output_loss: 0.2837 - output_rmse: 0.5326 - val_loss: 0.2743 - val_output_loss: 0.2743 - val_output_rmse: 0.5238 Epoch 71/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2926 - output_loss: 0.2926 - output_rmse: 0.5409 - val_loss: 0.2821 - val_output_loss: 0.2821 - val_output_rmse: 0.5311 Epoch 72/1000 26/26 [==============================] - 5s 209ms/step - loss: 0.2861 - output_loss: 0.2861 - output_rmse: 0.5348 - val_loss: 0.2715 - val_output_loss: 0.2715 - val_output_rmse: 0.5210 Epoch 73/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2831 - output_loss: 0.2831 - output_rmse: 0.5320 - val_loss: 0.2919 - val_output_loss: 0.2919 - val_output_rmse: 0.5402 Epoch 74/1000 26/26 [==============================] - 5s 209ms/step - loss: 0.2868 - output_loss: 0.2868 - output_rmse: 0.5356 - val_loss: 0.2917 - val_output_loss: 0.2917 - val_output_rmse: 0.5401 Epoch 75/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.2926 - output_loss: 0.2926 - output_rmse: 0.5409 - val_loss: 0.2867 - val_output_loss: 0.2867 - val_output_rmse: 0.5354 Epoch 76/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2915 - output_loss: 0.2915 - output_rmse: 0.5399 - val_loss: 0.2742 - val_output_loss: 0.2742 - val_output_rmse: 0.5237 Epoch 77/1000 26/26 [==============================] - 6s 209ms/step - loss: 0.2879 - output_loss: 0.2879 - output_rmse: 0.5366 - val_loss: 0.2768 - val_output_loss: 0.2768 - val_output_rmse: 0.5261 Epoch 78/1000 26/26 [==============================] - 5s 209ms/step - loss: 0.2785 - output_loss: 0.2785 - output_rmse: 0.5277 - val_loss: 0.2732 - val_output_loss: 0.2732 - val_output_rmse: 0.5227 Epoch 79/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2785 - output_loss: 0.2785 - output_rmse: 0.5277 - val_loss: 0.2835 - val_output_loss: 0.2835 - val_output_rmse: 0.5324 Epoch 80/1000 26/26 [==============================] - 6s 210ms/step - loss: 0.2798 - output_loss: 0.2798 - output_rmse: 0.5290 - val_loss: 0.2820 - val_output_loss: 0.2820 - val_output_rmse: 0.5311 Epoch 81/1000 26/26 [==============================] - 6s 213ms/step - loss: 0.2763 - output_loss: 0.2763 - output_rmse: 0.5256 - val_loss: 0.2888 - val_output_loss: 0.2888 - val_output_rmse: 0.5374 Epoch 82/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2827 - output_loss: 0.2827 - output_rmse: 0.5317 - val_loss: 0.2874 - val_output_loss: 0.2874 - val_output_rmse: 0.5361 Epoch 83/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2865 - output_loss: 0.2865 - output_rmse: 0.5353 - val_loss: 0.2784 - val_output_loss: 0.2784 - val_output_rmse: 0.5276 Epoch 84/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2749 - output_loss: 0.2749 - output_rmse: 0.5243 - val_loss: 0.2664 - val_output_loss: 0.2664 - val_output_rmse: 0.5162 Epoch 85/1000 26/26 [==============================] - 6s 209ms/step - loss: 0.2778 - output_loss: 0.2778 - output_rmse: 0.5271 - val_loss: 0.2678 - val_output_loss: 0.2678 - val_output_rmse: 0.5175 Epoch 86/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.2792 - output_loss: 0.2792 - output_rmse: 0.5284 - val_loss: 0.2776 - val_output_loss: 0.2776 - val_output_rmse: 0.5269 Epoch 87/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.2834 - output_loss: 0.2834 - output_rmse: 0.5324 - val_loss: 0.2963 - val_output_loss: 0.2963 - val_output_rmse: 0.5443 Epoch 88/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2807 - output_loss: 0.2807 - output_rmse: 0.5298 - val_loss: 0.2928 - val_output_loss: 0.2928 - val_output_rmse: 0.5411 Epoch 89/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2777 - output_loss: 0.2777 - output_rmse: 0.5270 - val_loss: 0.2773 - val_output_loss: 0.2773 - val_output_rmse: 0.5266 Epoch 90/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2705 - output_loss: 0.2705 - output_rmse: 0.5201 - val_loss: 0.2669 - val_output_loss: 0.2669 - val_output_rmse: 0.5166 Epoch 91/1000 26/26 [==============================] - 6s 209ms/step - loss: 0.2737 - output_loss: 0.2737 - output_rmse: 0.5232 - val_loss: 0.2689 - val_output_loss: 0.2689 - val_output_rmse: 0.5185 Epoch 92/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2624 - output_loss: 0.2624 - output_rmse: 0.5122 - val_loss: 0.2681 - val_output_loss: 0.2681 - val_output_rmse: 0.5178 Epoch 93/1000 26/26 [==============================] - 5s 208ms/step - loss: 0.2838 - output_loss: 0.2838 - output_rmse: 0.5327 - val_loss: 0.2762 - val_output_loss: 0.2762 - val_output_rmse: 0.5256 Epoch 94/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2718 - output_loss: 0.2718 - output_rmse: 0.5213 - val_loss: 0.2665 - val_output_loss: 0.2665 - val_output_rmse: 0.5162 Epoch 95/1000 26/26 [==============================] - 6s 210ms/step - loss: 0.2682 - output_loss: 0.2682 - output_rmse: 0.5179 - val_loss: 0.2737 - val_output_loss: 0.2737 - val_output_rmse: 0.5232 Epoch 96/1000 26/26 [==============================] - 6s 209ms/step - loss: 0.2665 - output_loss: 0.2665 - output_rmse: 0.5163 - val_loss: 0.2667 - val_output_loss: 0.2667 - val_output_rmse: 0.5164 Epoch 97/1000 26/26 [==============================] - 6s 212ms/step - loss: 0.2639 - output_loss: 0.2639 - output_rmse: 0.5137 - val_loss: 0.2759 - val_output_loss: 0.2759 - val_output_rmse: 0.5253 Epoch 98/1000 26/26 [==============================] - 5s 207ms/step - loss: 0.2618 - output_loss: 0.2618 - output_rmse: 0.5116 - val_loss: 0.2595 - val_output_loss: 0.2595 - val_output_rmse: 0.5094 Epoch 99/1000 26/26 [==============================] - 6s 216ms/step - loss: 0.2694 - output_loss: 0.2694 - output_rmse: 0.5191 - val_loss: 0.2705 - val_output_loss: 0.2705 - val_output_rmse: 0.5201 Epoch 100/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.2881 - output_loss: 0.2881 - output_rmse: 0.5367 - val_loss: 0.2834 - val_output_loss: 0.2834 - val_output_rmse: 0.5324 Epoch 101/1000 26/26 [==============================] - 6s 213ms/step - loss: 0.2700 - output_loss: 0.2700 - output_rmse: 0.5196 - val_loss: 0.2768 - val_output_loss: 0.2768 - val_output_rmse: 0.5262 Epoch 102/1000 26/26 [==============================] - 6s 227ms/step - loss: 0.2642 - output_loss: 0.2642 - output_rmse: 0.5140 - val_loss: 0.2635 - val_output_loss: 0.2635 - val_output_rmse: 0.5133 Epoch 103/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.2681 - output_loss: 0.2681 - output_rmse: 0.5178 - val_loss: 0.2597 - val_output_loss: 0.2597 - val_output_rmse: 0.5096 Epoch 104/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.2697 - output_loss: 0.2697 - output_rmse: 0.5193 - val_loss: 0.2888 - val_output_loss: 0.2888 - val_output_rmse: 0.5374 Epoch 105/1000 26/26 [==============================] - 6s 229ms/step - loss: 0.2658 - output_loss: 0.2658 - output_rmse: 0.5156 - val_loss: 0.2638 - val_output_loss: 0.2638 - val_output_rmse: 0.5136 Epoch 106/1000 26/26 [==============================] - 6s 219ms/step - loss: 0.2675 - output_loss: 0.2675 - output_rmse: 0.5172 - val_loss: 0.2707 - val_output_loss: 0.2707 - val_output_rmse: 0.5203 Epoch 107/1000 26/26 [==============================] - 6s 221ms/step - loss: 0.2633 - output_loss: 0.2633 - output_rmse: 0.5132 - val_loss: 0.2788 - val_output_loss: 0.2788 - val_output_rmse: 0.5280 Epoch 108/1000 26/26 [==============================] - 6s 216ms/step - loss: 0.2694 - output_loss: 0.2694 - output_rmse: 0.5190 - val_loss: 0.2733 - val_output_loss: 0.2733 - val_output_rmse: 0.5228 Epoch 109/1000 26/26 [==============================] - 6s 216ms/step - loss: 0.2664 - output_loss: 0.2664 - output_rmse: 0.5161 - val_loss: 0.2653 - val_output_loss: 0.2653 - val_output_rmse: 0.5151 Epoch 110/1000 26/26 [==============================] - 6s 219ms/step - loss: 0.2604 - output_loss: 0.2604 - output_rmse: 0.5103 - val_loss: 0.2609 - val_output_loss: 0.2609 - val_output_rmse: 0.5108 Epoch 111/1000 26/26 [==============================] - 6s 218ms/step - loss: 0.2578 - output_loss: 0.2578 - output_rmse: 0.5078 - val_loss: 0.2690 - val_output_loss: 0.2690 - val_output_rmse: 0.5187 Epoch 112/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.2656 - output_loss: 0.2656 - output_rmse: 0.5154 - val_loss: 0.2613 - val_output_loss: 0.2613 - val_output_rmse: 0.5112 Epoch 113/1000 26/26 [==============================] - 6s 215ms/step - loss: 0.2643 - output_loss: 0.2643 - output_rmse: 0.5141 - val_loss: 0.2546 - val_output_loss: 0.2546 - val_output_rmse: 0.5046 Epoch 114/1000 26/26 [==============================] - 6s 214ms/step - loss: 0.2584 - output_loss: 0.2584 - output_rmse: 0.5084 - val_loss: 0.2638 - val_output_loss: 0.2638 - val_output_rmse: 0.5136 Epoch 115/1000 26/26 [==============================] - 6s 220ms/step - loss: 0.2625 - output_loss: 0.2625 - output_rmse: 0.5123 - val_loss: 0.2825 - val_output_loss: 0.2825 - val_output_rmse: 0.5315 Epoch 116/1000 26/26 [==============================] - 6s 225ms/step - loss: 0.2584 - output_loss: 0.2584 - output_rmse: 0.5084 - val_loss: 0.2645 - val_output_loss: 0.2645 - val_output_rmse: 0.5143 Epoch 117/1000 26/26 [==============================] - 6s 233ms/step - loss: 0.2523 - output_loss: 0.2523 - output_rmse: 0.5023 - val_loss: 0.2572 - val_output_loss: 0.2572 - val_output_rmse: 0.5072 Epoch 118/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2657 - output_loss: 0.2657 - output_rmse: 0.5155 - val_loss: 0.2690 - val_output_loss: 0.2690 - val_output_rmse: 0.5187 Epoch 119/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.2577 - output_loss: 0.2577 - output_rmse: 0.5076 - val_loss: 0.2618 - val_output_loss: 0.2618 - val_output_rmse: 0.5117 Epoch 120/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.2579 - output_loss: 0.2579 - output_rmse: 0.5078 - val_loss: 0.2770 - val_output_loss: 0.2770 - val_output_rmse: 0.5263 Epoch 121/1000 26/26 [==============================] - 6s 235ms/step - loss: 0.2567 - output_loss: 0.2567 - output_rmse: 0.5066 - val_loss: 0.2685 - val_output_loss: 0.2685 - val_output_rmse: 0.5182 Epoch 122/1000 26/26 [==============================] - 6s 209ms/step - loss: 0.2599 - output_loss: 0.2599 - output_rmse: 0.5098 - val_loss: 0.2630 - val_output_loss: 0.2630 - val_output_rmse: 0.5128 Epoch 123/1000 26/26 [==============================] - 6s 220ms/step - loss: 0.2672 - output_loss: 0.2672 - output_rmse: 0.5169 - val_loss: 0.2754 - val_output_loss: 0.2754 - val_output_rmse: 0.5248 Epoch 124/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2718 - output_loss: 0.2718 - output_rmse: 0.5214 - val_loss: 0.2589 - val_output_loss: 0.2589 - val_output_rmse: 0.5088 Epoch 125/1000 26/26 [==============================] - 6s 227ms/step - loss: 0.2494 - output_loss: 0.2494 - output_rmse: 0.4994 - val_loss: 0.2640 - val_output_loss: 0.2640 - val_output_rmse: 0.5138 Epoch 126/1000 26/26 [==============================] - 6s 227ms/step - loss: 0.2479 - output_loss: 0.2479 - output_rmse: 0.4979 - val_loss: 0.2630 - val_output_loss: 0.2630 - val_output_rmse: 0.5129 Epoch 127/1000 26/26 [==============================] - 6s 233ms/step - loss: 0.2484 - output_loss: 0.2484 - output_rmse: 0.4984 - val_loss: 0.2596 - val_output_loss: 0.2596 - val_output_rmse: 0.5095 Epoch 128/1000 26/26 [==============================] - 6s 221ms/step - loss: 0.2516 - output_loss: 0.2516 - output_rmse: 0.5016 - val_loss: 0.2547 - val_output_loss: 0.2547 - val_output_rmse: 0.5047 Epoch 129/1000 26/26 [==============================] - 6s 221ms/step - loss: 0.2540 - output_loss: 0.2540 - output_rmse: 0.5040 - val_loss: 0.2630 - val_output_loss: 0.2630 - val_output_rmse: 0.5128
ft_periodic_encoder = FTTransformerEncoder(
numerical_features = NUMERIC_FEATURES,
categorical_features = [],
numerical_data = X_train[NUMERIC_FEATURES].values,
categorical_data = None,
y = None,
numerical_embedding_type='periodic',
numerical_bins=128,
embedding_dim=64,
depth=3,
heads=6,
attn_dropout=0.3,
ff_dropout=0.3,
explainable=True
)
# Pass th encoder to the model
ft_periodic_transformer = FTTransformer(
encoder=ft_periodic_encoder,
out_dim=1,
out_activation="relu",
)
LEARNING_RATE = 0.001
WEIGHT_DECAY = 0.0001
NUM_EPOCHS = 1000
optimizer = tfa.optimizers.AdamW(
learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
ft_periodic_transformer.compile(
optimizer = optimizer,
loss = {"output": tf.keras.losses.MeanSquaredError(name='mse'), "importances": None},
metrics= {"output": [tf.keras.metrics.RootMeanSquaredError(name='rmse')], "importances": None},
)
early = EarlyStopping(monitor="val_output_loss", mode="min", patience=16, restore_best_weights=True)
callback_list = [early]
ft_periodic_history = ft_periodic_transformer.fit(
train_dataset,
epochs=NUM_EPOCHS,
validation_data=val_dataset,
callbacks=callback_list
)
Epoch 1/1000 26/26 [==============================] - 11s 259ms/step - loss: 1.2407 - output_loss: 1.2407 - output_rmse: 1.1139 - val_loss: 0.6024 - val_output_loss: 0.6024 - val_output_rmse: 0.7761 Epoch 2/1000 26/26 [==============================] - 6s 245ms/step - loss: 0.5353 - output_loss: 0.5353 - output_rmse: 0.7316 - val_loss: 0.4126 - val_output_loss: 0.4126 - val_output_rmse: 0.6424 Epoch 3/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.4323 - output_loss: 0.4323 - output_rmse: 0.6575 - val_loss: 0.3607 - val_output_loss: 0.3607 - val_output_rmse: 0.6006 Epoch 4/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.3879 - output_loss: 0.3879 - output_rmse: 0.6228 - val_loss: 0.3430 - val_output_loss: 0.3430 - val_output_rmse: 0.5857 Epoch 5/1000 26/26 [==============================] - 6s 241ms/step - loss: 0.3541 - output_loss: 0.3541 - output_rmse: 0.5951 - val_loss: 0.3129 - val_output_loss: 0.3129 - val_output_rmse: 0.5594 Epoch 6/1000 26/26 [==============================] - 6s 236ms/step - loss: 0.3288 - output_loss: 0.3288 - output_rmse: 0.5734 - val_loss: 0.3024 - val_output_loss: 0.3024 - val_output_rmse: 0.5499 Epoch 7/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.3163 - output_loss: 0.3163 - output_rmse: 0.5624 - val_loss: 0.3017 - val_output_loss: 0.3017 - val_output_rmse: 0.5493 Epoch 8/1000 26/26 [==============================] - 6s 239ms/step - loss: 0.2997 - output_loss: 0.2997 - output_rmse: 0.5474 - val_loss: 0.2832 - val_output_loss: 0.2832 - val_output_rmse: 0.5322 Epoch 9/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.2930 - output_loss: 0.2930 - output_rmse: 0.5413 - val_loss: 0.2840 - val_output_loss: 0.2840 - val_output_rmse: 0.5329 Epoch 10/1000 26/26 [==============================] - 6s 246ms/step - loss: 0.2797 - output_loss: 0.2797 - output_rmse: 0.5289 - val_loss: 0.2738 - val_output_loss: 0.2738 - val_output_rmse: 0.5233 Epoch 11/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.2741 - output_loss: 0.2741 - output_rmse: 0.5236 - val_loss: 0.2676 - val_output_loss: 0.2676 - val_output_rmse: 0.5173 Epoch 12/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.2689 - output_loss: 0.2689 - output_rmse: 0.5185 - val_loss: 0.2651 - val_output_loss: 0.2651 - val_output_rmse: 0.5149 Epoch 13/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.2589 - output_loss: 0.2589 - output_rmse: 0.5088 - val_loss: 0.2551 - val_output_loss: 0.2551 - val_output_rmse: 0.5051 Epoch 14/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2547 - output_loss: 0.2547 - output_rmse: 0.5047 - val_loss: 0.2624 - val_output_loss: 0.2624 - val_output_rmse: 0.5123 Epoch 15/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.2456 - output_loss: 0.2456 - output_rmse: 0.4956 - val_loss: 0.2478 - val_output_loss: 0.2478 - val_output_rmse: 0.4978 Epoch 16/1000 26/26 [==============================] - 6s 240ms/step - loss: 0.2466 - output_loss: 0.2466 - output_rmse: 0.4966 - val_loss: 0.2423 - val_output_loss: 0.2423 - val_output_rmse: 0.4923 Epoch 17/1000 26/26 [==============================] - 6s 228ms/step - loss: 0.2413 - output_loss: 0.2413 - output_rmse: 0.4912 - val_loss: 0.2591 - val_output_loss: 0.2591 - val_output_rmse: 0.5090 Epoch 18/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.2378 - output_loss: 0.2378 - output_rmse: 0.4876 - val_loss: 0.2442 - val_output_loss: 0.2442 - val_output_rmse: 0.4942 Epoch 19/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.2283 - output_loss: 0.2283 - output_rmse: 0.4778 - val_loss: 0.2331 - val_output_loss: 0.2331 - val_output_rmse: 0.4828 Epoch 20/1000 26/26 [==============================] - 6s 235ms/step - loss: 0.2221 - output_loss: 0.2221 - output_rmse: 0.4712 - val_loss: 0.2320 - val_output_loss: 0.2320 - val_output_rmse: 0.4816 Epoch 21/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.2190 - output_loss: 0.2190 - output_rmse: 0.4680 - val_loss: 0.2368 - val_output_loss: 0.2368 - val_output_rmse: 0.4866 Epoch 22/1000 26/26 [==============================] - 6s 228ms/step - loss: 0.2206 - output_loss: 0.2206 - output_rmse: 0.4697 - val_loss: 0.2294 - val_output_loss: 0.2294 - val_output_rmse: 0.4789 Epoch 23/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2163 - output_loss: 0.2163 - output_rmse: 0.4651 - val_loss: 0.2380 - val_output_loss: 0.2380 - val_output_rmse: 0.4878 Epoch 24/1000 26/26 [==============================] - 6s 226ms/step - loss: 0.2129 - output_loss: 0.2129 - output_rmse: 0.4615 - val_loss: 0.2356 - val_output_loss: 0.2356 - val_output_rmse: 0.4854 Epoch 25/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.2085 - output_loss: 0.2085 - output_rmse: 0.4566 - val_loss: 0.2350 - val_output_loss: 0.2350 - val_output_rmse: 0.4847 Epoch 26/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.2054 - output_loss: 0.2054 - output_rmse: 0.4532 - val_loss: 0.2344 - val_output_loss: 0.2344 - val_output_rmse: 0.4841 Epoch 27/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.2041 - output_loss: 0.2041 - output_rmse: 0.4518 - val_loss: 0.2299 - val_output_loss: 0.2299 - val_output_rmse: 0.4795 Epoch 28/1000 26/26 [==============================] - 6s 228ms/step - loss: 0.2053 - output_loss: 0.2053 - output_rmse: 0.4531 - val_loss: 0.2346 - val_output_loss: 0.2346 - val_output_rmse: 0.4843 Epoch 29/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.1999 - output_loss: 0.1999 - output_rmse: 0.4471 - val_loss: 0.2312 - val_output_loss: 0.2312 - val_output_rmse: 0.4809 Epoch 30/1000 26/26 [==============================] - 6s 235ms/step - loss: 0.1979 - output_loss: 0.1979 - output_rmse: 0.4449 - val_loss: 0.2287 - val_output_loss: 0.2287 - val_output_rmse: 0.4782 Epoch 31/1000 26/26 [==============================] - 6s 240ms/step - loss: 0.1909 - output_loss: 0.1909 - output_rmse: 0.4369 - val_loss: 0.2397 - val_output_loss: 0.2397 - val_output_rmse: 0.4896 Epoch 32/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.1952 - output_loss: 0.1952 - output_rmse: 0.4418 - val_loss: 0.2251 - val_output_loss: 0.2251 - val_output_rmse: 0.4745 Epoch 33/1000 26/26 [==============================] - 6s 246ms/step - loss: 0.1900 - output_loss: 0.1900 - output_rmse: 0.4359 - val_loss: 0.2297 - val_output_loss: 0.2297 - val_output_rmse: 0.4792 Epoch 34/1000 26/26 [==============================] - 6s 237ms/step - loss: 0.1891 - output_loss: 0.1891 - output_rmse: 0.4348 - val_loss: 0.2364 - val_output_loss: 0.2364 - val_output_rmse: 0.4863 Epoch 35/1000 26/26 [==============================] - 6s 231ms/step - loss: 0.1883 - output_loss: 0.1883 - output_rmse: 0.4339 - val_loss: 0.2250 - val_output_loss: 0.2250 - val_output_rmse: 0.4743 Epoch 36/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.1850 - output_loss: 0.1850 - output_rmse: 0.4301 - val_loss: 0.2279 - val_output_loss: 0.2279 - val_output_rmse: 0.4774 Epoch 37/1000 26/26 [==============================] - 6s 228ms/step - loss: 0.1819 - output_loss: 0.1819 - output_rmse: 0.4264 - val_loss: 0.2227 - val_output_loss: 0.2227 - val_output_rmse: 0.4719 Epoch 38/1000 26/26 [==============================] - 6s 237ms/step - loss: 0.1808 - output_loss: 0.1808 - output_rmse: 0.4252 - val_loss: 0.2347 - val_output_loss: 0.2347 - val_output_rmse: 0.4844 Epoch 39/1000 26/26 [==============================] - 8s 320ms/step - loss: 0.1821 - output_loss: 0.1821 - output_rmse: 0.4267 - val_loss: 0.2348 - val_output_loss: 0.2348 - val_output_rmse: 0.4845 Epoch 40/1000 26/26 [==============================] - 6s 223ms/step - loss: 0.1742 - output_loss: 0.1742 - output_rmse: 0.4174 - val_loss: 0.2212 - val_output_loss: 0.2212 - val_output_rmse: 0.4704 Epoch 41/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.1740 - output_loss: 0.1740 - output_rmse: 0.4172 - val_loss: 0.2316 - val_output_loss: 0.2316 - val_output_rmse: 0.4813 Epoch 42/1000 26/26 [==============================] - 6s 235ms/step - loss: 0.1730 - output_loss: 0.1730 - output_rmse: 0.4160 - val_loss: 0.2302 - val_output_loss: 0.2302 - val_output_rmse: 0.4798 Epoch 43/1000 26/26 [==============================] - 7s 251ms/step - loss: 0.1702 - output_loss: 0.1702 - output_rmse: 0.4126 - val_loss: 0.2322 - val_output_loss: 0.2322 - val_output_rmse: 0.4819 Epoch 44/1000 26/26 [==============================] - 7s 268ms/step - loss: 0.1712 - output_loss: 0.1712 - output_rmse: 0.4138 - val_loss: 0.2409 - val_output_loss: 0.2409 - val_output_rmse: 0.4908 Epoch 45/1000 26/26 [==============================] - 6s 246ms/step - loss: 0.1658 - output_loss: 0.1658 - output_rmse: 0.4071 - val_loss: 0.2285 - val_output_loss: 0.2285 - val_output_rmse: 0.4780 Epoch 46/1000 26/26 [==============================] - 6s 237ms/step - loss: 0.1639 - output_loss: 0.1639 - output_rmse: 0.4049 - val_loss: 0.2276 - val_output_loss: 0.2276 - val_output_rmse: 0.4770 Epoch 47/1000 26/26 [==============================] - 6s 222ms/step - loss: 0.1656 - output_loss: 0.1656 - output_rmse: 0.4069 - val_loss: 0.2269 - val_output_loss: 0.2269 - val_output_rmse: 0.4763 Epoch 48/1000 26/26 [==============================] - 6s 232ms/step - loss: 0.1671 - output_loss: 0.1671 - output_rmse: 0.4088 - val_loss: 0.2274 - val_output_loss: 0.2274 - val_output_rmse: 0.4769 Epoch 49/1000 26/26 [==============================] - 6s 228ms/step - loss: 0.1649 - output_loss: 0.1649 - output_rmse: 0.4060 - val_loss: 0.2293 - val_output_loss: 0.2293 - val_output_rmse: 0.4788 Epoch 50/1000 26/26 [==============================] - 6s 240ms/step - loss: 0.1631 - output_loss: 0.1631 - output_rmse: 0.4038 - val_loss: 0.2289 - val_output_loss: 0.2289 - val_output_rmse: 0.4784 Epoch 51/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.1609 - output_loss: 0.1609 - output_rmse: 0.4011 - val_loss: 0.2238 - val_output_loss: 0.2238 - val_output_rmse: 0.4731 Epoch 52/1000 26/26 [==============================] - 7s 248ms/step - loss: 0.1575 - output_loss: 0.1575 - output_rmse: 0.3969 - val_loss: 0.2229 - val_output_loss: 0.2229 - val_output_rmse: 0.4721 Epoch 53/1000 26/26 [==============================] - 6s 243ms/step - loss: 0.1609 - output_loss: 0.1609 - output_rmse: 0.4012 - val_loss: 0.2400 - val_output_loss: 0.2400 - val_output_rmse: 0.4899 Epoch 54/1000 26/26 [==============================] - 7s 248ms/step - loss: 0.1548 - output_loss: 0.1548 - output_rmse: 0.3935 - val_loss: 0.2303 - val_output_loss: 0.2303 - val_output_rmse: 0.4799 Epoch 55/1000 26/26 [==============================] - 6s 242ms/step - loss: 0.1566 - output_loss: 0.1566 - output_rmse: 0.3957 - val_loss: 0.2180 - val_output_loss: 0.2180 - val_output_rmse: 0.4669 Epoch 56/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.1513 - output_loss: 0.1513 - output_rmse: 0.3890 - val_loss: 0.2211 - val_output_loss: 0.2211 - val_output_rmse: 0.4703 Epoch 57/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.1546 - output_loss: 0.1546 - output_rmse: 0.3932 - val_loss: 0.2226 - val_output_loss: 0.2226 - val_output_rmse: 0.4718 Epoch 58/1000 26/26 [==============================] - 6s 241ms/step - loss: 0.1513 - output_loss: 0.1513 - output_rmse: 0.3890 - val_loss: 0.2228 - val_output_loss: 0.2228 - val_output_rmse: 0.4720 Epoch 59/1000 26/26 [==============================] - 6s 246ms/step - loss: 0.1504 - output_loss: 0.1504 - output_rmse: 0.3879 - val_loss: 0.2208 - val_output_loss: 0.2208 - val_output_rmse: 0.4699 Epoch 60/1000 26/26 [==============================] - 6s 239ms/step - loss: 0.1465 - output_loss: 0.1465 - output_rmse: 0.3828 - val_loss: 0.2309 - val_output_loss: 0.2309 - val_output_rmse: 0.4805 Epoch 61/1000 26/26 [==============================] - 6s 240ms/step - loss: 0.1473 - output_loss: 0.1473 - output_rmse: 0.3838 - val_loss: 0.2387 - val_output_loss: 0.2387 - val_output_rmse: 0.4886 Epoch 62/1000 26/26 [==============================] - 7s 251ms/step - loss: 0.1471 - output_loss: 0.1471 - output_rmse: 0.3836 - val_loss: 0.2297 - val_output_loss: 0.2297 - val_output_rmse: 0.4793 Epoch 63/1000 26/26 [==============================] - 7s 249ms/step - loss: 0.1448 - output_loss: 0.1448 - output_rmse: 0.3805 - val_loss: 0.2314 - val_output_loss: 0.2314 - val_output_rmse: 0.4811 Epoch 64/1000 26/26 [==============================] - 6s 242ms/step - loss: 0.1428 - output_loss: 0.1428 - output_rmse: 0.3779 - val_loss: 0.2265 - val_output_loss: 0.2265 - val_output_rmse: 0.4759 Epoch 65/1000 26/26 [==============================] - 6s 233ms/step - loss: 0.1395 - output_loss: 0.1395 - output_rmse: 0.3735 - val_loss: 0.2355 - val_output_loss: 0.2355 - val_output_rmse: 0.4852 Epoch 66/1000 26/26 [==============================] - 6s 230ms/step - loss: 0.1425 - output_loss: 0.1425 - output_rmse: 0.3775 - val_loss: 0.2320 - val_output_loss: 0.2320 - val_output_rmse: 0.4817 Epoch 67/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.1378 - output_loss: 0.1378 - output_rmse: 0.3712 - val_loss: 0.2306 - val_output_loss: 0.2306 - val_output_rmse: 0.4802 Epoch 68/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.1375 - output_loss: 0.1375 - output_rmse: 0.3708 - val_loss: 0.2264 - val_output_loss: 0.2264 - val_output_rmse: 0.4758 Epoch 69/1000 26/26 [==============================] - 6s 242ms/step - loss: 0.1373 - output_loss: 0.1373 - output_rmse: 0.3705 - val_loss: 0.2245 - val_output_loss: 0.2245 - val_output_rmse: 0.4739 Epoch 70/1000 26/26 [==============================] - 6s 238ms/step - loss: 0.1385 - output_loss: 0.1385 - output_rmse: 0.3722 - val_loss: 0.2299 - val_output_loss: 0.2299 - val_output_rmse: 0.4795 Epoch 71/1000 26/26 [==============================] - 7s 249ms/step - loss: 0.1361 - output_loss: 0.1361 - output_rmse: 0.3689 - val_loss: 0.2340 - val_output_loss: 0.2340 - val_output_rmse: 0.4838
ft_pleq_encoder = FTTransformerEncoder(
numerical_features = NUMERIC_FEATURES,
categorical_features = [],
numerical_data = X_train[NUMERIC_FEATURES].values,
categorical_data = None,
y = None,
numerical_embedding_type='ple',
numerical_bins=128,
embedding_dim=64,
depth=3,
heads=6,
attn_dropout=0.3,
ff_dropout=0.3,
explainable=True
)
# Pass the encoder to the model
ft_pleq_transformer = FTTransformer(
encoder=ft_pleq_encoder,
out_dim=1,
out_activation="relu",
)
LEARNING_RATE = 0.001
WEIGHT_DECAY = 0.0001
NUM_EPOCHS = 1000
optimizer = tfa.optimizers.AdamW(
learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
ft_pleq_transformer.compile(
optimizer = optimizer,
loss = {"output": tf.keras.losses.MeanSquaredError(name='mse'), "importances": None},
metrics= {"output": [tf.keras.metrics.RootMeanSquaredError(name='rmse')], "importances": None},
)
early = EarlyStopping(monitor="val_loss", mode="min", patience=20, restore_best_weights=True)
callback_list = [early]
ft_pleq_history = ft_pleq_transformer.fit(
train_dataset,
epochs=NUM_EPOCHS,
validation_data=val_dataset,
callbacks=callback_list
)
Epoch 1/1000 26/26 [==============================] - 191s 3s/step - loss: 1.2836 - output_loss: 1.2836 - output_rmse: 1.1329 - val_loss: 0.4857 - val_output_loss: 0.4857 - val_output_rmse: 0.6969 Epoch 2/1000 26/26 [==============================] - 8s 289ms/step - loss: 0.4944 - output_loss: 0.4944 - output_rmse: 0.7032 - val_loss: 0.3911 - val_output_loss: 0.3911 - val_output_rmse: 0.6254 Epoch 3/1000 26/26 [==============================] - 7s 284ms/step - loss: 0.3846 - output_loss: 0.3846 - output_rmse: 0.6202 - val_loss: 0.3391 - val_output_loss: 0.3391 - val_output_rmse: 0.5823 Epoch 4/1000 26/26 [==============================] - 7s 272ms/step - loss: 0.3544 - output_loss: 0.3544 - output_rmse: 0.5953 - val_loss: 0.3212 - val_output_loss: 0.3212 - val_output_rmse: 0.5668 Epoch 5/1000 26/26 [==============================] - 7s 272ms/step - loss: 0.3272 - output_loss: 0.3272 - output_rmse: 0.5720 - val_loss: 0.2962 - val_output_loss: 0.2962 - val_output_rmse: 0.5443 Epoch 6/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.3298 - output_loss: 0.3298 - output_rmse: 0.5743 - val_loss: 0.2850 - val_output_loss: 0.2850 - val_output_rmse: 0.5339 Epoch 7/1000 26/26 [==============================] - 7s 269ms/step - loss: 0.3015 - output_loss: 0.3015 - output_rmse: 0.5491 - val_loss: 0.2822 - val_output_loss: 0.2822 - val_output_rmse: 0.5312 Epoch 8/1000 26/26 [==============================] - 7s 283ms/step - loss: 0.2922 - output_loss: 0.2922 - output_rmse: 0.5405 - val_loss: 0.2719 - val_output_loss: 0.2719 - val_output_rmse: 0.5214 Epoch 9/1000 26/26 [==============================] - 8s 290ms/step - loss: 0.2806 - output_loss: 0.2806 - output_rmse: 0.5297 - val_loss: 0.2698 - val_output_loss: 0.2698 - val_output_rmse: 0.5194 Epoch 10/1000 26/26 [==============================] - 7s 278ms/step - loss: 0.2699 - output_loss: 0.2699 - output_rmse: 0.5196 - val_loss: 0.2548 - val_output_loss: 0.2548 - val_output_rmse: 0.5048 Epoch 11/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.2626 - output_loss: 0.2626 - output_rmse: 0.5124 - val_loss: 0.2889 - val_output_loss: 0.2889 - val_output_rmse: 0.5375 Epoch 12/1000 26/26 [==============================] - 7s 276ms/step - loss: 0.2554 - output_loss: 0.2554 - output_rmse: 0.5053 - val_loss: 0.2409 - val_output_loss: 0.2409 - val_output_rmse: 0.4908 Epoch 13/1000 26/26 [==============================] - 7s 270ms/step - loss: 0.2481 - output_loss: 0.2481 - output_rmse: 0.4981 - val_loss: 0.2448 - val_output_loss: 0.2448 - val_output_rmse: 0.4948 Epoch 14/1000 26/26 [==============================] - 7s 283ms/step - loss: 0.2408 - output_loss: 0.2408 - output_rmse: 0.4907 - val_loss: 0.2598 - val_output_loss: 0.2598 - val_output_rmse: 0.5097 Epoch 15/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.2330 - output_loss: 0.2330 - output_rmse: 0.4827 - val_loss: 0.2340 - val_output_loss: 0.2340 - val_output_rmse: 0.4838 Epoch 16/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.2277 - output_loss: 0.2277 - output_rmse: 0.4772 - val_loss: 0.2311 - val_output_loss: 0.2311 - val_output_rmse: 0.4807 Epoch 17/1000 26/26 [==============================] - 8s 291ms/step - loss: 0.2265 - output_loss: 0.2265 - output_rmse: 0.4760 - val_loss: 0.2266 - val_output_loss: 0.2266 - val_output_rmse: 0.4760 Epoch 18/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.2233 - output_loss: 0.2233 - output_rmse: 0.4726 - val_loss: 0.2424 - val_output_loss: 0.2424 - val_output_rmse: 0.4923 Epoch 19/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.2210 - output_loss: 0.2210 - output_rmse: 0.4701 - val_loss: 0.2247 - val_output_loss: 0.2247 - val_output_rmse: 0.4740 Epoch 20/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.2152 - output_loss: 0.2152 - output_rmse: 0.4639 - val_loss: 0.2561 - val_output_loss: 0.2561 - val_output_rmse: 0.5061 Epoch 21/1000 26/26 [==============================] - 7s 275ms/step - loss: 0.2238 - output_loss: 0.2238 - output_rmse: 0.4730 - val_loss: 0.2261 - val_output_loss: 0.2261 - val_output_rmse: 0.4755 Epoch 22/1000 26/26 [==============================] - 7s 269ms/step - loss: 0.2136 - output_loss: 0.2136 - output_rmse: 0.4621 - val_loss: 0.2294 - val_output_loss: 0.2294 - val_output_rmse: 0.4790 Epoch 23/1000 26/26 [==============================] - 7s 278ms/step - loss: 0.2091 - output_loss: 0.2091 - output_rmse: 0.4572 - val_loss: 0.2231 - val_output_loss: 0.2231 - val_output_rmse: 0.4724 Epoch 24/1000 26/26 [==============================] - 8s 289ms/step - loss: 0.2010 - output_loss: 0.2010 - output_rmse: 0.4484 - val_loss: 0.2244 - val_output_loss: 0.2244 - val_output_rmse: 0.4737 Epoch 25/1000 26/26 [==============================] - 8s 320ms/step - loss: 0.1986 - output_loss: 0.1986 - output_rmse: 0.4457 - val_loss: 0.2174 - val_output_loss: 0.2174 - val_output_rmse: 0.4662 Epoch 26/1000 26/26 [==============================] - 7s 275ms/step - loss: 0.2007 - output_loss: 0.2007 - output_rmse: 0.4480 - val_loss: 0.2135 - val_output_loss: 0.2135 - val_output_rmse: 0.4621 Epoch 27/1000 26/26 [==============================] - 7s 283ms/step - loss: 0.2003 - output_loss: 0.2003 - output_rmse: 0.4476 - val_loss: 0.2185 - val_output_loss: 0.2185 - val_output_rmse: 0.4674 Epoch 28/1000 26/26 [==============================] - 8s 294ms/step - loss: 0.1987 - output_loss: 0.1987 - output_rmse: 0.4458 - val_loss: 0.2312 - val_output_loss: 0.2312 - val_output_rmse: 0.4808 Epoch 29/1000 26/26 [==============================] - 7s 266ms/step - loss: 0.1977 - output_loss: 0.1977 - output_rmse: 0.4446 - val_loss: 0.2130 - val_output_loss: 0.2130 - val_output_rmse: 0.4615 Epoch 30/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.1872 - output_loss: 0.1872 - output_rmse: 0.4327 - val_loss: 0.2211 - val_output_loss: 0.2211 - val_output_rmse: 0.4702 Epoch 31/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.1867 - output_loss: 0.1867 - output_rmse: 0.4321 - val_loss: 0.2102 - val_output_loss: 0.2102 - val_output_rmse: 0.4585 Epoch 32/1000 26/26 [==============================] - 7s 266ms/step - loss: 0.1870 - output_loss: 0.1870 - output_rmse: 0.4324 - val_loss: 0.2166 - val_output_loss: 0.2166 - val_output_rmse: 0.4654 Epoch 33/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.1826 - output_loss: 0.1826 - output_rmse: 0.4273 - val_loss: 0.2183 - val_output_loss: 0.2183 - val_output_rmse: 0.4672 Epoch 34/1000 26/26 [==============================] - 7s 275ms/step - loss: 0.1793 - output_loss: 0.1793 - output_rmse: 0.4234 - val_loss: 0.2127 - val_output_loss: 0.2127 - val_output_rmse: 0.4612 Epoch 35/1000 26/26 [==============================] - 7s 278ms/step - loss: 0.1828 - output_loss: 0.1828 - output_rmse: 0.4276 - val_loss: 0.2250 - val_output_loss: 0.2250 - val_output_rmse: 0.4744 Epoch 36/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.1798 - output_loss: 0.1798 - output_rmse: 0.4240 - val_loss: 0.2110 - val_output_loss: 0.2110 - val_output_rmse: 0.4594 Epoch 37/1000 26/26 [==============================] - 7s 284ms/step - loss: 0.1797 - output_loss: 0.1797 - output_rmse: 0.4239 - val_loss: 0.2227 - val_output_loss: 0.2227 - val_output_rmse: 0.4719 Epoch 38/1000 26/26 [==============================] - 8s 292ms/step - loss: 0.1803 - output_loss: 0.1803 - output_rmse: 0.4247 - val_loss: 0.2155 - val_output_loss: 0.2155 - val_output_rmse: 0.4642 Epoch 39/1000 26/26 [==============================] - 8s 317ms/step - loss: 0.1804 - output_loss: 0.1804 - output_rmse: 0.4248 - val_loss: 0.2079 - val_output_loss: 0.2079 - val_output_rmse: 0.4559 Epoch 40/1000 26/26 [==============================] - 8s 288ms/step - loss: 0.1812 - output_loss: 0.1812 - output_rmse: 0.4257 - val_loss: 0.2095 - val_output_loss: 0.2095 - val_output_rmse: 0.4577 Epoch 41/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.1753 - output_loss: 0.1753 - output_rmse: 0.4187 - val_loss: 0.2144 - val_output_loss: 0.2144 - val_output_rmse: 0.4630 Epoch 42/1000 26/26 [==============================] - 7s 269ms/step - loss: 0.1701 - output_loss: 0.1701 - output_rmse: 0.4124 - val_loss: 0.2085 - val_output_loss: 0.2085 - val_output_rmse: 0.4566 Epoch 43/1000 26/26 [==============================] - 7s 253ms/step - loss: 0.1683 - output_loss: 0.1683 - output_rmse: 0.4103 - val_loss: 0.2118 - val_output_loss: 0.2118 - val_output_rmse: 0.4602 Epoch 44/1000 26/26 [==============================] - 7s 250ms/step - loss: 0.1701 - output_loss: 0.1701 - output_rmse: 0.4124 - val_loss: 0.2156 - val_output_loss: 0.2156 - val_output_rmse: 0.4643 Epoch 45/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1692 - output_loss: 0.1692 - output_rmse: 0.4113 - val_loss: 0.2110 - val_output_loss: 0.2110 - val_output_rmse: 0.4593 Epoch 46/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.1734 - output_loss: 0.1734 - output_rmse: 0.4164 - val_loss: 0.2169 - val_output_loss: 0.2169 - val_output_rmse: 0.4657 Epoch 47/1000 26/26 [==============================] - 7s 266ms/step - loss: 0.1678 - output_loss: 0.1678 - output_rmse: 0.4096 - val_loss: 0.2120 - val_output_loss: 0.2120 - val_output_rmse: 0.4604 Epoch 48/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.1641 - output_loss: 0.1641 - output_rmse: 0.4050 - val_loss: 0.2170 - val_output_loss: 0.2170 - val_output_rmse: 0.4659 Epoch 49/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.1667 - output_loss: 0.1667 - output_rmse: 0.4083 - val_loss: 0.2077 - val_output_loss: 0.2077 - val_output_rmse: 0.4558 Epoch 50/1000 26/26 [==============================] - 7s 271ms/step - loss: 0.1673 - output_loss: 0.1673 - output_rmse: 0.4090 - val_loss: 0.2251 - val_output_loss: 0.2251 - val_output_rmse: 0.4744 Epoch 51/1000 26/26 [==============================] - 7s 273ms/step - loss: 0.1673 - output_loss: 0.1673 - output_rmse: 0.4091 - val_loss: 0.2204 - val_output_loss: 0.2204 - val_output_rmse: 0.4695 Epoch 52/1000 26/26 [==============================] - 7s 273ms/step - loss: 0.1624 - output_loss: 0.1624 - output_rmse: 0.4030 - val_loss: 0.2065 - val_output_loss: 0.2065 - val_output_rmse: 0.4544 Epoch 53/1000 26/26 [==============================] - 7s 264ms/step - loss: 0.1587 - output_loss: 0.1587 - output_rmse: 0.3983 - val_loss: 0.2187 - val_output_loss: 0.2187 - val_output_rmse: 0.4677 Epoch 54/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1652 - output_loss: 0.1652 - output_rmse: 0.4065 - val_loss: 0.2165 - val_output_loss: 0.2165 - val_output_rmse: 0.4653 Epoch 55/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.1585 - output_loss: 0.1585 - output_rmse: 0.3981 - val_loss: 0.2273 - val_output_loss: 0.2273 - val_output_rmse: 0.4768 Epoch 56/1000 26/26 [==============================] - 7s 263ms/step - loss: 0.1571 - output_loss: 0.1571 - output_rmse: 0.3964 - val_loss: 0.2124 - val_output_loss: 0.2124 - val_output_rmse: 0.4608 Epoch 57/1000 26/26 [==============================] - 7s 264ms/step - loss: 0.1505 - output_loss: 0.1505 - output_rmse: 0.3879 - val_loss: 0.2165 - val_output_loss: 0.2165 - val_output_rmse: 0.4653 Epoch 58/1000 26/26 [==============================] - 7s 263ms/step - loss: 0.1573 - output_loss: 0.1573 - output_rmse: 0.3966 - val_loss: 0.2274 - val_output_loss: 0.2274 - val_output_rmse: 0.4768 Epoch 59/1000 26/26 [==============================] - 7s 258ms/step - loss: 0.1541 - output_loss: 0.1541 - output_rmse: 0.3926 - val_loss: 0.2194 - val_output_loss: 0.2194 - val_output_rmse: 0.4684 Epoch 60/1000 26/26 [==============================] - 7s 262ms/step - loss: 0.1521 - output_loss: 0.1521 - output_rmse: 0.3900 - val_loss: 0.2198 - val_output_loss: 0.2198 - val_output_rmse: 0.4688 Epoch 61/1000 26/26 [==============================] - 7s 258ms/step - loss: 0.1537 - output_loss: 0.1537 - output_rmse: 0.3921 - val_loss: 0.2287 - val_output_loss: 0.2287 - val_output_rmse: 0.4782 Epoch 62/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.1570 - output_loss: 0.1570 - output_rmse: 0.3963 - val_loss: 0.2150 - val_output_loss: 0.2150 - val_output_rmse: 0.4637 Epoch 63/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1483 - output_loss: 0.1483 - output_rmse: 0.3851 - val_loss: 0.2140 - val_output_loss: 0.2140 - val_output_rmse: 0.4626 Epoch 64/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1461 - output_loss: 0.1461 - output_rmse: 0.3822 - val_loss: 0.2185 - val_output_loss: 0.2185 - val_output_rmse: 0.4674 Epoch 65/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.1440 - output_loss: 0.1440 - output_rmse: 0.3795 - val_loss: 0.2200 - val_output_loss: 0.2200 - val_output_rmse: 0.4691 Epoch 66/1000 26/26 [==============================] - 7s 261ms/step - loss: 0.1454 - output_loss: 0.1454 - output_rmse: 0.3813 - val_loss: 0.2211 - val_output_loss: 0.2211 - val_output_rmse: 0.4702 Epoch 67/1000 26/26 [==============================] - 7s 266ms/step - loss: 0.1461 - output_loss: 0.1461 - output_rmse: 0.3822 - val_loss: 0.2148 - val_output_loss: 0.2148 - val_output_rmse: 0.4634 Epoch 68/1000 26/26 [==============================] - 7s 259ms/step - loss: 0.1436 - output_loss: 0.1436 - output_rmse: 0.3790 - val_loss: 0.2195 - val_output_loss: 0.2195 - val_output_rmse: 0.4685 Epoch 69/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1390 - output_loss: 0.1390 - output_rmse: 0.3728 - val_loss: 0.2192 - val_output_loss: 0.2192 - val_output_rmse: 0.4682 Epoch 70/1000 26/26 [==============================] - 7s 258ms/step - loss: 0.1427 - output_loss: 0.1427 - output_rmse: 0.3777 - val_loss: 0.2263 - val_output_loss: 0.2263 - val_output_rmse: 0.4757 Epoch 71/1000 26/26 [==============================] - 7s 253ms/step - loss: 0.1414 - output_loss: 0.1414 - output_rmse: 0.3761 - val_loss: 0.2194 - val_output_loss: 0.2194 - val_output_rmse: 0.4684 Epoch 72/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.1393 - output_loss: 0.1393 - output_rmse: 0.3733 - val_loss: 0.2117 - val_output_loss: 0.2117 - val_output_rmse: 0.4601
ft_plet_encoder = FTTransformerEncoder(
numerical_features = NUMERIC_FEATURES,
categorical_features = [],
numerical_data = X_train[NUMERIC_FEATURES].values,
categorical_data = None,
y = X_train[LABEL].values,
task='regression',
numerical_embedding_type='ple',
numerical_bins=128,
embedding_dim=64,
depth=3,
heads=6,
attn_dropout=0.3,
ff_dropout=0.3,
ple_tree_params = {
"min_samples_leaf": 20,
},
explainable=True
)
# Pass th encoder to the model
ft_plet_transformer = FTTransformer(
encoder=ft_plet_encoder,
out_dim=1,
out_activation=None,
)
LEARNING_RATE = 0.001
WEIGHT_DECAY = 0.0001
NUM_EPOCHS = 1000
optimizer = tfa.optimizers.AdamW(
learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
ft_plet_transformer.compile(
optimizer = optimizer,
loss = {"output": tf.keras.losses.MeanSquaredError(name='mse'), "importances": None},
metrics= {"output": [tf.keras.metrics.RootMeanSquaredError(name='rmse')], "importances": None},
)
early = EarlyStopping(monitor="val_output_loss", mode="min", patience=20, restore_best_weights=True)
callback_list = [early]
ft_plet_history = ft_plet_transformer.fit(
train_dataset,
epochs=NUM_EPOCHS,
validation_data=val_dataset,
callbacks=callback_list
)
Epoch 1/1000 26/26 [==============================] - 298s 4s/step - loss: 1.3294 - output_loss: 1.3294 - output_rmse: 1.1530 - val_loss: 0.6608 - val_output_loss: 0.6608 - val_output_rmse: 0.8129 Epoch 2/1000 26/26 [==============================] - 9s 303ms/step - loss: 0.5900 - output_loss: 0.5900 - output_rmse: 0.7681 - val_loss: 0.4128 - val_output_loss: 0.4128 - val_output_rmse: 0.6425 Epoch 3/1000 26/26 [==============================] - 8s 288ms/step - loss: 0.4200 - output_loss: 0.4200 - output_rmse: 0.6481 - val_loss: 0.3458 - val_output_loss: 0.3458 - val_output_rmse: 0.5881 Epoch 4/1000 26/26 [==============================] - 8s 297ms/step - loss: 0.3787 - output_loss: 0.3787 - output_rmse: 0.6154 - val_loss: 0.3258 - val_output_loss: 0.3258 - val_output_rmse: 0.5708 Epoch 5/1000 26/26 [==============================] - 7s 283ms/step - loss: 0.3394 - output_loss: 0.3394 - output_rmse: 0.5826 - val_loss: 0.3331 - val_output_loss: 0.3331 - val_output_rmse: 0.5772 Epoch 6/1000 26/26 [==============================] - 7s 271ms/step - loss: 0.3221 - output_loss: 0.3221 - output_rmse: 0.5675 - val_loss: 0.3090 - val_output_loss: 0.3090 - val_output_rmse: 0.5559 Epoch 7/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.3047 - output_loss: 0.3047 - output_rmse: 0.5520 - val_loss: 0.2959 - val_output_loss: 0.2959 - val_output_rmse: 0.5440 Epoch 8/1000 26/26 [==============================] - 8s 295ms/step - loss: 0.2948 - output_loss: 0.2948 - output_rmse: 0.5429 - val_loss: 0.2946 - val_output_loss: 0.2946 - val_output_rmse: 0.5428 Epoch 9/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.2807 - output_loss: 0.2807 - output_rmse: 0.5298 - val_loss: 0.2755 - val_output_loss: 0.2755 - val_output_rmse: 0.5249 Epoch 10/1000 26/26 [==============================] - 7s 277ms/step - loss: 0.2758 - output_loss: 0.2758 - output_rmse: 0.5252 - val_loss: 0.2732 - val_output_loss: 0.2732 - val_output_rmse: 0.5227 Epoch 11/1000 26/26 [==============================] - 8s 290ms/step - loss: 0.2667 - output_loss: 0.2667 - output_rmse: 0.5164 - val_loss: 0.2720 - val_output_loss: 0.2720 - val_output_rmse: 0.5216 Epoch 12/1000 26/26 [==============================] - 7s 262ms/step - loss: 0.2606 - output_loss: 0.2606 - output_rmse: 0.5105 - val_loss: 0.2546 - val_output_loss: 0.2546 - val_output_rmse: 0.5046 Epoch 13/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.2732 - output_loss: 0.2732 - output_rmse: 0.5227 - val_loss: 0.3019 - val_output_loss: 0.3019 - val_output_rmse: 0.5495 Epoch 14/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.2659 - output_loss: 0.2659 - output_rmse: 0.5156 - val_loss: 0.2603 - val_output_loss: 0.2603 - val_output_rmse: 0.5102 Epoch 15/1000 26/26 [==============================] - 7s 261ms/step - loss: 0.2458 - output_loss: 0.2458 - output_rmse: 0.4958 - val_loss: 0.2555 - val_output_loss: 0.2555 - val_output_rmse: 0.5054 Epoch 16/1000 26/26 [==============================] - 7s 260ms/step - loss: 0.2438 - output_loss: 0.2438 - output_rmse: 0.4938 - val_loss: 0.2479 - val_output_loss: 0.2479 - val_output_rmse: 0.4978 Epoch 17/1000 26/26 [==============================] - 7s 252ms/step - loss: 0.2377 - output_loss: 0.2377 - output_rmse: 0.4876 - val_loss: 0.2439 - val_output_loss: 0.2439 - val_output_rmse: 0.4938 Epoch 18/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.2422 - output_loss: 0.2422 - output_rmse: 0.4922 - val_loss: 0.2465 - val_output_loss: 0.2465 - val_output_rmse: 0.4965 Epoch 19/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.2293 - output_loss: 0.2293 - output_rmse: 0.4788 - val_loss: 0.2381 - val_output_loss: 0.2381 - val_output_rmse: 0.4879 Epoch 20/1000 26/26 [==============================] - 7s 256ms/step - loss: 0.2210 - output_loss: 0.2210 - output_rmse: 0.4701 - val_loss: 0.2447 - val_output_loss: 0.2447 - val_output_rmse: 0.4946 Epoch 21/1000 26/26 [==============================] - 7s 260ms/step - loss: 0.2219 - output_loss: 0.2219 - output_rmse: 0.4711 - val_loss: 0.2287 - val_output_loss: 0.2287 - val_output_rmse: 0.4782 Epoch 22/1000 26/26 [==============================] - 7s 261ms/step - loss: 0.2160 - output_loss: 0.2160 - output_rmse: 0.4648 - val_loss: 0.2365 - val_output_loss: 0.2365 - val_output_rmse: 0.4863 Epoch 23/1000 26/26 [==============================] - 7s 263ms/step - loss: 0.2157 - output_loss: 0.2157 - output_rmse: 0.4644 - val_loss: 0.2325 - val_output_loss: 0.2325 - val_output_rmse: 0.4822 Epoch 24/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.2124 - output_loss: 0.2124 - output_rmse: 0.4609 - val_loss: 0.2319 - val_output_loss: 0.2319 - val_output_rmse: 0.4816 Epoch 25/1000 26/26 [==============================] - 8s 295ms/step - loss: 0.2086 - output_loss: 0.2086 - output_rmse: 0.4567 - val_loss: 0.2334 - val_output_loss: 0.2334 - val_output_rmse: 0.4832 Epoch 26/1000 26/26 [==============================] - 7s 258ms/step - loss: 0.2034 - output_loss: 0.2034 - output_rmse: 0.4510 - val_loss: 0.2241 - val_output_loss: 0.2241 - val_output_rmse: 0.4733 Epoch 27/1000 26/26 [==============================] - 7s 259ms/step - loss: 0.2000 - output_loss: 0.2000 - output_rmse: 0.4472 - val_loss: 0.2416 - val_output_loss: 0.2416 - val_output_rmse: 0.4915 Epoch 28/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.2068 - output_loss: 0.2068 - output_rmse: 0.4548 - val_loss: 0.2282 - val_output_loss: 0.2282 - val_output_rmse: 0.4777 Epoch 29/1000 26/26 [==============================] - 7s 271ms/step - loss: 0.1995 - output_loss: 0.1995 - output_rmse: 0.4467 - val_loss: 0.2275 - val_output_loss: 0.2275 - val_output_rmse: 0.4770 Epoch 30/1000 26/26 [==============================] - 7s 255ms/step - loss: 0.1986 - output_loss: 0.1986 - output_rmse: 0.4457 - val_loss: 0.2212 - val_output_loss: 0.2212 - val_output_rmse: 0.4703 Epoch 31/1000 26/26 [==============================] - 7s 260ms/step - loss: 0.1973 - output_loss: 0.1973 - output_rmse: 0.4442 - val_loss: 0.2206 - val_output_loss: 0.2206 - val_output_rmse: 0.4697 Epoch 32/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.2030 - output_loss: 0.2030 - output_rmse: 0.4506 - val_loss: 0.2353 - val_output_loss: 0.2353 - val_output_rmse: 0.4851 Epoch 33/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1977 - output_loss: 0.1977 - output_rmse: 0.4447 - val_loss: 0.2182 - val_output_loss: 0.2182 - val_output_rmse: 0.4671 Epoch 34/1000 26/26 [==============================] - 7s 254ms/step - loss: 0.1892 - output_loss: 0.1892 - output_rmse: 0.4350 - val_loss: 0.2196 - val_output_loss: 0.2196 - val_output_rmse: 0.4686 Epoch 35/1000 26/26 [==============================] - 7s 260ms/step - loss: 0.1848 - output_loss: 0.1848 - output_rmse: 0.4299 - val_loss: 0.2259 - val_output_loss: 0.2259 - val_output_rmse: 0.4753 Epoch 36/1000 26/26 [==============================] - 7s 270ms/step - loss: 0.1835 - output_loss: 0.1835 - output_rmse: 0.4284 - val_loss: 0.2150 - val_output_loss: 0.2150 - val_output_rmse: 0.4636 Epoch 37/1000 26/26 [==============================] - 8s 287ms/step - loss: 0.1852 - output_loss: 0.1852 - output_rmse: 0.4304 - val_loss: 0.2164 - val_output_loss: 0.2164 - val_output_rmse: 0.4652 Epoch 38/1000 26/26 [==============================] - 7s 267ms/step - loss: 0.1817 - output_loss: 0.1817 - output_rmse: 0.4263 - val_loss: 0.2225 - val_output_loss: 0.2225 - val_output_rmse: 0.4717 Epoch 39/1000 26/26 [==============================] - 7s 269ms/step - loss: 0.1801 - output_loss: 0.1801 - output_rmse: 0.4244 - val_loss: 0.2191 - val_output_loss: 0.2191 - val_output_rmse: 0.4681 Epoch 40/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.1835 - output_loss: 0.1835 - output_rmse: 0.4284 - val_loss: 0.2173 - val_output_loss: 0.2173 - val_output_rmse: 0.4661 Epoch 41/1000 26/26 [==============================] - 8s 297ms/step - loss: 0.1800 - output_loss: 0.1800 - output_rmse: 0.4243 - val_loss: 0.2167 - val_output_loss: 0.2167 - val_output_rmse: 0.4655 Epoch 42/1000 26/26 [==============================] - 7s 261ms/step - loss: 0.1807 - output_loss: 0.1807 - output_rmse: 0.4251 - val_loss: 0.2086 - val_output_loss: 0.2086 - val_output_rmse: 0.4567 Epoch 43/1000 26/26 [==============================] - 7s 256ms/step - loss: 0.1754 - output_loss: 0.1754 - output_rmse: 0.4188 - val_loss: 0.2227 - val_output_loss: 0.2227 - val_output_rmse: 0.4719 Epoch 44/1000 26/26 [==============================] - 7s 259ms/step - loss: 0.1782 - output_loss: 0.1782 - output_rmse: 0.4221 - val_loss: 0.2148 - val_output_loss: 0.2148 - val_output_rmse: 0.4634 Epoch 45/1000 26/26 [==============================] - 7s 274ms/step - loss: 0.1760 - output_loss: 0.1760 - output_rmse: 0.4195 - val_loss: 0.2068 - val_output_loss: 0.2068 - val_output_rmse: 0.4547 Epoch 46/1000 26/26 [==============================] - 7s 263ms/step - loss: 0.1717 - output_loss: 0.1717 - output_rmse: 0.4144 - val_loss: 0.2104 - val_output_loss: 0.2104 - val_output_rmse: 0.4587 Epoch 47/1000 26/26 [==============================] - 7s 256ms/step - loss: 0.1696 - output_loss: 0.1696 - output_rmse: 0.4118 - val_loss: 0.2188 - val_output_loss: 0.2188 - val_output_rmse: 0.4678 Epoch 48/1000 26/26 [==============================] - 7s 256ms/step - loss: 0.1709 - output_loss: 0.1709 - output_rmse: 0.4134 - val_loss: 0.2140 - val_output_loss: 0.2140 - val_output_rmse: 0.4626 Epoch 49/1000 26/26 [==============================] - 7s 256ms/step - loss: 0.1757 - output_loss: 0.1757 - output_rmse: 0.4192 - val_loss: 0.2465 - val_output_loss: 0.2465 - val_output_rmse: 0.4965 Epoch 50/1000 26/26 [==============================] - 7s 258ms/step - loss: 0.1725 - output_loss: 0.1725 - output_rmse: 0.4153 - val_loss: 0.2124 - val_output_loss: 0.2124 - val_output_rmse: 0.4608 Epoch 51/1000 26/26 [==============================] - 7s 264ms/step - loss: 0.1633 - output_loss: 0.1633 - output_rmse: 0.4041 - val_loss: 0.2126 - val_output_loss: 0.2126 - val_output_rmse: 0.4611 Epoch 52/1000 26/26 [==============================] - 7s 257ms/step - loss: 0.1647 - output_loss: 0.1647 - output_rmse: 0.4058 - val_loss: 0.2205 - val_output_loss: 0.2205 - val_output_rmse: 0.4696 Epoch 53/1000 26/26 [==============================] - 7s 280ms/step - loss: 0.1607 - output_loss: 0.1607 - output_rmse: 0.4009 - val_loss: 0.2252 - val_output_loss: 0.2252 - val_output_rmse: 0.4745 Epoch 54/1000 26/26 [==============================] - 7s 281ms/step - loss: 0.1646 - output_loss: 0.1646 - output_rmse: 0.4058 - val_loss: 0.2145 - val_output_loss: 0.2145 - val_output_rmse: 0.4631 Epoch 55/1000 26/26 [==============================] - 8s 288ms/step - loss: 0.1640 - output_loss: 0.1640 - output_rmse: 0.4050 - val_loss: 0.2168 - val_output_loss: 0.2168 - val_output_rmse: 0.4656 Epoch 56/1000 26/26 [==============================] - 7s 272ms/step - loss: 0.1641 - output_loss: 0.1641 - output_rmse: 0.4051 - val_loss: 0.2337 - val_output_loss: 0.2337 - val_output_rmse: 0.4834 Epoch 57/1000 26/26 [==============================] - 7s 281ms/step - loss: 0.1663 - output_loss: 0.1663 - output_rmse: 0.4077 - val_loss: 0.2169 - val_output_loss: 0.2169 - val_output_rmse: 0.4657 Epoch 58/1000 26/26 [==============================] - 7s 265ms/step - loss: 0.1533 - output_loss: 0.1533 - output_rmse: 0.3916 - val_loss: 0.2260 - val_output_loss: 0.2260 - val_output_rmse: 0.4754 Epoch 59/1000 26/26 [==============================] - 7s 271ms/step - loss: 0.1541 - output_loss: 0.1541 - output_rmse: 0.3925 - val_loss: 0.2295 - val_output_loss: 0.2295 - val_output_rmse: 0.4790 Epoch 60/1000 26/26 [==============================] - 7s 277ms/step - loss: 0.1604 - output_loss: 0.1604 - output_rmse: 0.4005 - val_loss: 0.2173 - val_output_loss: 0.2173 - val_output_rmse: 0.4662 Epoch 61/1000 26/26 [==============================] - 7s 275ms/step - loss: 0.1848 - output_loss: 0.1848 - output_rmse: 0.4299 - val_loss: 0.2305 - val_output_loss: 0.2305 - val_output_rmse: 0.4801 Epoch 62/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.1650 - output_loss: 0.1650 - output_rmse: 0.4062 - val_loss: 0.2169 - val_output_loss: 0.2169 - val_output_rmse: 0.4657 Epoch 63/1000 26/26 [==============================] - 7s 279ms/step - loss: 0.1541 - output_loss: 0.1541 - output_rmse: 0.3926 - val_loss: 0.2126 - val_output_loss: 0.2126 - val_output_rmse: 0.4611 Epoch 64/1000 26/26 [==============================] - 7s 284ms/step - loss: 0.1532 - output_loss: 0.1532 - output_rmse: 0.3914 - val_loss: 0.2187 - val_output_loss: 0.2187 - val_output_rmse: 0.4677 Epoch 65/1000 26/26 [==============================] - 7s 265ms/step - loss: 0.1502 - output_loss: 0.1502 - output_rmse: 0.3876 - val_loss: 0.2281 - val_output_loss: 0.2281 - val_output_rmse: 0.4776
plt.plot(ft_linear_history.history['val_loss'][:72], label='Linear Val Loss')
plt.plot(ft_periodic_history.history['val_loss'][:100], label='Periodic Val Loss')
plt.plot(ft_pleq_history.history['val_loss'][:100], label='PLE Quantile Val Loss')
plt.plot(ft_plet_history.history['val_loss'][:100], label='PLE Target Val Loss')
plt.title('Model Validation Loss')
plt.legend()
plt.show()
linear_test_preds = ft_linear_transformer.predict(test_dataset)
linear_rms = mean_squared_error(test_data[LABEL], linear_test_preds['output'].ravel(), squared=False)
periodic_test_preds = ft_periodic_transformer.predict(test_dataset)
periodic_rms = mean_squared_error(test_data[LABEL], periodic_test_preds['output'].ravel()., squared=False)
pleq_test_preds = ft_pleq_transformer.predict(test_dataset)
pleq_rms = mean_squared_error(test_data[LABEL], pleq_test_preds['output'].ravel(), squared=False)
plet_test_preds = ft_plet_transformer.predict(test_dataset)
plet_rms = mean_squared_error(test_data[LABEL], plet_test_preds['output'].ravel(), squared=False)
print("-" * 28 + " FT Transformer " + "-" * 27)
print("Linear Encoding RMSE:", linear_rms.round(4))
print("Periodic Encoding RMSE:", periodic_rms.round(4))
print("PLE Encoding with Qantile Binning RMSE:", pleq_rms.round(4))
print("PLE Encoding with Target Binning RMSE:", plet_rms.round(4))
print("")
print("-" * 30 + " Baselines " + "-" * 30)
print("Random Forest RMSE:", rf_rms.round(4))
print("Catboost RMSE:", catb_rms.round(4))
---------------------------- FT Transformer --------------------------- Linear Encoding RMSE: 0.5153 Periodic Encoding RMSE: 0.4848 PLE Encoding with Qantile Binning RMSE: 0.4553 PLE Encoding with Target Binning RMSE: 0.4578 ------------------------------ Baselines ------------------------------ Random Forest RMSE: 0.5111 Catboost RMSE: 0.4487
# import optuna
# import gc
# def objective(trial):
# ft_encoder = FTTransformerEncoder(
# numerical_features = NUMERIC_FEATURES,
# categorical_features = [],
# numerical_data = X_train[NUMERIC_FEATURES].values,
# categorical_data = None,
# y = X_train[LABEL].values,
# task='regression',
# numerical_embedding_type= 'ple',
# numerical_bins=trial.suggest_int('numerical_bins', 20, 200),
# embedding_dim=trial.suggest_int('embedding_dim', 8, 100),
# depth=trial.suggest_int('depth', 1, 6),
# heads=trial.suggest_int('heads', 2, 8),
# attn_dropout=trial.suggest_float('attn_dropout', 0., 0.5),
# ff_dropout=trial.suggest_float('ff_dropout', 0., 0.5),
# explainable=True
# )
# # Pass th encoder to the model
# ft_transformer = FTTransformer(
# encoder=ft_encoder,
# out_dim=1,
# out_activation=housing_act
# )
# LEARNING_RATE = 0.001
# WEIGHT_DECAY = 0.00001
# NUM_EPOCHS = 1000
# optimizer = tfa.optimizers.AdamW(
# learning_rate=LEARNING_RATE, weight_decay=WEIGHT_DECAY
# )
# ft_transformer.compile(
# optimizer = optimizer,
# loss = {"output": tf.keras.losses.MeanSquaredError(name='mse'), "importances": None},
# metrics= {"output": [tf.keras.metrics.RootMeanSquaredError(name='rmse')], "importances": None},
# )
# early = EarlyStopping(monitor="val_output_loss", mode="min", patience=20, restore_best_weights=True)
# callback_list = [early]
# ft_history = ft_transformer.fit(
# train_dataset,
# epochs=NUM_EPOCHS,
# validation_data=val_dataset,
# callbacks=callback_list
# )
# preds = ft_transformer.predict(test_dataset)
# rmse = mean_squared_error(test_data[LABEL], preds['output'].ravel().clip(0, 5),squared=False)
# gc.collect()
# return rmse
# study = optuna.create_study(direction='minimize')
# study.optimize(objective, n_trials=50)
# print('Number of finished trials:', len(study.trials))
# print('Best trial:', study.best_trial.params)