In [1]:
import copy
from glob import glob
import pickle
import munch
import math
import torch as th
import numpy as np
import scipy as sp
import scipy.signal as _
import matplotlib.pyplot as plt
import matplotlib as mpl
%matplotlib inline
plt.rcParams['figure.figsize'] = [14, 8.5]
Autoencoder Data Preparation¶
In [2]:
filenames = sorted(glob('data/rest/*.pkl'))
filenames
Out[2]:
['data/rest/s08-3minutes.pkl', 'data/rest/s08-letter-b.pkl', 'data/rest/s08-letter-d.pkl', 'data/rest/s08-letter-p.pkl', 'data/rest/s08-mentaltasks.pkl', 'data/rest/s10-3minutes.pkl', 'data/rest/s10-letter-b.pkl', 'data/rest/s10-letter-d.pkl', 'data/rest/s10-letter-p.pkl', 'data/rest/s10-mentaltasks.pkl', 'data/rest/s11-3minutes.pkl', 'data/rest/s11-letter-b.pkl', 'data/rest/s11-letter-d.pkl', 'data/rest/s11-letter-p.pkl', 'data/rest/s11-mentaltasks.pkl', 'data/rest/s12-3minutes.pkl', 'data/rest/s12-letter-b.pkl', 'data/rest/s12-letter-d.pkl', 'data/rest/s12-letter-p.pkl', 'data/rest/s12-mentaltasks.pkl', 'data/rest/s13-3minutes.pkl', 'data/rest/s13-letter-b.pkl', 'data/rest/s13-letter-d.pkl', 'data/rest/s13-letter-p.pkl', 'data/rest/s13-mentaltasks.pkl', 'data/rest/s15-3minutes.pkl', 'data/rest/s15-letter-b.pkl', 'data/rest/s15-letter-d.pkl', 'data/rest/s15-letter-p.pkl', 'data/rest/s15-mentaltasks.pkl', 'data/rest/s16-3minutes.pkl', 'data/rest/s16-letter-b.pkl', 'data/rest/s16-letter-d.pkl', 'data/rest/s16-letter-p.pkl', 'data/rest/s16-mentaltasks.pkl', 'data/rest/s20-3minutes.pkl', 'data/rest/s20-letter-b.pkl', 'data/rest/s20-letter-d.pkl', 'data/rest/s20-letter-p.pkl', 'data/rest/s20-mentaltasks.pkl', 'data/rest/s21-3minutes.pkl', 'data/rest/s21-letter-b.pkl', 'data/rest/s21-letter-d.pkl', 'data/rest/s21-letter-p.pkl', 'data/rest/s21-mentaltasks.pkl', 'data/rest/s22-3minutes.pkl', 'data/rest/s22-letter-b.pkl', 'data/rest/s22-letter-d.pkl', 'data/rest/s22-letter-p.pkl', 'data/rest/s22-mentaltasks.pkl', 'data/rest/s23-3minutes.pkl', 'data/rest/s23-letter-b.pkl', 'data/rest/s23-letter-d.pkl', 'data/rest/s23-letter-p.pkl', 'data/rest/s23-mentaltasks.pkl', 'data/rest/s24-3minutes.pkl', 'data/rest/s24-letter-b.pkl', 'data/rest/s24-letter-d.pkl', 'data/rest/s24-letter-p.pkl', 'data/rest/s24-mentaltasks.pkl', 'data/rest/s25-3minutes.pkl', 'data/rest/s25-letter-b.pkl', 'data/rest/s25-letter-d.pkl', 'data/rest/s25-letter-p.pkl', 'data/rest/s25-mentaltasks.pkl', 'data/rest/s26-3minutes.pkl', 'data/rest/s26-letter-b.pkl', 'data/rest/s26-letter-d.pkl', 'data/rest/s26-letter-p.pkl', 'data/rest/s26-mentaltasks.pkl', 'data/rest/s27-3minutes.pkl', 'data/rest/s27-letter-b.pkl', 'data/rest/s27-letter-d.pkl', 'data/rest/s27-letter-p.pkl', 'data/rest/s27-mentaltasks.pkl', 'data/rest/s28-3minutes.pkl', 'data/rest/s28-letter-b.pkl', 'data/rest/s28-letter-d.pkl', 'data/rest/s28-letter-p.pkl', 'data/rest/s28-mentaltasks.pkl']
In [3]:
subjects = (8, 10, 11, 12, 13, 15, 16, 20, 21, 22, 23, 24, 25, 26, 27, 28)
len(subjects)
Out[3]:
16
In [4]:
chan_names_biosemi_32 = (
'Fp1', 'AF3', 'F7', 'F3', 'FC1', 'FC5', 'T7', 'C3',
'CP1', 'CP5', 'P7', 'P3', 'Pz', 'PO3', 'O1', 'Oz',
'O2', 'PO4', 'P4', 'P8', 'CP6', 'CP2', 'C4', 'T8',
'FC6', 'FC2', 'F4', 'F8', 'AF4', 'Fp2', 'Fz', 'Cz',
'EXG1','EXG2','EXG3','EXG4','EXG5','EXG6','EXG7','EXG8',
)
chan_names_biosemi_64 = (
'Fp1', 'AF7', 'AF3', 'F1', 'F3', 'F5', 'F7', 'FT7',
'FC5', 'FC3', 'FC1', 'C1', 'C3', 'C5', 'T7', 'TP7',
'CP5', 'CP3', 'CP1', 'P1', 'P3', 'P5', 'P7', 'P9',
'PO7', 'PO3', 'O1', 'Iz', 'Oz', 'POz', 'Pz', 'CPz',
'Fpz', 'Fp2', 'AF8', 'AF4', 'AFz', 'Fz', 'F2', 'F4',
'F6', 'F8', 'FT8', 'FC6', 'FC4', 'FC2', 'FCz', 'Cz',
'C2', 'C4', 'C6', 'T8', 'TP8', 'CP6', 'CP4', 'CP2',
'P2', 'P4', 'P6', 'P8', 'P10', 'PO8', 'PO4', 'O2',
'EXG1','EXG2','EXG3','EXG4','EXG5','EXG6','EXG7','EXG8',
)
In [5]:
select_chan_names = (
'C3', 'C4', 'Cz', 'F3', 'F4', 'F7', 'F8', 'Fp1', 'Fp2',
'Fz', 'O1', 'O2', 'P3', 'P4', 'P7', 'P8', 'Pz', 'T7', 'T8',
)
#select_chan_names = (
# 'Fp1', 'AF3', 'F7', 'F3', 'FC1', 'FC5', 'T7', 'C3',
# 'CP1', 'CP5', 'P7', 'P3', 'Pz', 'PO3', 'O1', 'Oz',
# 'O2', 'PO4', 'P4', 'P8', 'CP6', 'CP2', 'C4', 'T8',
# 'FC6', 'FC2', 'F4', 'F8', 'AF4', 'Fp2', 'Fz', 'Cz',
#)
len(select_chan_names)
Out[5]:
19
In [6]:
class EEGDataset(th.utils.data.Dataset):
def __init__(self, data, samp_rate, chan_names):
super().__init__()
self.data = data
self.samp_rate = samp_rate
self.chan_names = chan_names
@staticmethod
def _make_bandpass(samp_rate, low_freq, high_freq, order=6):
'''Bidirectional Butterworth linear IIR bandpass filter.
'''
# compute the filter coefficients
#numer_coef, denom_coef = sp.signal.iirfilter(
# order, (low_freq, high_freq), ftype='butter', btype='bandpass', fs=samp_rate)
sos = sp.signal.iirfilter(
order, (low_freq, high_freq), ftype='butter', btype='bandpass', fs=samp_rate, output='sos')
def bandpass(data):
# apply the filter in both directions
return th.as_tensor(
#sp.signal.filtfilt(numer_coef, denom_coef, data.numpy(), axis=0, padtype='even').copy(),
sp.signal.sosfiltfilt(sos, data.numpy(), axis=0, padtype='even').copy(),
dtype=th.float32)
return bandpass
@staticmethod
def _make_notch(samp_rate, freq, quality=30.0):
'''Bidirectional Butterworth linear IIR notch filter.
'''
numer_coef, denom_coef = sp.signal.iirnotch(freq, quality, fs=samp_rate)
def notch(data):
return th.as_tensor(
sp.signal.filtfilt(numer_coef, denom_coef, data.numpy(), axis=0, padtype='even').copy(),
dtype=th.float32)
return notch
@staticmethod
def _prep(data, full_chan_names, select_chan_names,
bandpass_filter, notch_filter, clamp, downsample_factor):
ref_chan_idxs = full_chan_names.index('EXG5'), full_chan_names.index('EXG6')
#eog_vchan_idxs = full_chan_names.index('EXG1'), full_chan_names.index('EXG2')
#eog_hchan_idxs = full_chan_names.index('EXG3'), full_chan_names.index('EXG4')
select_chan_idxs = tuple(full_chan_names.index(chan_name) for chan_name in select_chan_names)
data -= data.mean(0)
reference = data[:, ref_chan_idxs].mean(1)[:, None]
data -= reference
data -= data.mean(0)
data = notch_filter(data)
data = bandpass_filter(data)[::downsample_factor]
#veog = data[:, eog_vchan_idxs[1]] - data[:, eog_vchan_idxs[0]]
#heog = data[:, eog_hchan_idxs[1]] - data[:, eog_hchan_idxs[0]]
#eog = th.stack((veog, heog, th.ones(veog.shape[0]))).T
data = data[:, select_chan_idxs]
#eog_weights = th.pinverse(eog) @ data
#data -= eog @ eog_weights
data = data.clamp(-clamp, clamp)
return data
def plot_trace(self, n, scale_factor=1.0):
segment = self[n] if isinstance(n, int) else n
segment = segment.T
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
nsec = segment.shape[0] / self.samp_rate
nchan = len(self.chan_names)
scale = th.max(th.abs(segment))
sep = -th.arange(segment.shape[1]) * scale_factor * scale
segment = segment + sep
time = th.linspace(0, nsec, segment.shape[0])
ax.plot(time, segment)
ax.set_xlabel(r'Time ($s$)')
ax.set_ylabel(r'Signal ($\mu V$) Across Channels')
ax.set_yticks(sep)
ax.set_yticklabels([c for i, c in enumerate(self.chan_names)])
ax.autoscale(tight=True)
def plot_psd(self, n):
segment = self[n] if isinstance(n, int) else n
freqs, powers = sp.signal.welch(segment, self.samp_rate)
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(freqs, powers.T);
ax.set_xlabel('Frequency (Hz)');
ax.set_ylabel(r'Power Density ($\mu V^2/Hz$');
ax.set_yscale('log');
ax.grid();
ax.legend(self.chan_names, ncol=2);
ax.autoscale(tight=True)
def __getitem__(self, idx):
return self.data[idx]
def __len__(self):
return self.data.shape[0]
@property
def nseg(self):
return self.data.shape[0]
@property
def nchan(self):
return self.data.shape[1]
@property
def nstep(self):
return self.data.shape[2]
In [7]:
class AutoencoderDataset(EEGDataset):
def __init__(self, subjects, nsec=2.0, overlap=0.5, samp_rate=1024.0,
low_freq=0.5, high_freq=100.0, downsample_factor=4, clamp=60.0,
full_chan_names=chan_names_biosemi_32, select_chan_names=select_chan_names,
means=None, stdevs=None):
orig_samp_rate = samp_rate
new_samp_rate = samp_rate / downsample_factor
del samp_rate
data_mats = []
bandpass_filter = self._make_bandpass(orig_samp_rate, low_freq, high_freq)
notch_filter = self._make_notch(orig_samp_rate, 60.0)
for subject in subjects:
filenames = sorted(glob(f'data/rest/s{subject:02}-*.pkl'))
for filename in filenames:
with open(filename, mode='rb') as fh:
raw_data = th.as_tensor(pickle.load(fh), dtype=th.float32)
data = self._prep(
data=raw_data,
full_chan_names=full_chan_names,
select_chan_names=select_chan_names,
bandpass_filter=bandpass_filter,
notch_filter=notch_filter,
clamp=clamp,
downsample_factor=downsample_factor)
width = int(new_samp_rate * nsec)
stride = int(width * (1.0 - overlap))
for start in range(0, data.shape[0], stride):
end = start + width
if end < data.shape[0]:
data_mats.append(data[start:end].clone())
data = th.stack(data_mats)
assert len(select_chan_names) == data.shape[-1]
nchan = data.shape[-1]
self.means = data.reshape((-1, nchan)).mean(0) if means is None else means
assert self.means.shape == (nchan,)
data -= self.means
self.stdevs = data.reshape((-1, nchan)).std(0) if stdevs is None else stdevs
assert self.stdevs.shape == (nchan,)
data /= self.stdevs
# we switch to shape (nseg, nchan, nobs) here since pytorch
# convolution operators expect channel first
data = data.permute(0, 2, 1)
super().__init__(data, new_samp_rate, select_chan_names)
In [8]:
train_data = AutoencoderDataset(subjects=subjects[1:-1])
valid_data = AutoencoderDataset(
subjects=(subjects[0], subjects[-1]),
means=train_data.means,
stdevs=train_data.stdevs)
len(train_data), len(valid_data)
Out[8]:
(10279, 1477)
In [9]:
train_data.means.shape, train_data.stdevs.shape
Out[9]:
(torch.Size([19]), torch.Size([19]))
In [10]:
train_data[0].shape, train_data[0].reshape((-1, 19)).mean(0)
Out[10]:
(torch.Size([19, 512]),
tensor([0.0308, 0.0263, 0.0235, 0.0166, 0.0162, 0.0230, 0.0058, 0.0125, 0.0422,
0.0388, 0.0277, 0.0361, 0.0347, 0.0263, 0.0306, 0.0310, 0.0329, 0.0392,
0.0337]))
In [11]:
train_data[0].shape, train_data[0].reshape((-1, 19)).std(0)
Out[11]:
(torch.Size([19, 512]),
tensor([0.5583, 0.5835, 0.5545, 0.5387, 0.5678, 0.5852, 0.6017, 0.6083, 0.5898,
0.5633, 0.5606, 0.5729, 0.5773, 0.5732, 0.5759, 0.5855, 0.5739, 0.5619,
0.5551]))
In [12]:
1024 / 4
Out[12]:
256.0
In [13]:
train_data.samp_rate
Out[13]:
256.0
In [ ]:
In [ ]:
In [14]:
train_data.plot_trace(0)
In [15]:
train_data.plot_trace(1)
In [16]:
train_data.plot_trace(2)
In [17]:
train_data.plot_trace(3)
In [18]:
train_data.plot_trace(4)
In [ ]:
In [19]:
train_data.plot_trace(5)
In [20]:
train_data.plot_psd(0)
In [21]:
train_data.plot_psd(1)
Autoencoder Networks¶
In [22]:
class ConvEncodeBlock(th.nn.Module):
def __init__(self, nchan, nfilt, kernel_size, stride, transfer):
super().__init__()
self.conv = th.nn.Conv1d(
nchan, nfilt,
kernel_size=kernel_size,
padding=kernel_size // 2,
stride=stride)
self.transfer = transfer()
def forward(self, segs):
assert segs.ndim == 3
return self.transfer(self.conv(segs))
In [23]:
class ResidualEncodeBlock(th.nn.Module):
def __init__(self, nchan, nfilt, kernel_size, stride, transfer):
super().__init__()
#self.batch_norm = th.nn.BatchNorm1d(
# nchan, momentum=0.05)
if nchan == nfilt:
self.remix = lambda segs: segs
else:
self.remix = th.nn.Conv1d(nchan, nfilt, kernel_size=1)
if stride == 1:
self.downsample = lambda segs: segs
else:
self.downsample = th.nn.AvgPool1d(stride, ceil_mode=True)
#self.pad = th.nn.ReflectionPad1d(kernel_size // 2)
self.conv = th.nn.Conv1d(
nchan, nfilt,
kernel_size=kernel_size,
padding=kernel_size // 2,
stride=stride)
self.transfer = transfer()
self.combine = th.nn.Conv1d(nfilt, nfilt, kernel_size=1)
def forward(self, segs):
assert segs.ndim == 3
#norm_segs = self.batch_norm(segs)
remix_out = self.downsample(self.remix(segs))
resid_out = self.combine(self.transfer(self.conv(segs)))
assert remix_out.shape == resid_out.shape
return remix_out + resid_out
In [24]:
class Encoder(th.nn.Module):
def __init__(self, nchan, nstep, conv_specs, transfer, block_arch):
super().__init__()
self.nchan, self.nstep = nchan, nstep
self.output_shapes = [(nchan, nstep)]
self.conv_specs = conv_specs
if block_arch == 'conv':
encode_block_class = ConvEncodeBlock
elif block_arch == 'resid':
encode_block_class = ResidualEncodeBlock
else:
raise ValueError(f'invalid `block_arch`: {block_arch}')
convs = []
layer_in, layer_out = None, self.nchan
for nfilt, kernel_size, stride in self.conv_specs:
layer_in, layer_out = layer_out, nfilt
convs.append(
encode_block_class(
layer_in, nfilt,
kernel_size=kernel_size,
stride=stride,
transfer=transfer))
nstep = math.ceil(nstep / stride) #- kernel_size // 2
self.output_shapes.append((layer_out, nstep))
self.conv = th.nn.Sequential(*convs)
def forward(self, segs):
assert isinstance(segs, th.Tensor)
assert segs.ndim == 3
assert segs.shape[1] == self.nchan
#assert segs.shape[2] == self.nstep
batch_size = segs.shape[0]
conv_out = self.conv(segs)
assert conv_out.ndim == 3
assert conv_out.shape[0] == batch_size
assert conv_out.shape[1] == self.output_shapes[-1][0] # nfilt
#assert conv_out.shape[2] == self.output_shapes[-1][1] # nstep
codes = conv_out.reshape(batch_size, -1)
assert codes.ndim == 2
assert codes.shape[0] == batch_size
return codes
In [25]:
class ConvDecodeBlock(th.nn.Module):
def __init__(self, nchan, nfilt, kernel_size, stride, transfer, output_shape):
super().__init__()
self.output_shape = output_shape
self.deconv = th.nn.ConvTranspose1d(
nchan, nfilt,
kernel_size=kernel_size,
padding=kernel_size // 2,
stride=stride)
self.transfer = transfer()
def forward(self, segs):
assert segs.ndim == 3
return self.transfer(self.deconv(segs, self.output_shape))
In [26]:
class ResidualDecodeBlock(th.nn.Module):
def __init__(self, nchan, nfilt, kernel_size, stride, transfer, output_shape):
super().__init__()
self.output_shape = output_shape
#self.batch_norm = th.nn.BatchNorm1d(
# nchan, momentum=0.05)
if nchan == nfilt:
self.remix = lambda segs: segs
else:
self.remix = th.nn.Conv1d(nchan, nfilt, kernel_size=1)
if stride == 1:
self.upsample = lambda segs: segs
else:
self.upsample = th.nn.Upsample(
size=self.output_shape,
mode='linear',
align_corners=False)
self.deconv = th.nn.ConvTranspose1d(
nchan, nfilt,
kernel_size=kernel_size,
padding=kernel_size // 2,
stride=stride)
self.transfer = transfer()
self.combine = th.nn.Conv1d(nfilt, nfilt, kernel_size=1)
def forward(self, segs):
assert segs.ndim == 3
#norm_segs = self.batch_norm(segs)
remix_out = self.upsample(self.remix(segs))
resid_out = self.combine(self.transfer(
self.deconv(segs, self.output_shape)))
assert remix_out.shape == resid_out.shape
return remix_out + resid_out
In [27]:
class Decoder(th.nn.Module):
def __init__(self, output_shapes, conv_specs, transfer, block_arch):
super().__init__()
self.output_shapes = output_shapes
self.conv_specs = conv_specs
if block_arch == 'conv':
decode_block_class = ConvDecodeBlock
elif block_arch == 'resid':
decode_block_class = ResidualDecodeBlock
else:
raise ValueError(f'invalid `block_arch`: {block_arch}')
deconvs = []
layer_in, layer_out = None, output_shapes[0][0]
for i, (nfilt, kernel_size, stride) in enumerate(self.conv_specs):
layer_in, layer_out = layer_out, nfilt
deconvs.append(
decode_block_class(
layer_in, nfilt,
kernel_size=kernel_size,
stride=stride,
transfer=transfer,
output_shape=self.output_shapes[i+1][1:]))
self.deconv = th.nn.Sequential(*deconvs)
nchan, _nstep = self.output_shapes[-1]
#self.readout = th.nn.Sequential(
# th.nn.BatchNorm1d(layer_out, momentum=0.05),
# th.nn.Conv1d(layer_out, nchan, kernel_size=1, padding=0))
self.readout = th.nn.Conv1d(layer_out, nchan, kernel_size=1, padding=0)
def forward(self, codes):
assert isinstance(codes, th.Tensor)
assert codes.ndim == 2
batch_size = codes.shape[0]
unflat_codes = codes.reshape((batch_size,) + self.output_shapes[0])
assert unflat_codes.ndim == 3
assert unflat_codes.shape[0] == batch_size
assert unflat_codes.shape[1] == self.output_shapes[0][0] # nfilt
assert unflat_codes.shape[2] == self.output_shapes[0][1] # nstep
deconv_out = self.deconv(unflat_codes)
assert deconv_out.ndim == 3
assert deconv_out.shape[0] == batch_size
assert deconv_out.shape[2] == self.output_shapes[-1][1] # nstep
segs = self.readout(deconv_out)
assert segs.ndim == 3
assert segs.shape[0] == batch_size
assert segs.shape[1] == self.output_shapes[-1][0] # nchan
assert segs.shape[2] == self.output_shapes[-1][1] # nstep
return segs
In [28]:
class Autoencoder(th.nn.Module):
def __init__(self, nchan, nstep, conv_specs, transfer=th.nn.ReLU, block_arch='conv'):
super().__init__()
assert nchan > 0
assert nstep > 0
self.nchan, self.nstep = nchan, nstep
assert isinstance(conv_specs, tuple)
self.conv_specs = conv_specs
assert issubclass(transfer, th.nn.Module)
self.encoder = Encoder(
nchan=self.nchan,
nstep=self.nstep,
conv_specs=self.conv_specs,
transfer=transfer,
block_arch=block_arch)
rev_conv_specs = tuple(reversed(self.conv_specs))
rev_output_shapes = tuple(reversed(self.encoder.output_shapes))
self.decoder = Decoder(
output_shapes=rev_output_shapes,
conv_specs=rev_conv_specs,
transfer=transfer,
block_arch=block_arch)
def forward(self, segs):
return self.decoder(self.encoder(segs))
@th.no_grad()
def encode(self, segs):
return self.encoder(segs)
@th.no_grad()
def recon(self, segs):
return self(segs)
@property
def nparameters(self):
return sum(p.nelement() for p in self.parameters())
def save(self, filename):
with open(filename, mode='wb') as fh:
th.save(self.state_dict(), fh)
Training¶
In [29]:
def train_adam(model, train_data, valid_data, max_epochs=500, stop_epochs=50,
learning_rate=0.0125, learning_rate_decay=0.996, loss_method='huber',
clip_norm=6, batch_size=128, load_workers=0, use_gpu=True):
train_loader = th.utils.data.DataLoader(
train_data,
batch_size=batch_size,
pin_memory=use_gpu,
num_workers=load_workers,
drop_last=True,
shuffle=True)
valid_loader = th.utils.data.DataLoader(
valid_data,
batch_size=batch_size,
pin_memory=use_gpu,
num_workers=load_workers,
shuffle=False)
if use_gpu:
model.cuda()
train_losses = []
valid_losses = []
best_train_loss = float('inf')
best_valid_loss = float('inf')
best_epoch = 0
best_state = None
if loss_method == 'mse':
loss_func = th.nn.MSELoss(reduction='mean')
elif loss_method == 'huber':
loss_func = th.nn.SmoothL1Loss(reduction='mean')
else:
raise ValueError(f'invalid `loss_method`: {loss_method}')
# all parameters with `requires_grad` set
parameters = list(filter(lambda param: param.requires_grad, model.parameters()))
# initialize the optimizer
optimizer = th.optim.Adam(parameters, lr=learning_rate)
# exponential learning rate decay
lr_scheduler = th.optim.lr_scheduler.ExponentialLR(optimizer, learning_rate_decay)
for epoch in range(max_epochs):
# enter training mode
model.train()
# training pass
epoch_train_losses = []
for i, segs in enumerate(train_loader):
optimizer.zero_grad()
if use_gpu:
segs = segs.cuda()
recons = model(segs)
loss = loss_func(recons, segs)
loss_f = loss.item()
epoch_train_losses.append(loss_f)
loss.backward()
if clip_norm is not None:
th.nn.utils.clip_grad_norm_(parameters, clip_norm)
optimizer.step()
train_losses.append(np.mean(epoch_train_losses))
# back to evaluation mode
model.eval()
# validation loss
with th.no_grad():
epoch_valid_losses = []
for i, segs in enumerate(valid_loader):
if use_gpu:
segs = segs.cuda()
recons = model(segs)
loss = loss_func(recons, segs)
loss_f = loss.item()
epoch_valid_losses.append(loss_f)
valid_losses.append(np.mean(epoch_valid_losses))
if valid_losses[-1] < best_valid_loss:
best_train_loss = train_losses[-1]
best_valid_loss = valid_losses[-1]
best_epoch = epoch
best_state = copy.deepcopy(model.state_dict())
print('epoch:', epoch)
print('learning rate:', next(iter(optimizer.param_groups))['lr'])
print('train loss:', train_losses[-1])
print('valid loss:', valid_losses[-1])
print('best epoch:', best_epoch)
print('=======')
if (best_epoch + stop_epochs) <= epoch:
print(f'terminating after {stop_epochs} of no improvement')
break
lr_scheduler.step()
if best_state is None:
raise RuntimeError('best state not set!')
model.load_state_dict(best_state)
if use_gpu:
model.cpu()
return munch.munchify({
'train_losses': train_losses,
'valid_losses': valid_losses,
'best_train_loss': best_train_loss,
'best_valid_loss': best_valid_loss,
'best_epoch': best_epoch,
})
In [30]:
def train_rprop(model, train_data, valid_data, max_epochs=1500, stop_epochs=50,
init_step=0.005, loss_method='huber', batch_size=128, load_workers=0,
use_gpu=True):
train_loader = th.utils.data.DataLoader(
train_data,
batch_size=batch_size,
pin_memory=use_gpu,
num_workers=load_workers,
drop_last=False,
shuffle=False)
valid_loader = th.utils.data.DataLoader(
valid_data,
batch_size=batch_size,
pin_memory=use_gpu,
num_workers=load_workers,
shuffle=False)
if use_gpu:
model.cuda()
train_losses = []
valid_losses = []
best_train_loss = float('inf')
best_valid_loss = float('inf')
best_epoch = 0
best_state = None
if loss_method == 'mse':
loss_func = th.nn.MSELoss(reduction='mean')
elif loss_method == 'huber':
loss_func = th.nn.SmoothL1Loss(reduction='mean')
else:
raise ValueError(f'invalid `loss_method`: {loss_method}')
# all parameters with `requires_grad` set
parameters = list(filter(lambda param: param.requires_grad, model.parameters()))
# initialize the optimizer
optimizer = th.optim.Rprop(parameters, lr=init_step)
for epoch in range(max_epochs):
# enter training mode
model.train()
# zero gradients
# outside of the batch loop since gradients are accumulated
optimizer.zero_grad()
# training pass
epoch_train_losses = []
for i, segs in enumerate(train_loader):
if use_gpu:
segs = segs.cuda()
recons = model(segs)
loss = loss_func(recons, segs)
loss_f = loss.item()
epoch_train_losses.append(loss_f)
loss.backward()
optimizer.step()
train_losses.append(np.mean(epoch_train_losses))
# back to evaluation mode
model.eval()
# validation loss
with th.no_grad():
epoch_valid_losses = []
for i, segs in enumerate(valid_loader):
if use_gpu:
segs = segs.cuda()
recons = model(segs)
loss = loss_func(recons, segs)
loss_f = loss.item()
epoch_valid_losses.append(loss_f)
valid_losses.append(np.mean(epoch_valid_losses))
if valid_losses[-1] < best_valid_loss:
best_train_loss = train_losses[-1]
best_valid_loss = valid_losses[-1]
best_epoch = epoch
best_state = copy.deepcopy(model.state_dict())
print('epoch:', epoch)
print('train loss:', train_losses[-1])
print('valid loss:', valid_losses[-1])
print('best epoch:', best_epoch)
print('=======')
if (best_epoch + stop_epochs) <= epoch:
print(f'terminating after {stop_epochs} of no improvement')
break
if best_state is None:
raise RuntimeError('best state not set!')
model.load_state_dict(best_state)
if use_gpu:
model.cpu()
return munch.munchify({
'train_losses': train_losses,
'valid_losses': valid_losses,
'best_train_loss': best_train_loss,
'best_valid_loss': best_valid_loss,
'best_epoch': best_epoch,
})
In [31]:
def train(model, train_data, valid_data, optim_method='rprop', **kwargs):
if optim_method == 'rprop':
return train_rprop(model, train_data, valid_data, **kwargs)
elif optim_method == 'adam':
return train_adam(model, train_data, valid_data, **kwargs)
else:
raise ValueError(f'invalid `optim_method`: {optim_method}')
In [32]:
autoencoder = Autoencoder(
nchan=train_data.nchan,
nstep=train_data.nstep,
#conv_specs=((32, 21, 1), (32, 19, 2), # 128Hz
# (48, 17, 1), (48, 15, 2), # 64Hz
# (64, 13, 1), (64, 11, 2), # 32Hz
# (80, 9, 1), (80, 7, 2), # 16Hz
# (96, 5, 1), (96, 3, 2)), # 8Hz
# #(72, 3, 1), (72, 3, 2)), # 4Hz
#conv_specs=(
# (32, 19, 2), # 128Hz
# (48, 15, 2), # 64Hz
# (64, 11, 2), # 32Hz
# (80, 7, 2), # 16Hz
# (96, 5, 2), # 8Hz
# (112, 3, 2)), # 4Hz
#conv_specs=((32, 11, 1), (32, 11, 1), (32, 11, 2), # 128Hz
# (48, 9, 1), (48, 9, 1), (48, 9, 2), # 64Hz
# (64, 7, 1), (64, 7, 1), (64, 7, 2), # 32Hz
# (80, 5, 1), (80, 5, 1), (80, 5, 2), # 16Hz
# ),
#conv_specs=((32, 11, 2), # 128Hz
# (48, 9, 2), # 64Hz
# (64, 7, 2), # 32Hz
# (80, 5, 2), # 16Hz
# ),
#conv_specs=((32, 11, 2), # 128Hz
# (40, 9, 2), # 64Hz
# (48, 7, 2), # 32Hz
# (56, 5, 2), # 16Hz
# (64, 3, 2), # 8Hz
# ),
conv_specs=((24, 11, 2), # 128Hz
(32, 9, 2), # 64Hz
(40, 7, 2), # 32Hz
(48, 5, 2), # 16Hz
),
block_arch='resid',
transfer=th.nn.ReLU)
autoencoder
Out[32]:
Autoencoder(
(encoder): Encoder(
(conv): Sequential(
(0): ResidualEncodeBlock(
(remix): Conv1d(19, 24, kernel_size=(1,), stride=(1,))
(downsample): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
(conv): Conv1d(19, 24, kernel_size=(11,), stride=(2,), padding=(5,))
(transfer): ReLU()
(combine): Conv1d(24, 24, kernel_size=(1,), stride=(1,))
)
(1): ResidualEncodeBlock(
(remix): Conv1d(24, 32, kernel_size=(1,), stride=(1,))
(downsample): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
(conv): Conv1d(24, 32, kernel_size=(9,), stride=(2,), padding=(4,))
(transfer): ReLU()
(combine): Conv1d(32, 32, kernel_size=(1,), stride=(1,))
)
(2): ResidualEncodeBlock(
(remix): Conv1d(32, 40, kernel_size=(1,), stride=(1,))
(downsample): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
(conv): Conv1d(32, 40, kernel_size=(7,), stride=(2,), padding=(3,))
(transfer): ReLU()
(combine): Conv1d(40, 40, kernel_size=(1,), stride=(1,))
)
(3): ResidualEncodeBlock(
(remix): Conv1d(40, 48, kernel_size=(1,), stride=(1,))
(downsample): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
(conv): Conv1d(40, 48, kernel_size=(5,), stride=(2,), padding=(2,))
(transfer): ReLU()
(combine): Conv1d(48, 48, kernel_size=(1,), stride=(1,))
)
)
)
(decoder): Decoder(
(deconv): Sequential(
(0): ResidualDecodeBlock(
(upsample): Upsample(size=(64,), mode=linear)
(deconv): ConvTranspose1d(48, 48, kernel_size=(5,), stride=(2,), padding=(2,))
(transfer): ReLU()
(combine): Conv1d(48, 48, kernel_size=(1,), stride=(1,))
)
(1): ResidualDecodeBlock(
(remix): Conv1d(48, 40, kernel_size=(1,), stride=(1,))
(upsample): Upsample(size=(128,), mode=linear)
(deconv): ConvTranspose1d(48, 40, kernel_size=(7,), stride=(2,), padding=(3,))
(transfer): ReLU()
(combine): Conv1d(40, 40, kernel_size=(1,), stride=(1,))
)
(2): ResidualDecodeBlock(
(remix): Conv1d(40, 32, kernel_size=(1,), stride=(1,))
(upsample): Upsample(size=(256,), mode=linear)
(deconv): ConvTranspose1d(40, 32, kernel_size=(9,), stride=(2,), padding=(4,))
(transfer): ReLU()
(combine): Conv1d(32, 32, kernel_size=(1,), stride=(1,))
)
(3): ResidualDecodeBlock(
(remix): Conv1d(32, 24, kernel_size=(1,), stride=(1,))
(upsample): Upsample(size=(512,), mode=linear)
(deconv): ConvTranspose1d(32, 24, kernel_size=(11,), stride=(2,), padding=(5,))
(transfer): ReLU()
(combine): Conv1d(24, 24, kernel_size=(1,), stride=(1,))
)
)
(readout): Conv1d(24, 19, kernel_size=(1,), stride=(1,))
)
)
Total number of parameters in the autoencoder:
In [33]:
sum(p.nelement() for p in autoencoder.parameters())
Out[33]:
96107
In [34]:
first_segment = train_data[0][None, ...]
In [35]:
first_segment.shape, first_segment.shape[0] * first_segment.shape[1]
Out[35]:
(torch.Size([1, 19, 512]), 19)
In [36]:
autoencoder.eval()
autoencoder.encode(first_segment).shape
Out[36]:
torch.Size([1, 1536])
In [37]:
autoencoder.encoder.output_shapes
Out[37]:
[(19, 512), (24, 256), (32, 128), (40, 64), (48, 32)]
In [38]:
recon = autoencoder.recon(first_segment)
recon.shape, recon.min(), recon.max()
Out[38]:
(torch.Size([1, 19, 512]), tensor(-0.3372), tensor(0.3437))
In [39]:
train_data.plot_trace(recon[0])
In [40]:
train_result = train(autoencoder, train_data, valid_data, optim_method='rprop', loss_method='huber', use_gpu=False)
epoch: 0 train loss: 0.3519292790212749 valid loss: 0.2404336966574192 best epoch: 0 ======= epoch: 1 train loss: 0.33484673086139893 valid loss: 0.2652034175892671 best epoch: 0 ======= epoch: 2 train loss: 0.36373437094835587 valid loss: 0.18885906164844832 best epoch: 2 ======= epoch: 3 train loss: 0.2619668830324102 valid loss: 0.24536302934090296 best epoch: 2 ======= epoch: 4 train loss: 0.3470724161576342 valid loss: 0.175558403134346 best epoch: 4 ======= epoch: 5 train loss: 0.24878019903912957 valid loss: 0.21418170382579169 best epoch: 4 ======= epoch: 6 train loss: 0.2933356237632257 valid loss: 0.1899312362074852 best epoch: 4 ======= epoch: 7 train loss: 0.2618975754321357 valid loss: 0.16888864152133465 best epoch: 7 ======= epoch: 8 train loss: 0.23779664676130555 valid loss: 0.16766771922508875 best epoch: 8 ======= epoch: 9 train loss: 0.24536670565053267 valid loss: 0.15924314657847086 best epoch: 9 ======= epoch: 10 train loss: 0.23097897822290292 valid loss: 0.1519436084975799 best epoch: 10 ======= epoch: 11 train loss: 0.21561033609840605 valid loss: 0.1531929454455773 best epoch: 10 ======= epoch: 12 train loss: 0.21710317126578754 valid loss: 0.14967636888225874 best epoch: 12 ======= epoch: 13 train loss: 0.21094702864870613 valid loss: 0.14542658937474093 best epoch: 13 ======= epoch: 14 train loss: 0.2068421843427199 valid loss: 0.1427115686237812 best epoch: 14 ======= epoch: 15 train loss: 0.20408311711601268 valid loss: 0.13984219978253046 best epoch: 15 ======= epoch: 16 train loss: 0.20108160972135303 valid loss: 0.13686704821884632 best epoch: 16 ======= epoch: 17 train loss: 0.19746736779717017 valid loss: 0.13478184615572295 best epoch: 17 ======= epoch: 18 train loss: 0.1952096017330517 valid loss: 0.13305466622114182 best epoch: 18 ======= epoch: 19 train loss: 0.1931376403864519 valid loss: 0.13087865275641283 best epoch: 19 ======= epoch: 20 train loss: 0.19119209538639328 valid loss: 0.12890711737175783 best epoch: 20 ======= epoch: 21 train loss: 0.18953000416082363 valid loss: 0.1272704458485047 best epoch: 21 ======= epoch: 22 train loss: 0.18787718911505777 valid loss: 0.12593192420899868 best epoch: 22 ======= epoch: 23 train loss: 0.18622066338121154 valid loss: 0.12469574684898059 best epoch: 23 ======= epoch: 24 train loss: 0.18467787080616863 valid loss: 0.12319960196812947 best epoch: 24 ======= epoch: 25 train loss: 0.18313241133719316 valid loss: 0.12203957823415597 best epoch: 25 ======= epoch: 26 train loss: 0.1817236244218585 valid loss: 0.12065495985249679 best epoch: 26 ======= epoch: 27 train loss: 0.1803010180362581 valid loss: 0.11932762588063876 best epoch: 27 ======= epoch: 28 train loss: 0.1787945407408255 valid loss: 0.11802790313959122 best epoch: 28 ======= epoch: 29 train loss: 0.17709090834322536 valid loss: 0.1164020908375581 best epoch: 29 ======= epoch: 30 train loss: 0.175091760692957 valid loss: 0.1147329310576121 best epoch: 30 ======= epoch: 31 train loss: 0.1728601697832346 valid loss: 0.11342100116113822 best epoch: 31 ======= epoch: 32 train loss: 0.17079940215213063 valid loss: 0.11168361206849416 best epoch: 32 ======= epoch: 33 train loss: 0.16911093030630805 valid loss: 0.11033506443103154 best epoch: 33 ======= epoch: 34 train loss: 0.16773880985967907 valid loss: 0.10916137012342612 best epoch: 34 ======= epoch: 35 train loss: 0.1664615356259876 valid loss: 0.10816944142182668 best epoch: 35 ======= epoch: 36 train loss: 0.16517286531535197 valid loss: 0.10733194090425968 best epoch: 36 ======= epoch: 37 train loss: 0.16400821569065252 valid loss: 0.10647703396777312 best epoch: 37 ======= epoch: 38 train loss: 0.16282709582168378 valid loss: 0.10573883727192879 best epoch: 38 ======= epoch: 39 train loss: 0.1616902426575069 valid loss: 0.10496159022053082 best epoch: 39 ======= epoch: 40 train loss: 0.16063299733731481 valid loss: 0.1044571070621411 best epoch: 40 ======= epoch: 41 train loss: 0.15977046188013053 valid loss: 0.10372846387326717 best epoch: 41 ======= epoch: 42 train loss: 0.15880969769240896 valid loss: 0.10314853427310784 best epoch: 42 ======= epoch: 43 train loss: 0.1580141365620089 valid loss: 0.1024061981588602 best epoch: 43 ======= epoch: 44 train loss: 0.1572363655784248 valid loss: 0.1017798117051522 best epoch: 44 ======= epoch: 45 train loss: 0.15653479439608844 valid loss: 0.10130410889784495 best epoch: 45 ======= epoch: 46 train loss: 0.15585803962600084 valid loss: 0.10093476499120395 best epoch: 46 ======= epoch: 47 train loss: 0.155251299985397 valid loss: 0.10046326431135337 best epoch: 47 ======= epoch: 48 train loss: 0.1546534224424833 valid loss: 0.10018227808177471 best epoch: 48 ======= epoch: 49 train loss: 0.1540537411921922 valid loss: 0.0997743730743726 best epoch: 49 ======= epoch: 50 train loss: 0.15345020840565363 valid loss: 0.09943055051068465 best epoch: 50 ======= epoch: 51 train loss: 0.15282906633284357 valid loss: 0.09911005385220051 best epoch: 51 ======= epoch: 52 train loss: 0.15223700215143185 valid loss: 0.09872362638513248 best epoch: 52 ======= epoch: 53 train loss: 0.15164572869737944 valid loss: 0.09828647288183372 best epoch: 53 ======= epoch: 54 train loss: 0.15103650079281242 valid loss: 0.09797353421648343 best epoch: 54 ======= epoch: 55 train loss: 0.15041587253411612 valid loss: 0.09753502967456977 best epoch: 55 ======= epoch: 56 train loss: 0.14974322101032292 valid loss: 0.09713193277517955 best epoch: 56 ======= epoch: 57 train loss: 0.14903837626363026 valid loss: 0.09670732170343399 best epoch: 57 ======= epoch: 58 train loss: 0.14831168903612796 valid loss: 0.09634605733056863 best epoch: 58 ======= epoch: 59 train loss: 0.14762221067499967 valid loss: 0.09591176888595025 best epoch: 59 ======= epoch: 60 train loss: 0.14700353037519956 valid loss: 0.095652609753112 best epoch: 60 ======= epoch: 61 train loss: 0.14642718137690314 valid loss: 0.09536106574038665 best epoch: 61 ======= epoch: 62 train loss: 0.14589819475364538 valid loss: 0.09503065918882687 best epoch: 62 ======= epoch: 63 train loss: 0.1454174994594521 valid loss: 0.0947963393603762 best epoch: 63 ======= epoch: 64 train loss: 0.14491353685289254 valid loss: 0.09447363018989563 best epoch: 64 ======= epoch: 65 train loss: 0.1444300660794532 valid loss: 0.09423134662210941 best epoch: 65 ======= epoch: 66 train loss: 0.14395015493587213 valid loss: 0.09402700544645388 best epoch: 66 ======= epoch: 67 train loss: 0.1434685994905454 valid loss: 0.09379470503578584 best epoch: 67 ======= epoch: 68 train loss: 0.14298382433660237 valid loss: 0.09360756631940603 best epoch: 68 ======= epoch: 69 train loss: 0.1425060470714981 valid loss: 0.09339640444765489 best epoch: 69 ======= epoch: 70 train loss: 0.1420218787719438 valid loss: 0.09312314415971439 best epoch: 70 ======= epoch: 71 train loss: 0.1415356195727248 valid loss: 0.0929022990167141 best epoch: 71 ======= epoch: 72 train loss: 0.1410692119166071 valid loss: 0.0926263617972533 best epoch: 72 ======= epoch: 73 train loss: 0.14059759538482736 valid loss: 0.09240841058393319 best epoch: 73 ======= epoch: 74 train loss: 0.14013965906184397 valid loss: 0.0921506726493438 best epoch: 74 ======= epoch: 75 train loss: 0.13967295138188351 valid loss: 0.09200155083090067 best epoch: 75 ======= epoch: 76 train loss: 0.13919445812518214 valid loss: 0.09182512480765581 best epoch: 76 ======= epoch: 77 train loss: 0.13870950693977468 valid loss: 0.0916633444527785 best epoch: 77 ======= epoch: 78 train loss: 0.1382296207809338 valid loss: 0.09162312652915716 best epoch: 78 ======= epoch: 79 train loss: 0.1377729596340178 valid loss: 0.09138239454478025 best epoch: 79 ======= epoch: 80 train loss: 0.1373276626461266 valid loss: 0.0911738940825065 best epoch: 80 ======= epoch: 81 train loss: 0.1369071200168059 valid loss: 0.09094087624301513 best epoch: 81 ======= epoch: 82 train loss: 0.1364630480285412 valid loss: 0.09075646692266066 best epoch: 82 ======= epoch: 83 train loss: 0.13603791831360187 valid loss: 0.09050173591822386 best epoch: 83 ======= epoch: 84 train loss: 0.13561244549252738 valid loss: 0.09040703220913808 best epoch: 84 ======= epoch: 85 train loss: 0.13520143599605855 valid loss: 0.0901961512863636 best epoch: 85 ======= epoch: 86 train loss: 0.1348157988079352 valid loss: 0.09013295421997707 best epoch: 86 ======= epoch: 87 train loss: 0.13444915865720422 valid loss: 0.08987699014445145 best epoch: 87 ======= epoch: 88 train loss: 0.13408070161110824 valid loss: 0.08975908812135458 best epoch: 88 ======= epoch: 89 train loss: 0.1337205855589774 valid loss: 0.08958730722467105 best epoch: 89 ======= epoch: 90 train loss: 0.13335413759413325 valid loss: 0.08939941258480151 best epoch: 90 ======= epoch: 91 train loss: 0.13298286366517897 valid loss: 0.0892857713624835 best epoch: 91 ======= epoch: 92 train loss: 0.1326202401707386 valid loss: 0.08911892864853144 best epoch: 92 ======= epoch: 93 train loss: 0.13225818095429812 valid loss: 0.08889074965069692 best epoch: 93 ======= epoch: 94 train loss: 0.13189695782407565 valid loss: 0.08869586295137803 best epoch: 94 ======= epoch: 95 train loss: 0.1315414948089991 valid loss: 0.08847203198820353 best epoch: 95 ======= epoch: 96 train loss: 0.1312064920655555 valid loss: 0.08841258256385724 best epoch: 96 ======= epoch: 97 train loss: 0.1308811954854999 valid loss: 0.08812800577531259 best epoch: 97 ======= epoch: 98 train loss: 0.13056863621337178 valid loss: 0.08806908751527469 best epoch: 98 ======= epoch: 99 train loss: 0.1302386354195115 valid loss: 0.08780243030438821 best epoch: 99 ======= epoch: 100 train loss: 0.1299089867460691 valid loss: 0.08763670859237511 best epoch: 100 ======= epoch: 101 train loss: 0.1295648320681519 valid loss: 0.08745255880057812 best epoch: 101 ======= epoch: 102 train loss: 0.12921798078596225 valid loss: 0.08733636358131965 best epoch: 102 ======= epoch: 103 train loss: 0.1288792468768395 valid loss: 0.08715220323453347 best epoch: 103 ======= epoch: 104 train loss: 0.12854148416469494 valid loss: 0.08701358828693628 best epoch: 104 ======= epoch: 105 train loss: 0.1282028511143577 valid loss: 0.08687576465308666 best epoch: 105 ======= epoch: 106 train loss: 0.12790585315015948 valid loss: 0.08668080841501553 best epoch: 106 ======= epoch: 107 train loss: 0.12761198254417122 valid loss: 0.08653126284480095 best epoch: 107 ======= epoch: 108 train loss: 0.12730154984941086 valid loss: 0.08633183656881253 best epoch: 108 ======= epoch: 109 train loss: 0.12699187526272404 valid loss: 0.08612203535934289 best epoch: 109 ======= epoch: 110 train loss: 0.12665301531461287 valid loss: 0.08591429268320401 best epoch: 110 ======= epoch: 111 train loss: 0.12629861194916345 valid loss: 0.08568748893837135 best epoch: 111 ======= epoch: 112 train loss: 0.12591806520926363 valid loss: 0.08549944435556729 best epoch: 112 ======= epoch: 113 train loss: 0.1255227974558502 valid loss: 0.08518485352396965 best epoch: 113 ======= epoch: 114 train loss: 0.12513070208606897 valid loss: 0.08497320332874854 best epoch: 114 ======= epoch: 115 train loss: 0.12471816101237948 valid loss: 0.08469747720907132 best epoch: 115 ======= epoch: 116 train loss: 0.12429393100103846 valid loss: 0.08442534444232781 best epoch: 116 ======= epoch: 117 train loss: 0.12385088427245249 valid loss: 0.08410365010301273 best epoch: 117 ======= epoch: 118 train loss: 0.12339519801331156 valid loss: 0.0838572212184469 best epoch: 118 ======= epoch: 119 train loss: 0.12295688198948349 valid loss: 0.08357339693854253 best epoch: 119 ======= epoch: 120 train loss: 0.12251292808372298 valid loss: 0.08333123723665874 best epoch: 120 ======= epoch: 121 train loss: 0.12208784153156074 valid loss: 0.08313812998433907 best epoch: 121 ======= epoch: 122 train loss: 0.12169323390188777 valid loss: 0.08282787787417571 best epoch: 122 ======= epoch: 123 train loss: 0.12130308894003615 valid loss: 0.08262508890281121 best epoch: 123 ======= epoch: 124 train loss: 0.12093048852029038 valid loss: 0.08240001120915015 best epoch: 124 ======= epoch: 125 train loss: 0.12056980897201064 valid loss: 0.08225453831255436 best epoch: 125 ======= epoch: 126 train loss: 0.12022974553667469 valid loss: 0.08203641728808482 best epoch: 126 ======= epoch: 127 train loss: 0.11989367936082461 valid loss: 0.08190104768921931 best epoch: 127 ======= epoch: 128 train loss: 0.11958192823523725 valid loss: 0.08170468763758738 best epoch: 128 ======= epoch: 129 train loss: 0.119267887585325 valid loss: 0.08153551102926333 best epoch: 129 ======= epoch: 130 train loss: 0.1189598932310387 valid loss: 0.0813440065830946 best epoch: 130 ======= epoch: 131 train loss: 0.11864649964335525 valid loss: 0.08109783598532279 best epoch: 131 ======= epoch: 132 train loss: 0.11832268167011163 valid loss: 0.08091019839048386 best epoch: 132 ======= epoch: 133 train loss: 0.1180072229148613 valid loss: 0.08067462003479402 best epoch: 133 ======= epoch: 134 train loss: 0.11769326209424087 valid loss: 0.08051179566731055 best epoch: 134 ======= epoch: 135 train loss: 0.11739344198118758 valid loss: 0.080378416304787 best epoch: 135 ======= epoch: 136 train loss: 0.11710093703902798 valid loss: 0.08018997590988874 best epoch: 136 ======= epoch: 137 train loss: 0.11681791200635978 valid loss: 0.08010472139964502 best epoch: 137 ======= epoch: 138 train loss: 0.11653955931556814 valid loss: 0.07992145232856274 best epoch: 138 ======= epoch: 139 train loss: 0.11626996849606067 valid loss: 0.07977427169680595 best epoch: 139 ======= epoch: 140 train loss: 0.11598910041797308 valid loss: 0.07956972811371088 best epoch: 140 ======= epoch: 141 train loss: 0.11570391861468922 valid loss: 0.079411664356788 best epoch: 141 ======= epoch: 142 train loss: 0.11541185749947656 valid loss: 0.07923008656750123 best epoch: 142 ======= epoch: 143 train loss: 0.11510769150185364 valid loss: 0.07907516571382682 best epoch: 143 ======= epoch: 144 train loss: 0.11479644723420526 valid loss: 0.07893719027439754 best epoch: 144 ======= epoch: 145 train loss: 0.11449487548735407 valid loss: 0.07883436294893424 best epoch: 145 ======= epoch: 146 train loss: 0.11421015289508635 valid loss: 0.07866447015355031 best epoch: 146 ======= epoch: 147 train loss: 0.11392313613337868 valid loss: 0.07855271206547816 best epoch: 147 ======= epoch: 148 train loss: 0.11363535383601248 valid loss: 0.07847938779741526 best epoch: 148 ======= epoch: 149 train loss: 0.11335141138162141 valid loss: 0.07834603730589151 best epoch: 149 ======= epoch: 150 train loss: 0.11306639287022897 valid loss: 0.07828824563572805 best epoch: 150 ======= epoch: 151 train loss: 0.11279300583043585 valid loss: 0.07818963813285033 best epoch: 151 ======= epoch: 152 train loss: 0.11251007436694187 valid loss: 0.07807698380202055 best epoch: 152 ======= epoch: 153 train loss: 0.11222350906672669 valid loss: 0.07801211935778458 best epoch: 153 ======= epoch: 154 train loss: 0.11192810330769898 valid loss: 0.07783307880163193 best epoch: 154 ======= epoch: 155 train loss: 0.11163205150974753 valid loss: 0.07772417242328326 best epoch: 155 ======= epoch: 156 train loss: 0.11133829881379634 valid loss: 0.07757092857112487 best epoch: 156 ======= epoch: 157 train loss: 0.11105299338606404 valid loss: 0.07750914245843887 best epoch: 157 ======= epoch: 158 train loss: 0.1107637055080246 valid loss: 0.07736630303164323 best epoch: 158 ======= epoch: 159 train loss: 0.11048542182708596 valid loss: 0.0772836289058129 best epoch: 159 ======= epoch: 160 train loss: 0.11021967762276347 valid loss: 0.07716525439172983 best epoch: 160 ======= epoch: 161 train loss: 0.10995798349886397 valid loss: 0.07705173299958308 best epoch: 161 ======= epoch: 162 train loss: 0.10969894099980593 valid loss: 0.0769294419636329 best epoch: 162 ======= epoch: 163 train loss: 0.10944682935735693 valid loss: 0.07682119651387136 best epoch: 163 ======= epoch: 164 train loss: 0.10920748034101209 valid loss: 0.07671342665950458 best epoch: 164 ======= epoch: 165 train loss: 0.10897966078770013 valid loss: 0.0766112372900049 best epoch: 165 ======= epoch: 166 train loss: 0.10875058483424378 valid loss: 0.07652026942620675 best epoch: 166 ======= epoch: 167 train loss: 0.10853846741403327 valid loss: 0.07642008705685537 best epoch: 167 ======= epoch: 168 train loss: 0.1083255763637063 valid loss: 0.07633871336778005 best epoch: 168 ======= epoch: 169 train loss: 0.10812019690135379 valid loss: 0.07622725330293179 best epoch: 169 ======= epoch: 170 train loss: 0.10792075462041446 valid loss: 0.07618350225190322 best epoch: 170 ======= epoch: 171 train loss: 0.10772638103384295 valid loss: 0.07608196729173262 best epoch: 171 ======= epoch: 172 train loss: 0.10753585169390764 valid loss: 0.07593227364122868 best epoch: 172 ======= epoch: 173 train loss: 0.10734908862246408 valid loss: 0.07583351743717988 best epoch: 173 ======= epoch: 174 train loss: 0.1071651965080772 valid loss: 0.07577955070883036 best epoch: 174 ======= epoch: 175 train loss: 0.10699316121085926 valid loss: 0.07560405476639669 best epoch: 175 ======= epoch: 176 train loss: 0.10682726397328539 valid loss: 0.07556098885834217 best epoch: 176 ======= epoch: 177 train loss: 0.1066615207373728 valid loss: 0.07540871482342482 best epoch: 177 ======= epoch: 178 train loss: 0.10649845688201395 valid loss: 0.07531197275966406 best epoch: 178 ======= epoch: 179 train loss: 0.10633550656752454 valid loss: 0.07517431676387787 best epoch: 179 ======= epoch: 180 train loss: 0.10617423965883108 valid loss: 0.07504943789293368 best epoch: 180 ======= epoch: 181 train loss: 0.1060101333775638 valid loss: 0.07495108433067799 best epoch: 181 ======= epoch: 182 train loss: 0.10584431762496631 valid loss: 0.07483604519317548 best epoch: 182 ======= epoch: 183 train loss: 0.1056786988183488 valid loss: 0.07475675207873185 best epoch: 183 ======= epoch: 184 train loss: 0.10551274403423806 valid loss: 0.07463701566060384 best epoch: 184 ======= epoch: 185 train loss: 0.10536746394441084 valid loss: 0.07459784640620153 best epoch: 185 ======= epoch: 186 train loss: 0.10522130176562954 valid loss: 0.07448210163662831 best epoch: 186 ======= epoch: 187 train loss: 0.1050840172295769 valid loss: 0.07444423250854015 best epoch: 187 ======= epoch: 188 train loss: 0.1049481616527946 valid loss: 0.07435660436749458 best epoch: 188 ======= epoch: 189 train loss: 0.10481239697769468 valid loss: 0.07427195770045121 best epoch: 189 ======= epoch: 190 train loss: 0.10467223832268774 valid loss: 0.07417177439977725 best epoch: 190 ======= epoch: 191 train loss: 0.10452853185757074 valid loss: 0.0740813659504056 best epoch: 191 ======= epoch: 192 train loss: 0.10437674676700506 valid loss: 0.0739681925624609 best epoch: 192 ======= epoch: 193 train loss: 0.10422630989809095 valid loss: 0.07385442188630502 best epoch: 193 ======= epoch: 194 train loss: 0.10408089335225983 valid loss: 0.07372726034373045 best epoch: 194 ======= epoch: 195 train loss: 0.10393891812382657 valid loss: 0.07367233869930108 best epoch: 195 ======= epoch: 196 train loss: 0.10379412701284443 valid loss: 0.07354089710861444 best epoch: 196 ======= epoch: 197 train loss: 0.1036495591548306 valid loss: 0.07347580511122942 best epoch: 197 ======= epoch: 198 train loss: 0.1035129165759793 valid loss: 0.07334824961920579 best epoch: 198 ======= epoch: 199 train loss: 0.10337418478764134 valid loss: 0.07328980043530464 best epoch: 199 ======= epoch: 200 train loss: 0.10323559920544978 valid loss: 0.07319104950875044 best epoch: 200 ======= epoch: 201 train loss: 0.10309373886312967 valid loss: 0.07311131029079358 best epoch: 201 ======= epoch: 202 train loss: 0.10295831237310245 valid loss: 0.0730396502961715 best epoch: 202 ======= epoch: 203 train loss: 0.10282271316297996 valid loss: 0.07291635560492675 best epoch: 203 ======= epoch: 204 train loss: 0.10269020811689121 valid loss: 0.07280802012731631 best epoch: 204 ======= epoch: 205 train loss: 0.10255125446681991 valid loss: 0.07272388723989327 best epoch: 205 ======= epoch: 206 train loss: 0.10240828781676145 valid loss: 0.0726319815342625 best epoch: 206 ======= epoch: 207 train loss: 0.10226582076952413 valid loss: 0.07255444675683975 best epoch: 207 ======= epoch: 208 train loss: 0.10212389061250436 valid loss: 0.07248408005883296 best epoch: 208 ======= epoch: 209 train loss: 0.10199804389044459 valid loss: 0.07240066149582465 best epoch: 209 ======= epoch: 210 train loss: 0.10186336142367419 valid loss: 0.07230476383119822 best epoch: 210 ======= epoch: 211 train loss: 0.10173077968719565 valid loss: 0.07222506279746692 best epoch: 211 ======= epoch: 212 train loss: 0.10159778887015065 valid loss: 0.07215406652539968 best epoch: 212 ======= epoch: 213 train loss: 0.10146352404981483 valid loss: 0.072106731745104 best epoch: 213 ======= epoch: 214 train loss: 0.10132472158444149 valid loss: 0.07201957764724891 best epoch: 214 ======= epoch: 215 train loss: 0.10118025402182046 valid loss: 0.07193793635815382 best epoch: 215 ======= epoch: 216 train loss: 0.1010397572507277 valid loss: 0.07189675762007634 best epoch: 216 ======= epoch: 217 train loss: 0.10090728668656981 valid loss: 0.07181328162550926 best epoch: 217 ======= epoch: 218 train loss: 0.10077662664799043 valid loss: 0.0717478096485138 best epoch: 218 ======= epoch: 219 train loss: 0.10064889556141915 valid loss: 0.07169181512047847 best epoch: 219 ======= epoch: 220 train loss: 0.10052060065676033 valid loss: 0.07162779942154884 best epoch: 220 ======= epoch: 221 train loss: 0.10039111878722906 valid loss: 0.0715728619446357 best epoch: 221 ======= epoch: 222 train loss: 0.10026138194227292 valid loss: 0.07151515719791253 best epoch: 222 ======= epoch: 223 train loss: 0.10013054151455929 valid loss: 0.07147282393028338 best epoch: 223 ======= epoch: 224 train loss: 0.09999714459488053 valid loss: 0.07144494323680799 best epoch: 224 ======= epoch: 225 train loss: 0.09985765320375378 valid loss: 0.07138792828967173 best epoch: 225 ======= epoch: 226 train loss: 0.09971866179487587 valid loss: 0.0713529825831453 best epoch: 226 ======= epoch: 227 train loss: 0.09957765649866175 valid loss: 0.07129399416347344 best epoch: 227 ======= epoch: 228 train loss: 0.09943221647430349 valid loss: 0.07125361325840156 best epoch: 228 ======= epoch: 229 train loss: 0.09928647601218135 valid loss: 0.07122686598449945 best epoch: 229 ======= epoch: 230 train loss: 0.09913865979300973 valid loss: 0.07120456670721372 best epoch: 230 ======= epoch: 231 train loss: 0.09899024935554207 valid loss: 0.07116312937190135 best epoch: 231 ======= epoch: 232 train loss: 0.0988489845537661 valid loss: 0.07112740632146597 best epoch: 232 ======= epoch: 233 train loss: 0.09870787055726037 valid loss: 0.07111933610091607 best epoch: 233 ======= epoch: 234 train loss: 0.0985842734851219 valid loss: 0.07108307971308629 best epoch: 234 ======= epoch: 235 train loss: 0.09844594206201074 valid loss: 0.07104504791398843 best epoch: 235 ======= epoch: 236 train loss: 0.09831669157439912 valid loss: 0.07101928132275741 best epoch: 236 ======= epoch: 237 train loss: 0.09817566583507958 valid loss: 0.07098563015460968 best epoch: 237 ======= epoch: 238 train loss: 0.09803137824767166 valid loss: 0.07095190354933341 best epoch: 238 ======= epoch: 239 train loss: 0.09788482246437559 valid loss: 0.07092252653092146 best epoch: 239 ======= epoch: 240 train loss: 0.09773756164689491 valid loss: 0.07088483932117622 best epoch: 240 ======= epoch: 241 train loss: 0.09759280909956605 valid loss: 0.07087155183156331 best epoch: 241 ======= epoch: 242 train loss: 0.09745581895342947 valid loss: 0.0708369817584753 best epoch: 242 ======= epoch: 243 train loss: 0.09732692526584422 valid loss: 0.07079259896030028 best epoch: 243 ======= epoch: 244 train loss: 0.09719901118013594 valid loss: 0.07077700272202492 best epoch: 244 ======= epoch: 245 train loss: 0.09707290296339327 valid loss: 0.07072930100063483 best epoch: 245 ======= epoch: 246 train loss: 0.0969485738027243 valid loss: 0.0706891513740023 best epoch: 246 ======= epoch: 247 train loss: 0.09682473819702864 valid loss: 0.07065771706402302 best epoch: 247 ======= epoch: 248 train loss: 0.09669464771394376 valid loss: 0.07061746871719758 best epoch: 248 ======= epoch: 249 train loss: 0.09656129750986527 valid loss: 0.07058504782617092 best epoch: 249 ======= epoch: 250 train loss: 0.09642263956046031 valid loss: 0.07054829845825832 best epoch: 250 ======= epoch: 251 train loss: 0.0962785702244735 valid loss: 0.07051846850663424 best epoch: 251 ======= epoch: 252 train loss: 0.0961287949970107 valid loss: 0.07047038308034341 best epoch: 252 ======= epoch: 253 train loss: 0.09597507518152763 valid loss: 0.07043077113727729 best epoch: 253 ======= epoch: 254 train loss: 0.09582529755102263 valid loss: 0.0703937237461408 best epoch: 254 ======= epoch: 255 train loss: 0.09568736618269741 valid loss: 0.07032817633201678 best epoch: 255 ======= epoch: 256 train loss: 0.09555788841788415 valid loss: 0.07032434766491254 best epoch: 256 ======= epoch: 257 train loss: 0.09542893559706432 valid loss: 0.0703099196155866 best epoch: 257 ======= epoch: 258 train loss: 0.09530235043968316 valid loss: 0.07028749585151672 best epoch: 258 ======= epoch: 259 train loss: 0.09517063178627948 valid loss: 0.07026491779834032 best epoch: 259 ======= epoch: 260 train loss: 0.09503659507098758 valid loss: 0.07024055284758408 best epoch: 260 ======= epoch: 261 train loss: 0.09489542539059012 valid loss: 0.07020301092416048 best epoch: 261 ======= epoch: 262 train loss: 0.09475126073776204 valid loss: 0.0701703770707051 best epoch: 262 ======= epoch: 263 train loss: 0.09459735768168796 valid loss: 0.07014116458594799 best epoch: 263 ======= epoch: 264 train loss: 0.09443930941231457 valid loss: 0.07010872972508271 best epoch: 264 ======= epoch: 265 train loss: 0.09427301225792846 valid loss: 0.07008877862244844 best epoch: 265 ======= epoch: 266 train loss: 0.09411089575309077 valid loss: 0.07005552388727665 best epoch: 266 ======= epoch: 267 train loss: 0.0939632040551967 valid loss: 0.07003466195116441 best epoch: 267 ======= epoch: 268 train loss: 0.09381024108302814 valid loss: 0.07000413164496422 best epoch: 268 ======= epoch: 269 train loss: 0.0936588755245378 valid loss: 0.06998477503657341 best epoch: 269 ======= epoch: 270 train loss: 0.09350394287411078 valid loss: 0.06995798150698344 best epoch: 270 ======= epoch: 271 train loss: 0.09334033355116844 valid loss: 0.06993772120525439 best epoch: 271 ======= epoch: 272 train loss: 0.09316529442820652 valid loss: 0.06991271457324426 best epoch: 272 ======= epoch: 273 train loss: 0.09298096212018052 valid loss: 0.06988347756365935 best epoch: 273 ======= epoch: 274 train loss: 0.0927912676476954 valid loss: 0.06985792548706134 best epoch: 274 ======= epoch: 275 train loss: 0.09261049945367339 valid loss: 0.06983152652780215 best epoch: 275 ======= epoch: 276 train loss: 0.09242445182193208 valid loss: 0.06978236604481936 best epoch: 276 ======= epoch: 277 train loss: 0.09224153043311319 valid loss: 0.06975737897058328 best epoch: 277 ======= epoch: 278 train loss: 0.09205403833155647 valid loss: 0.06973692122846842 best epoch: 278 ======= epoch: 279 train loss: 0.09185755308027621 valid loss: 0.06970278608302276 best epoch: 279 ======= epoch: 280 train loss: 0.09165218398894792 valid loss: 0.06966233781228463 best epoch: 280 ======= epoch: 281 train loss: 0.09144121035933495 valid loss: 0.06960861198604107 best epoch: 281 ======= epoch: 282 train loss: 0.09122937155404577 valid loss: 0.06959617789834738 best epoch: 282 ======= epoch: 283 train loss: 0.09103992059366939 valid loss: 0.06952677791317304 best epoch: 283 ======= epoch: 284 train loss: 0.09084609002564792 valid loss: 0.06950352092583974 best epoch: 284 ======= epoch: 285 train loss: 0.09065021346840593 valid loss: 0.06946300280590852 best epoch: 285 ======= epoch: 286 train loss: 0.0904604665544114 valid loss: 0.0694188525279363 best epoch: 286 ======= epoch: 287 train loss: 0.09026291052362433 valid loss: 0.06938355136662722 best epoch: 287 ======= epoch: 288 train loss: 0.0900600193137372 valid loss: 0.06935223968078692 best epoch: 288 ======= epoch: 289 train loss: 0.08985206319226159 valid loss: 0.06931098860998948 best epoch: 289 ======= epoch: 290 train loss: 0.08964077207363313 valid loss: 0.06924102734774351 best epoch: 290 ======= epoch: 291 train loss: 0.08943362405270706 valid loss: 0.06921242860456307 best epoch: 291 ======= epoch: 292 train loss: 0.08924174475495095 valid loss: 0.06916376482695341 best epoch: 292 ======= epoch: 293 train loss: 0.08904617038313989 valid loss: 0.06911152973771095 best epoch: 293 ======= epoch: 294 train loss: 0.08887635322034727 valid loss: 0.06907481420785189 best epoch: 294 ======= epoch: 295 train loss: 0.08868781990015212 valid loss: 0.06900944374501705 best epoch: 295 ======= epoch: 296 train loss: 0.08850680902186367 valid loss: 0.06897218897938728 best epoch: 296 ======= epoch: 297 train loss: 0.08832723316219118 valid loss: 0.06889858065793912 best epoch: 297 ======= epoch: 298 train loss: 0.08814968153788352 valid loss: 0.06885003671050072 best epoch: 298 ======= epoch: 299 train loss: 0.08797158130709036 valid loss: 0.0687793077280124 best epoch: 299 ======= epoch: 300 train loss: 0.08779692533713432 valid loss: 0.06873745564371347 best epoch: 300 ======= epoch: 301 train loss: 0.08762165277414484 valid loss: 0.06867575086653233 best epoch: 301 ======= epoch: 302 train loss: 0.08745603648554763 valid loss: 0.06861023046076298 best epoch: 302 ======= epoch: 303 train loss: 0.08728563524552702 valid loss: 0.06855609981964032 best epoch: 303 ======= epoch: 304 train loss: 0.08711955957344653 valid loss: 0.06848240705827872 best epoch: 304 ======= epoch: 305 train loss: 0.08695737240307125 valid loss: 0.06841387848059337 best epoch: 305 ======= epoch: 306 train loss: 0.08679395748509301 valid loss: 0.06834766765435536 best epoch: 306 ======= epoch: 307 train loss: 0.08662878907067173 valid loss: 0.06828826386481524 best epoch: 307 ======= epoch: 308 train loss: 0.08645798891414831 valid loss: 0.06822289278109868 best epoch: 308 ======= epoch: 309 train loss: 0.08629308046520125 valid loss: 0.0681612476085623 best epoch: 309 ======= epoch: 310 train loss: 0.08612913376203289 valid loss: 0.06809098273515701 best epoch: 310 ======= epoch: 311 train loss: 0.0859718912821493 valid loss: 0.0680596437305212 best epoch: 311 ======= epoch: 312 train loss: 0.08582193123889559 valid loss: 0.06799207100023826 best epoch: 312 ======= epoch: 313 train loss: 0.08567820813644815 valid loss: 0.06794432426492374 best epoch: 313 ======= epoch: 314 train loss: 0.08553419571461869 valid loss: 0.06788442687441905 best epoch: 314 ======= epoch: 315 train loss: 0.08538955845582633 valid loss: 0.06782277828703324 best epoch: 315 ======= epoch: 316 train loss: 0.08524442636580379 valid loss: 0.06777281935016315 best epoch: 316 ======= epoch: 317 train loss: 0.0850993430264938 valid loss: 0.0677174727121989 best epoch: 317 ======= epoch: 318 train loss: 0.08495868549302772 valid loss: 0.06768008445700009 best epoch: 318 ======= epoch: 319 train loss: 0.08482201525044662 valid loss: 0.06763334944844246 best epoch: 319 ======= epoch: 320 train loss: 0.08468766868068481 valid loss: 0.06759342551231384 best epoch: 320 ======= epoch: 321 train loss: 0.08455002582689494 valid loss: 0.06754038451860349 best epoch: 321 ======= epoch: 322 train loss: 0.08441371440795468 valid loss: 0.06750055868178606 best epoch: 322 ======= epoch: 323 train loss: 0.0842752578081908 valid loss: 0.06745049667855103 best epoch: 323 ======= epoch: 324 train loss: 0.08414308228932413 valid loss: 0.0674143557747205 best epoch: 324 ======= epoch: 325 train loss: 0.08401654890657943 valid loss: 0.06737163414557774 best epoch: 325 ======= epoch: 326 train loss: 0.08388545590648312 valid loss: 0.06733238448699315 best epoch: 326 ======= epoch: 327 train loss: 0.08375077271535072 valid loss: 0.06729676605512698 best epoch: 327 ======= epoch: 328 train loss: 0.08361548227890406 valid loss: 0.06725437628726165 best epoch: 328 ======= epoch: 329 train loss: 0.08348065082351734 valid loss: 0.06719494021187226 best epoch: 329 ======= epoch: 330 train loss: 0.08335207294626737 valid loss: 0.06716875514636438 best epoch: 330 ======= epoch: 331 train loss: 0.08322499210681812 valid loss: 0.0671137881775697 best epoch: 331 ======= epoch: 332 train loss: 0.0831051158523302 valid loss: 0.06708369528253873 best epoch: 332 ======= epoch: 333 train loss: 0.08298418393795505 valid loss: 0.06703507838149865 best epoch: 333 ======= epoch: 334 train loss: 0.08286418126505098 valid loss: 0.06697516019145648 best epoch: 334 ======= epoch: 335 train loss: 0.08275392514733988 valid loss: 0.06693818358083566 best epoch: 335 ======= epoch: 336 train loss: 0.08263972563738073 valid loss: 0.06689522167046864 best epoch: 336 ======= epoch: 337 train loss: 0.08253031814999787 valid loss: 0.06686531069378059 best epoch: 337 ======= epoch: 338 train loss: 0.08242158707875161 valid loss: 0.06681773128608863 best epoch: 338 ======= epoch: 339 train loss: 0.08231099990460976 valid loss: 0.06677747269471486 best epoch: 339 ======= epoch: 340 train loss: 0.08219750546332863 valid loss: 0.06673673850794633 best epoch: 340 ======= epoch: 341 train loss: 0.08208084781366734 valid loss: 0.06669996989270051 best epoch: 341 ======= epoch: 342 train loss: 0.08196268912496758 valid loss: 0.06663163689275582 best epoch: 342 ======= epoch: 343 train loss: 0.08184739692067658 valid loss: 0.06660479369262855 best epoch: 343 ======= epoch: 344 train loss: 0.08173947712337529 valid loss: 0.06653822089234988 best epoch: 344 ======= epoch: 345 train loss: 0.0816347068865542 valid loss: 0.06650629794845979 best epoch: 345 ======= epoch: 346 train loss: 0.08153367088532743 valid loss: 0.06646898450950782 best epoch: 346 ======= epoch: 347 train loss: 0.08143249496725606 valid loss: 0.06642462126910686 best epoch: 347 ======= epoch: 348 train loss: 0.08133187572713252 valid loss: 0.06638397990415494 best epoch: 348 ======= epoch: 349 train loss: 0.08122838048241389 valid loss: 0.06631852438052495 best epoch: 349 ======= epoch: 350 train loss: 0.08113330197922977 valid loss: 0.0662832222878933 best epoch: 350 ======= epoch: 351 train loss: 0.08103530338884872 valid loss: 0.06623158883303404 best epoch: 351 ======= epoch: 352 train loss: 0.08093828161787471 valid loss: 0.06618762295693159 best epoch: 352 ======= epoch: 353 train loss: 0.08084186967922213 valid loss: 0.06613262463361025 best epoch: 353 ======= epoch: 354 train loss: 0.08073949837988173 valid loss: 0.06607044829676549 best epoch: 354 ======= epoch: 355 train loss: 0.08063434524300658 valid loss: 0.06604013436784346 best epoch: 355 ======= epoch: 356 train loss: 0.08052810535616713 valid loss: 0.06597983123113711 best epoch: 356 ======= epoch: 357 train loss: 0.08042015856200899 valid loss: 0.06594987617184718 best epoch: 357 ======= epoch: 358 train loss: 0.08031657565799025 valid loss: 0.06589569865415494 best epoch: 358 ======= epoch: 359 train loss: 0.08021525742003212 valid loss: 0.06585237725327413 best epoch: 359 ======= epoch: 360 train loss: 0.080113249083544 valid loss: 0.0658049900084734 best epoch: 360 ======= epoch: 361 train loss: 0.08001591614734979 valid loss: 0.06575323610256116 best epoch: 361 ======= epoch: 362 train loss: 0.07991843356716412 valid loss: 0.06572555750608444 best epoch: 362 ======= epoch: 363 train loss: 0.07982466220395801 valid loss: 0.06568192162861426 best epoch: 363 ======= epoch: 364 train loss: 0.07973077180392948 valid loss: 0.06565520291527112 best epoch: 364 ======= epoch: 365 train loss: 0.07963945983368674 valid loss: 0.06560890221347411 best epoch: 365 ======= epoch: 366 train loss: 0.07954825518399845 valid loss: 0.06559158458064 best epoch: 366 ======= epoch: 367 train loss: 0.0794619569255982 valid loss: 0.06551992458601792 best epoch: 367 ======= epoch: 368 train loss: 0.07937502082621242 valid loss: 0.06550674047321081 best epoch: 368 ======= epoch: 369 train loss: 0.07928840565368717 valid loss: 0.06546931186070044 best epoch: 369 ======= epoch: 370 train loss: 0.0791981703153363 valid loss: 0.06543699465692043 best epoch: 370 ======= epoch: 371 train loss: 0.0791106068379717 valid loss: 0.06540179438889027 best epoch: 371 ======= epoch: 372 train loss: 0.07902495569928929 valid loss: 0.06537143203119437 best epoch: 372 ======= epoch: 373 train loss: 0.07893793461959303 valid loss: 0.06533349988361199 best epoch: 373 ======= epoch: 374 train loss: 0.07885149108636895 valid loss: 0.06528929993510246 best epoch: 374 ======= epoch: 375 train loss: 0.07877153429536173 valid loss: 0.06527663550029199 best epoch: 375 ======= epoch: 376 train loss: 0.07869220577916246 valid loss: 0.06523990631103516 best epoch: 376 ======= epoch: 377 train loss: 0.07862185252209504 valid loss: 0.06521404845019181 best epoch: 377 ======= epoch: 378 train loss: 0.07854794599345805 valid loss: 0.06519374903291464 best epoch: 378 ======= epoch: 379 train loss: 0.07847832731994582 valid loss: 0.06516851236422856 best epoch: 379 ======= epoch: 380 train loss: 0.07841048316087251 valid loss: 0.06514435851325591 best epoch: 380 ======= epoch: 381 train loss: 0.07833937948776616 valid loss: 0.06511158837626378 best epoch: 381 ======= epoch: 382 train loss: 0.07826524594637715 valid loss: 0.06509184464812279 best epoch: 382 ======= epoch: 383 train loss: 0.07819320591465559 valid loss: 0.06505838129669428 best epoch: 383 ======= epoch: 384 train loss: 0.07812260352305056 valid loss: 0.06503027305006981 best epoch: 384 ======= epoch: 385 train loss: 0.07805008417064394 valid loss: 0.06499902221063773 best epoch: 385 ======= epoch: 386 train loss: 0.07797753070423633 valid loss: 0.06496966754396756 best epoch: 386 ======= epoch: 387 train loss: 0.07790325969503617 valid loss: 0.0649530366063118 best epoch: 387 ======= epoch: 388 train loss: 0.07782784498722096 valid loss: 0.06491128044823806 best epoch: 388 ======= epoch: 389 train loss: 0.07775430053610492 valid loss: 0.06490117652962606 best epoch: 389 ======= epoch: 390 train loss: 0.07768209723548757 valid loss: 0.06486912692586581 best epoch: 390 ======= epoch: 391 train loss: 0.07761222650699409 valid loss: 0.06485250468055408 best epoch: 391 ======= epoch: 392 train loss: 0.07754274865865339 valid loss: 0.06482812203466892 best epoch: 392 ======= epoch: 393 train loss: 0.07747456355503311 valid loss: 0.06481074883292119 best epoch: 393 ======= epoch: 394 train loss: 0.07740385262410582 valid loss: 0.06479623386015494 best epoch: 394 ======= epoch: 395 train loss: 0.07733473849747284 valid loss: 0.06476498271028201 best epoch: 395 ======= epoch: 396 train loss: 0.07726459997349683 valid loss: 0.06474942155182362 best epoch: 396 ======= epoch: 397 train loss: 0.07719449813903113 valid loss: 0.06471920168648164 best epoch: 397 ======= epoch: 398 train loss: 0.07712468358883519 valid loss: 0.06470748378584783 best epoch: 398 ======= epoch: 399 train loss: 0.07705479883301405 valid loss: 0.0646896818652749 best epoch: 399 ======= epoch: 400 train loss: 0.07698839982212694 valid loss: 0.06468405947089195 best epoch: 400 ======= epoch: 401 train loss: 0.07692703293107542 valid loss: 0.06466747137407462 best epoch: 401 ======= epoch: 402 train loss: 0.07686665788706806 valid loss: 0.06464454438537359 best epoch: 402 ======= epoch: 403 train loss: 0.07680647046431716 valid loss: 0.06463513523340225 best epoch: 403 ======= epoch: 404 train loss: 0.07675161194286229 valid loss: 0.06461032014340162 best epoch: 404 ======= epoch: 405 train loss: 0.07669200924121672 valid loss: 0.0645907698199153 best epoch: 405 ======= epoch: 406 train loss: 0.07663442418669109 valid loss: 0.06456407283743222 best epoch: 406 ======= epoch: 407 train loss: 0.07657481577063416 valid loss: 0.06452947109937668 best epoch: 407 ======= epoch: 408 train loss: 0.07651244596014788 valid loss: 0.06450927288581927 best epoch: 408 ======= epoch: 409 train loss: 0.07644747198777802 valid loss: 0.06447554690142472 best epoch: 409 ======= epoch: 410 train loss: 0.07638354692608118 valid loss: 0.0644563337167104 best epoch: 410 ======= epoch: 411 train loss: 0.07632123064939622 valid loss: 0.0644295917203029 best epoch: 411 ======= epoch: 412 train loss: 0.07625715300026867 valid loss: 0.06440057295064132 best epoch: 412 ======= epoch: 413 train loss: 0.07619284250899966 valid loss: 0.06436759233474731 best epoch: 413 ======= epoch: 414 train loss: 0.07613243721425533 valid loss: 0.06434025056660175 best epoch: 414 ======= epoch: 415 train loss: 0.07607147772509007 valid loss: 0.06431298423558474 best epoch: 415 ======= epoch: 416 train loss: 0.07601097982698753 valid loss: 0.06428466240564983 best epoch: 416 ======= epoch: 417 train loss: 0.07594862902238045 valid loss: 0.06425549524525802 best epoch: 417 ======= epoch: 418 train loss: 0.07588613518502241 valid loss: 0.06421970545003812 best epoch: 418 ======= epoch: 419 train loss: 0.07582496203206203 valid loss: 0.06418833664307992 best epoch: 419 ======= epoch: 420 train loss: 0.07576239336695936 valid loss: 0.06415257633974154 best epoch: 420 ======= epoch: 421 train loss: 0.07569730190997138 valid loss: 0.06411026852826278 best epoch: 421 ======= epoch: 422 train loss: 0.07563320746254405 valid loss: 0.06409974054743846 best epoch: 422 ======= epoch: 423 train loss: 0.07557192217144701 valid loss: 0.06405474555989106 best epoch: 423 ======= epoch: 424 train loss: 0.0755123968500598 valid loss: 0.06405256688594818 best epoch: 424 ======= epoch: 425 train loss: 0.07545535257201136 valid loss: 0.06403222338606913 best epoch: 425 ======= epoch: 426 train loss: 0.07539752007312613 valid loss: 0.06401276029646397 best epoch: 426 ======= epoch: 427 train loss: 0.07534305614499766 valid loss: 0.06398896748820941 best epoch: 427 ======= epoch: 428 train loss: 0.07529031550856652 valid loss: 0.06398049959292014 best epoch: 428 ======= epoch: 429 train loss: 0.07523524893424761 valid loss: 0.06396342410395543 best epoch: 429 ======= epoch: 430 train loss: 0.07517903720109183 valid loss: 0.06395008942733209 best epoch: 430 ======= epoch: 431 train loss: 0.07512301740087109 valid loss: 0.06393647442261378 best epoch: 431 ======= epoch: 432 train loss: 0.07506591494390626 valid loss: 0.06391834436605374 best epoch: 432 ======= epoch: 433 train loss: 0.07501119392475596 valid loss: 0.0639155072470506 best epoch: 433 ======= epoch: 434 train loss: 0.07495735409773058 valid loss: 0.06389836159845193 best epoch: 434 ======= epoch: 435 train loss: 0.07490308866778642 valid loss: 0.0638897434497873 best epoch: 435 ======= epoch: 436 train loss: 0.07485169881701838 valid loss: 0.0638801126430432 best epoch: 436 ======= epoch: 437 train loss: 0.07479956058532368 valid loss: 0.06386742958178122 best epoch: 437 ======= epoch: 438 train loss: 0.07474754666794597 valid loss: 0.06383966809759538 best epoch: 438 ======= epoch: 439 train loss: 0.07469402048966767 valid loss: 0.06383614428341389 best epoch: 439 ======= epoch: 440 train loss: 0.07464252493953631 valid loss: 0.06380748903999726 best epoch: 440 ======= epoch: 441 train loss: 0.07459541721798388 valid loss: 0.06378340659042199 best epoch: 441 ======= epoch: 442 train loss: 0.07454882260917882 valid loss: 0.06377044878900051 best epoch: 442 ======= epoch: 443 train loss: 0.07450301141512615 valid loss: 0.06376341233650844 best epoch: 443 ======= epoch: 444 train loss: 0.07445890814019943 valid loss: 0.06375240844984849 best epoch: 444 ======= epoch: 445 train loss: 0.07441390534563933 valid loss: 0.06375281202296416 best epoch: 444 ======= epoch: 446 train loss: 0.07437078300633548 valid loss: 0.06372530727336805 best epoch: 446 ======= epoch: 447 train loss: 0.0743291071259681 valid loss: 0.06372851878404617 best epoch: 446 ======= epoch: 448 train loss: 0.07428656793256969 valid loss: 0.06371424129853646 best epoch: 448 ======= epoch: 449 train loss: 0.07424527390595571 valid loss: 0.06370211485773325 best epoch: 449 ======= epoch: 450 train loss: 0.07420337196715453 valid loss: 0.06368863582611084 best epoch: 450 ======= epoch: 451 train loss: 0.07416108467926581 valid loss: 0.06368029210716486 best epoch: 451 ======= epoch: 452 train loss: 0.07411977109487777 valid loss: 0.06366233993321657 best epoch: 452 ======= epoch: 453 train loss: 0.07407683053594312 valid loss: 0.06365713166693847 best epoch: 453 ======= epoch: 454 train loss: 0.07403467953159118 valid loss: 0.06364951469004154 best epoch: 454 ======= epoch: 455 train loss: 0.07399321292285566 valid loss: 0.06363426521420479 best epoch: 455 ======= epoch: 456 train loss: 0.07395088404738977 valid loss: 0.06361676814655463 best epoch: 456 ======= epoch: 457 train loss: 0.0739099861778043 valid loss: 0.06360316338638465 best epoch: 457 ======= epoch: 458 train loss: 0.07386792460341512 valid loss: 0.06358847425629695 best epoch: 458 ======= epoch: 459 train loss: 0.07382686333845795 valid loss: 0.06358448540170987 best epoch: 459 ======= epoch: 460 train loss: 0.07378904053504452 valid loss: 0.06357409587750833 best epoch: 460 ======= epoch: 461 train loss: 0.07374997119660731 valid loss: 0.06355473678559065 best epoch: 461 ======= epoch: 462 train loss: 0.07371153785950607 valid loss: 0.06353730615228415 best epoch: 462 ======= epoch: 463 train loss: 0.07367325552313785 valid loss: 0.06352417264133692 best epoch: 463 ======= epoch: 464 train loss: 0.0736344547390386 valid loss: 0.06349544848004977 best epoch: 464 ======= epoch: 465 train loss: 0.0735961149941072 valid loss: 0.06349087102959554 best epoch: 465 ======= epoch: 466 train loss: 0.07355720670548854 valid loss: 0.0634693323324124 best epoch: 466 ======= epoch: 467 train loss: 0.07351640183018682 valid loss: 0.06345007320245107 best epoch: 467 ======= epoch: 468 train loss: 0.07347545119715326 valid loss: 0.06342964712530375 best epoch: 468 ======= epoch: 469 train loss: 0.07343469911979304 valid loss: 0.06341814498106639 best epoch: 469 ======= epoch: 470 train loss: 0.07339359371474495 valid loss: 0.06338873816033204 best epoch: 470 ======= epoch: 471 train loss: 0.07335345341283599 valid loss: 0.06338162937512 best epoch: 471 ======= epoch: 472 train loss: 0.07331241464909212 valid loss: 0.06335460146268208 best epoch: 472 ======= epoch: 473 train loss: 0.07327283775134955 valid loss: 0.06334722476700942 best epoch: 473 ======= epoch: 474 train loss: 0.07323414020608236 valid loss: 0.06333668312678735 best epoch: 474 ======= epoch: 475 train loss: 0.07319827838858337 valid loss: 0.06331741530448198 best epoch: 475 ======= epoch: 476 train loss: 0.07316185376480405 valid loss: 0.06329919708271821 best epoch: 476 ======= epoch: 477 train loss: 0.07312558881110615 valid loss: 0.06328405160456896 best epoch: 477 ======= epoch: 478 train loss: 0.07309153801359144 valid loss: 0.0632594923178355 best epoch: 478 ======= epoch: 479 train loss: 0.07305541755286632 valid loss: 0.06324411525080602 best epoch: 479 ======= epoch: 480 train loss: 0.07301941286358568 valid loss: 0.06322053788850705 best epoch: 480 ======= epoch: 481 train loss: 0.0729830500635284 valid loss: 0.06321120945115884 best epoch: 481 ======= epoch: 482 train loss: 0.07294683540124952 valid loss: 0.06318792266150315 best epoch: 482 ======= epoch: 483 train loss: 0.07291345501801481 valid loss: 0.06318803938726585 best epoch: 482 ======= epoch: 484 train loss: 0.0728795603952474 valid loss: 0.06316224796076615 best epoch: 484 ======= epoch: 485 train loss: 0.07284743142210776 valid loss: 0.06315006005267303 best epoch: 485 ======= epoch: 486 train loss: 0.072814219309316 valid loss: 0.06313140504062176 best epoch: 486 ======= epoch: 487 train loss: 0.0727810768617524 valid loss: 0.06312021147459745 best epoch: 487 ======= epoch: 488 train loss: 0.07274839776441638 valid loss: 0.0630927924066782 best epoch: 488 ======= epoch: 489 train loss: 0.07271444519268877 valid loss: 0.06309458489219348 best epoch: 488 ======= epoch: 490 train loss: 0.0726806166133395 valid loss: 0.0630781843016545 best epoch: 490 ======= epoch: 491 train loss: 0.07264984725618068 valid loss: 0.06307302974164486 best epoch: 491 ======= epoch: 492 train loss: 0.07262129051449859 valid loss: 0.06305481400340796 best epoch: 492 ======= epoch: 493 train loss: 0.07259310775057033 valid loss: 0.0630432798837622 best epoch: 493 ======= epoch: 494 train loss: 0.0725624358833388 valid loss: 0.06303315112988155 best epoch: 494 ======= epoch: 495 train loss: 0.07253297979449048 valid loss: 0.06302162756522496 best epoch: 495 ======= epoch: 496 train loss: 0.07250402530908216 valid loss: 0.06301705384006102 best epoch: 496 ======= epoch: 497 train loss: 0.07247521408638101 valid loss: 0.06300089818735917 best epoch: 497 ======= epoch: 498 train loss: 0.07244608462730676 valid loss: 0.06298695349444945 best epoch: 498 ======= epoch: 499 train loss: 0.07241818411160766 valid loss: 0.06298755295574665 best epoch: 498 ======= epoch: 500 train loss: 0.07239040311563898 valid loss: 0.06296188198029995 best epoch: 500 ======= epoch: 501 train loss: 0.07236105394492179 valid loss: 0.06295529442528884 best epoch: 501 ======= epoch: 502 train loss: 0.07233053343485535 valid loss: 0.06293264372895162 best epoch: 502 ======= epoch: 503 train loss: 0.0722995258545802 valid loss: 0.06291980079064767 best epoch: 503 ======= epoch: 504 train loss: 0.07226963378029105 valid loss: 0.06290266538659732 best epoch: 504 ======= epoch: 505 train loss: 0.0722406546090856 valid loss: 0.0628874571993947 best epoch: 505 ======= epoch: 506 train loss: 0.0722110850483547 valid loss: 0.06286058357606332 best epoch: 506 ======= epoch: 507 train loss: 0.07218206507142311 valid loss: 0.0628427630290389 best epoch: 507 ======= epoch: 508 train loss: 0.07215325597581672 valid loss: 0.06283069495111704 best epoch: 508 ======= epoch: 509 train loss: 0.07212438386071611 valid loss: 0.06282440790285666 best epoch: 509 ======= epoch: 510 train loss: 0.07209605122284021 valid loss: 0.0628117589900891 best epoch: 510 ======= epoch: 511 train loss: 0.07206659589284732 valid loss: 0.0627933545038104 best epoch: 511 ======= epoch: 512 train loss: 0.07203801684541467 valid loss: 0.06278713947782914 best epoch: 512 ======= epoch: 513 train loss: 0.07201182680135523 valid loss: 0.06276444345712662 best epoch: 513 ======= epoch: 514 train loss: 0.07198526659681473 valid loss: 0.0627550274754564 best epoch: 514 ======= epoch: 515 train loss: 0.07195921528532549 valid loss: 0.06274681476255257 best epoch: 515 ======= epoch: 516 train loss: 0.07193460305117899 valid loss: 0.06273023132234812 best epoch: 516 ======= epoch: 517 train loss: 0.07190797605586273 valid loss: 0.06272513481477897 best epoch: 517 ======= epoch: 518 train loss: 0.07188184082186148 valid loss: 0.06270949790875117 best epoch: 518 ======= epoch: 519 train loss: 0.07185467379742566 valid loss: 0.06269560226549704 best epoch: 519 ======= epoch: 520 train loss: 0.07182913630005018 valid loss: 0.06268877442926168 best epoch: 520 ======= epoch: 521 train loss: 0.07180450011596635 valid loss: 0.06266648446520169 best epoch: 521 ======= epoch: 522 train loss: 0.07177922058344623 valid loss: 0.06266115574787061 best epoch: 522 ======= epoch: 523 train loss: 0.07175428984847701 valid loss: 0.06264468934386969 best epoch: 523 ======= epoch: 524 train loss: 0.07172997491319238 valid loss: 0.06263519947727521 best epoch: 524 ======= epoch: 525 train loss: 0.07170325002552551 valid loss: 0.06263086137672265 best epoch: 525 ======= epoch: 526 train loss: 0.07167746081810307 valid loss: 0.06261213465283315 best epoch: 526 ======= epoch: 527 train loss: 0.07165241086234649 valid loss: 0.06261275553454955 best epoch: 526 ======= epoch: 528 train loss: 0.07162659282991547 valid loss: 0.06259697365264098 best epoch: 528 ======= epoch: 529 train loss: 0.07160028043766449 valid loss: 0.0625857322787245 best epoch: 529 ======= epoch: 530 train loss: 0.07157410137401318 valid loss: 0.06257836427539587 best epoch: 530 ======= epoch: 531 train loss: 0.07154667970758898 valid loss: 0.06257170376678307 best epoch: 531 ======= epoch: 532 train loss: 0.07152109724228028 valid loss: 0.06255806268503268 best epoch: 532 ======= epoch: 533 train loss: 0.07149444298565756 valid loss: 0.06255506537854671 best epoch: 533 ======= epoch: 534 train loss: 0.07146887469714806 valid loss: 0.0625382090608279 best epoch: 534 ======= epoch: 535 train loss: 0.0714436345446257 valid loss: 0.06254541222006083 best epoch: 534 ======= epoch: 536 train loss: 0.07142017145905598 valid loss: 0.06253262205670278 best epoch: 536 ======= epoch: 537 train loss: 0.07139572418398327 valid loss: 0.06252564645061891 best epoch: 537 ======= epoch: 538 train loss: 0.07137217465788126 valid loss: 0.06251490643868844 best epoch: 538 ======= epoch: 539 train loss: 0.0713484402814949 valid loss: 0.06250452809035778 best epoch: 539 ======= epoch: 540 train loss: 0.07132494147223087 valid loss: 0.062493386367956795 best epoch: 540 ======= epoch: 541 train loss: 0.0713006647211718 valid loss: 0.06248677087326845 best epoch: 541 ======= epoch: 542 train loss: 0.07127604423159803 valid loss: 0.06247999674330155 best epoch: 542 ======= epoch: 543 train loss: 0.07125194033860792 valid loss: 0.06247940597434839 best epoch: 543 ======= epoch: 544 train loss: 0.07122953351081153 valid loss: 0.06246605173995098 best epoch: 544 ======= epoch: 545 train loss: 0.07120569181387071 valid loss: 0.062468743262191616 best epoch: 544 ======= epoch: 546 train loss: 0.07118298698584606 valid loss: 0.06245153211057186 best epoch: 546 ======= epoch: 547 train loss: 0.07116017673440554 valid loss: 0.062449415835241474 best epoch: 547 ======= epoch: 548 train loss: 0.07113801413158208 valid loss: 0.062436356830100216 best epoch: 548 ======= epoch: 549 train loss: 0.07111497577509762 valid loss: 0.06243137860049804 best epoch: 549 ======= epoch: 550 train loss: 0.07109188138978717 valid loss: 0.062420547008514404 best epoch: 550 ======= epoch: 551 train loss: 0.07106892803660882 valid loss: 0.06240869189302126 best epoch: 551 ======= epoch: 552 train loss: 0.07104685740476405 valid loss: 0.06240482535213232 best epoch: 552 ======= epoch: 553 train loss: 0.07102447991937767 valid loss: 0.0623802129800121 best epoch: 553 ======= epoch: 554 train loss: 0.07100494787740855 valid loss: 0.0623761306827267 best epoch: 554 ======= epoch: 555 train loss: 0.07098258469529727 valid loss: 0.06236688606441021 best epoch: 555 ======= epoch: 556 train loss: 0.07096142673658 valid loss: 0.06235388697435459 best epoch: 556 ======= epoch: 557 train loss: 0.07094054420789082 valid loss: 0.06235189208139976 best epoch: 557 ======= epoch: 558 train loss: 0.07091864106463797 valid loss: 0.0623328456034263 best epoch: 558 ======= epoch: 559 train loss: 0.0708971577524035 valid loss: 0.06233069021254778 best epoch: 559 ======= epoch: 560 train loss: 0.0708771119883031 valid loss: 0.06231547333300114 best epoch: 560 ======= epoch: 561 train loss: 0.07085572833908194 valid loss: 0.06231875717639923 best epoch: 560 ======= epoch: 562 train loss: 0.07083445808125867 valid loss: 0.06230617314577103 best epoch: 562 ======= epoch: 563 train loss: 0.07081256482612204 valid loss: 0.06230293648938338 best epoch: 563 ======= epoch: 564 train loss: 0.07079007155779335 valid loss: 0.062294453692932926 best epoch: 564 ======= epoch: 565 train loss: 0.07076754056515148 valid loss: 0.062288565250734486 best epoch: 565 ======= epoch: 566 train loss: 0.07074645207619962 valid loss: 0.06227431415269772 best epoch: 566 ======= epoch: 567 train loss: 0.07072532594341922 valid loss: 0.062272803547481693 best epoch: 567 ======= epoch: 568 train loss: 0.07070436660936218 valid loss: 0.06226297343770663 best epoch: 568 ======= epoch: 569 train loss: 0.07068266307958114 valid loss: 0.062260162395735584 best epoch: 569 ======= epoch: 570 train loss: 0.07066173017898827 valid loss: 0.06225165786842505 best epoch: 570 ======= epoch: 571 train loss: 0.0706416652135459 valid loss: 0.0622473806142807 best epoch: 571 ======= epoch: 572 train loss: 0.07062137930619496 valid loss: 0.06224296614527702 best epoch: 572 ======= epoch: 573 train loss: 0.07060124604376378 valid loss: 0.062237621595462166 best epoch: 573 ======= epoch: 574 train loss: 0.07058250362536421 valid loss: 0.06223773676902056 best epoch: 573 ======= epoch: 575 train loss: 0.07056260524018679 valid loss: 0.062228722808261715 best epoch: 575 ======= epoch: 576 train loss: 0.07054390591557747 valid loss: 0.062218341045081615 best epoch: 576 ======= epoch: 577 train loss: 0.0705236289995136 valid loss: 0.062215620962282024 best epoch: 577 ======= epoch: 578 train loss: 0.07050374349188289 valid loss: 0.062200444440046944 best epoch: 578 ======= epoch: 579 train loss: 0.07048242081554583 valid loss: 0.06219697402169307 best epoch: 579 ======= epoch: 580 train loss: 0.07046165393182525 valid loss: 0.06218559636423985 best epoch: 580 ======= epoch: 581 train loss: 0.07044108759289906 valid loss: 0.06217400139818589 best epoch: 581 ======= epoch: 582 train loss: 0.07042157246420781 valid loss: 0.06216739987333616 best epoch: 582 ======= epoch: 583 train loss: 0.07040165140534993 valid loss: 0.062157745783527694 best epoch: 583 ======= epoch: 584 train loss: 0.07038286436395144 valid loss: 0.06215134263038635 best epoch: 584 ======= epoch: 585 train loss: 0.07036409811059266 valid loss: 0.06215204546848933 best epoch: 584 ======= epoch: 586 train loss: 0.07034622993964104 valid loss: 0.062135426017145313 best epoch: 586 ======= epoch: 587 train loss: 0.07032817804519041 valid loss: 0.06213804893195629 best epoch: 586 ======= epoch: 588 train loss: 0.07031105059762428 valid loss: 0.062127585522830486 best epoch: 588 ======= epoch: 589 train loss: 0.07029289401561757 valid loss: 0.06212082256873449 best epoch: 589 ======= epoch: 590 train loss: 0.0702744545645 valid loss: 0.06211299014588197 best epoch: 590 ======= epoch: 591 train loss: 0.07025644138317418 valid loss: 0.06210816434274117 best epoch: 591 ======= epoch: 592 train loss: 0.07023769415869022 valid loss: 0.062096090987324715 best epoch: 592 ======= epoch: 593 train loss: 0.07021920893479276 valid loss: 0.06208922372510036 best epoch: 593 ======= epoch: 594 train loss: 0.07020130952429256 valid loss: 0.06208482043196758 best epoch: 594 ======= epoch: 595 train loss: 0.07018375416274791 valid loss: 0.062074777049322925 best epoch: 595 ======= epoch: 596 train loss: 0.0701655343082952 valid loss: 0.062068469201525055 best epoch: 596 ======= epoch: 597 train loss: 0.07014658978139912 valid loss: 0.062067072528103985 best epoch: 597 ======= epoch: 598 train loss: 0.07012801274749232 valid loss: 0.06205044935146967 best epoch: 598 ======= epoch: 599 train loss: 0.07010820913875912 valid loss: 0.06204720679670572 best epoch: 599 ======= epoch: 600 train loss: 0.07008854403263992 valid loss: 0.06203349338223537 best epoch: 600 ======= epoch: 601 train loss: 0.07007199606317797 valid loss: 0.062030150865515075 best epoch: 601 ======= epoch: 602 train loss: 0.07005354555116759 valid loss: 0.062019851679603256 best epoch: 602 ======= epoch: 603 train loss: 0.07003718423714608 valid loss: 0.06201527267694473 best epoch: 603 ======= epoch: 604 train loss: 0.07002024222625258 valid loss: 0.062002637423574924 best epoch: 604 ======= epoch: 605 train loss: 0.07000215644775717 valid loss: 0.06199935699502627 best epoch: 605 ======= epoch: 606 train loss: 0.06998463657995065 valid loss: 0.06198939401656389 best epoch: 606 ======= epoch: 607 train loss: 0.06996621751444942 valid loss: 0.06198345279941956 best epoch: 607 ======= epoch: 608 train loss: 0.06994923659496838 valid loss: 0.06197195469091336 best epoch: 608 ======= epoch: 609 train loss: 0.06993131145237037 valid loss: 0.06195833099385103 best epoch: 609 ======= epoch: 610 train loss: 0.06991442343519058 valid loss: 0.061951368736724056 best epoch: 610 ======= epoch: 611 train loss: 0.06989731162464545 valid loss: 0.06193968250105778 best epoch: 611 ======= epoch: 612 train loss: 0.06988215625837997 valid loss: 0.06192989957829317 best epoch: 612 ======= epoch: 613 train loss: 0.06986530955282995 valid loss: 0.06192744957904021 best epoch: 613 ======= epoch: 614 train loss: 0.06985055775968013 valid loss: 0.06191334035247564 best epoch: 614 ======= epoch: 615 train loss: 0.06983453936782884 valid loss: 0.06190862165143093 best epoch: 615 ======= epoch: 616 train loss: 0.06981903825093198 valid loss: 0.06190224861105283 best epoch: 616 ======= epoch: 617 train loss: 0.06980292496765837 valid loss: 0.06188997346907854 best epoch: 617 ======= epoch: 618 train loss: 0.06978617707428372 valid loss: 0.06188406515866518 best epoch: 618 ======= epoch: 619 train loss: 0.06976950497446972 valid loss: 0.06187279615551233 best epoch: 619 ======= epoch: 620 train loss: 0.06975244012097885 valid loss: 0.061866010228792824 best epoch: 620 ======= epoch: 621 train loss: 0.06973524341060791 valid loss: 0.061862548192342125 best epoch: 621 ======= epoch: 622 train loss: 0.06971840264942543 valid loss: 0.06185775498549143 best epoch: 622 ======= epoch: 623 train loss: 0.06970173662045488 valid loss: 0.06184907195468744 best epoch: 623 ======= epoch: 624 train loss: 0.06968617880785907 valid loss: 0.06184385251253843 best epoch: 624 ======= epoch: 625 train loss: 0.06967007986235398 valid loss: 0.06183723360300064 best epoch: 625 ======= epoch: 626 train loss: 0.06965378347646307 valid loss: 0.061820805072784424 best epoch: 626 ======= epoch: 627 train loss: 0.06963694568171545 valid loss: 0.061815397503475346 best epoch: 627 ======= epoch: 628 train loss: 0.06962037004852369 valid loss: 0.06180321890860796 best epoch: 628 ======= epoch: 629 train loss: 0.06960294732571017 valid loss: 0.06179706410815319 best epoch: 629 ======= epoch: 630 train loss: 0.06958768029271821 valid loss: 0.061785840118924774 best epoch: 630 ======= epoch: 631 train loss: 0.06957046778435692 valid loss: 0.061781407333910465 best epoch: 631 ======= epoch: 632 train loss: 0.06955606325954934 valid loss: 0.061769651559491955 best epoch: 632 ======= epoch: 633 train loss: 0.06953974319966855 valid loss: 0.061764278759559 best epoch: 633 ======= epoch: 634 train loss: 0.06952463833178267 valid loss: 0.06175424996763468 best epoch: 634 ======= epoch: 635 train loss: 0.06950923594657654 valid loss: 0.06174962564061085 best epoch: 635 ======= epoch: 636 train loss: 0.06949350560934824 valid loss: 0.06173278267184893 best epoch: 636 ======= epoch: 637 train loss: 0.06947790312776227 valid loss: 0.061735580675303936 best epoch: 636 ======= epoch: 638 train loss: 0.06946283390308604 valid loss: 0.06171789579093456 best epoch: 638 ======= epoch: 639 train loss: 0.06944772281488519 valid loss: 0.061717274909218155 best epoch: 639 ======= epoch: 640 train loss: 0.06943447878699244 valid loss: 0.06170195868859688 best epoch: 640 ======= epoch: 641 train loss: 0.06941980075229097 valid loss: 0.06169800336162249 best epoch: 641 ======= epoch: 642 train loss: 0.06940668736251047 valid loss: 0.06168688957889875 best epoch: 642 ======= epoch: 643 train loss: 0.06939296680790039 valid loss: 0.06167894663910071 best epoch: 643 ======= epoch: 644 train loss: 0.06937933495889108 valid loss: 0.06167067618419727 best epoch: 644 ======= epoch: 645 train loss: 0.06936534290650377 valid loss: 0.06166050986697277 best epoch: 645 ======= epoch: 646 train loss: 0.06935392110895963 valid loss: 0.061658586064974465 best epoch: 646 ======= epoch: 647 train loss: 0.06934013633908313 valid loss: 0.061652631809314094 best epoch: 647 ======= epoch: 648 train loss: 0.0693278314522755 valid loss: 0.06164343965550264 best epoch: 648 ======= epoch: 649 train loss: 0.06931482549802756 valid loss: 0.06163814074049393 best epoch: 649 ======= epoch: 650 train loss: 0.0693018544856229 valid loss: 0.06162719366451105 best epoch: 650 ======= epoch: 651 train loss: 0.06928829676299184 valid loss: 0.06161960338552793 best epoch: 651 ======= epoch: 652 train loss: 0.0692744363260306 valid loss: 0.061610586320360504 best epoch: 652 ======= epoch: 653 train loss: 0.06925982169025106 valid loss: 0.06160397765537103 best epoch: 653 ======= epoch: 654 train loss: 0.06924467717791781 valid loss: 0.06159514902780453 best epoch: 654 ======= epoch: 655 train loss: 0.06922880577406397 valid loss: 0.06159108659873406 best epoch: 655 ======= epoch: 656 train loss: 0.06921399790609692 valid loss: 0.06158078151444594 best epoch: 656 ======= epoch: 657 train loss: 0.06919811912064935 valid loss: 0.06157939384380976 best epoch: 657 ======= epoch: 658 train loss: 0.06918343649832187 valid loss: 0.06157313380390406 best epoch: 658 ======= epoch: 659 train loss: 0.06916844985091392 valid loss: 0.06156709076215824 best epoch: 659 ======= epoch: 660 train loss: 0.06915495728636965 valid loss: 0.06156145284573237 best epoch: 660 ======= epoch: 661 train loss: 0.06914131965396213 valid loss: 0.06155678412566582 best epoch: 661 ======= epoch: 662 train loss: 0.06912984571385163 valid loss: 0.061551704381903015 best epoch: 662 ======= epoch: 663 train loss: 0.06911649755029767 valid loss: 0.06154540988306204 best epoch: 663 ======= epoch: 664 train loss: 0.06910439117317214 valid loss: 0.06153919827193022 best epoch: 664 ======= epoch: 665 train loss: 0.06909124048449743 valid loss: 0.06153330517311891 best epoch: 665 ======= epoch: 666 train loss: 0.06907935587712276 valid loss: 0.06152852779875199 best epoch: 666 ======= epoch: 667 train loss: 0.06906588483647799 valid loss: 0.06152652483433485 best epoch: 667 ======= epoch: 668 train loss: 0.06905365793931263 valid loss: 0.061516083776950836 best epoch: 668 ======= epoch: 669 train loss: 0.06903983019062399 valid loss: 0.06151784242441257 best epoch: 668 ======= epoch: 670 train loss: 0.06902700209599218 valid loss: 0.06150594539940357 best epoch: 670 ======= epoch: 671 train loss: 0.06901323601005623 valid loss: 0.06151050142943859 best epoch: 670 ======= epoch: 672 train loss: 0.0690002687919287 valid loss: 0.061497767455875874 best epoch: 672 ======= epoch: 673 train loss: 0.06898659452749992 valid loss: 0.06149824863920609 best epoch: 672 ======= epoch: 674 train loss: 0.06897295820584268 valid loss: 0.06148730808248123 best epoch: 674 ======= epoch: 675 train loss: 0.06895892328962132 valid loss: 0.06148188405980667 best epoch: 675 ======= epoch: 676 train loss: 0.06894471304330194 valid loss: 0.06147832361360391 best epoch: 676 ======= epoch: 677 train loss: 0.06893090461093335 valid loss: 0.061470553278923035 best epoch: 677 ======= epoch: 678 train loss: 0.06891669055883899 valid loss: 0.06146576379736265 best epoch: 678 ======= epoch: 679 train loss: 0.06890268897476756 valid loss: 0.061463783184687294 best epoch: 679 ======= epoch: 680 train loss: 0.06888985992581756 valid loss: 0.061458749075730644 best epoch: 680 ======= epoch: 681 train loss: 0.06887674606462688 valid loss: 0.06145572600265344 best epoch: 681 ======= epoch: 682 train loss: 0.06886386862139643 valid loss: 0.06145051339020332 best epoch: 682 ======= epoch: 683 train loss: 0.06885040659503436 valid loss: 0.061448522843420506 best epoch: 683 ======= epoch: 684 train loss: 0.0688371538977932 valid loss: 0.06144239070514838 best epoch: 684 ======= epoch: 685 train loss: 0.06882317686154518 valid loss: 0.06144257759054502 best epoch: 684 ======= epoch: 686 train loss: 0.06880958010385066 valid loss: 0.06143186272432407 best epoch: 686 ======= epoch: 687 train loss: 0.06879628297907335 valid loss: 0.06143845710903406 best epoch: 686 ======= epoch: 688 train loss: 0.0687833083908499 valid loss: 0.06142556915680567 best epoch: 688 ======= epoch: 689 train loss: 0.06877027309419197 valid loss: 0.061435735784471035 best epoch: 688 ======= epoch: 690 train loss: 0.06875837255085324 valid loss: 0.061426121120651565 best epoch: 688 ======= epoch: 691 train loss: 0.06874581472373303 valid loss: 0.06142665290584167 best epoch: 688 ======= epoch: 692 train loss: 0.06873348623192237 valid loss: 0.061424980871379375 best epoch: 692 ======= epoch: 693 train loss: 0.06872130120977943 valid loss: 0.06141820022215446 best epoch: 693 ======= epoch: 694 train loss: 0.06870789939744605 valid loss: 0.06141794100403786 best epoch: 694 ======= epoch: 695 train loss: 0.06869489369614992 valid loss: 0.061413789788881935 best epoch: 695 ======= epoch: 696 train loss: 0.0686812030358447 valid loss: 0.06140665772060553 best epoch: 696 ======= epoch: 697 train loss: 0.0686680964987587 valid loss: 0.0614101200674971 best epoch: 696 ======= epoch: 698 train loss: 0.06865557566009185 valid loss: 0.061401776038110256 best epoch: 698 ======= epoch: 699 train loss: 0.06864246199436394 valid loss: 0.0614039096981287 best epoch: 698 ======= epoch: 700 train loss: 0.06863043039899182 valid loss: 0.06139920341471831 best epoch: 700 ======= epoch: 701 train loss: 0.06861752866088976 valid loss: 0.06139873992651701 best epoch: 701 ======= epoch: 702 train loss: 0.068605549449538 valid loss: 0.06139838354041179 best epoch: 702 ======= epoch: 703 train loss: 0.06859290712501531 valid loss: 0.06139459585150083 best epoch: 703 ======= epoch: 704 train loss: 0.06858055383242943 valid loss: 0.0613938570022583 best epoch: 704 ======= epoch: 705 train loss: 0.06856768091933595 valid loss: 0.06139416620135307 best epoch: 704 ======= epoch: 706 train loss: 0.06855519866731799 valid loss: 0.06138780495772759 best epoch: 706 ======= epoch: 707 train loss: 0.06854320867102087 valid loss: 0.06138762334982554 best epoch: 707 ======= epoch: 708 train loss: 0.06853165692523305 valid loss: 0.06138119846582413 best epoch: 708 ======= epoch: 709 train loss: 0.06851915833482772 valid loss: 0.061381357721984386 best epoch: 708 ======= epoch: 710 train loss: 0.0685081532638934 valid loss: 0.06137365320076545 best epoch: 710 ======= epoch: 711 train loss: 0.06849578951979861 valid loss: 0.061374142145117126 best epoch: 710 ======= epoch: 712 train loss: 0.06848388821760445 valid loss: 0.06136218272149563 best epoch: 712 ======= epoch: 713 train loss: 0.0684713674939157 valid loss: 0.0613614267980059 best epoch: 713 ======= epoch: 714 train loss: 0.06845987926202791 valid loss: 0.06135307780156533 best epoch: 714 ======= epoch: 715 train loss: 0.06844815805002495 valid loss: 0.061354175520439945 best epoch: 714 ======= epoch: 716 train loss: 0.06843643148004273 valid loss: 0.06134613541265329 best epoch: 716 ======= epoch: 717 train loss: 0.06842443824136331 valid loss: 0.06134545989334583 best epoch: 717 ======= epoch: 718 train loss: 0.06841293906356082 valid loss: 0.061338046876092754 best epoch: 718 ======= epoch: 719 train loss: 0.06840048800510021 valid loss: 0.06133645183096329 best epoch: 719 ======= epoch: 720 train loss: 0.06838822094608604 valid loss: 0.061329045643409096 best epoch: 720 ======= epoch: 721 train loss: 0.06837482077793942 valid loss: 0.06132915988564491 best epoch: 720 ======= epoch: 722 train loss: 0.06836207687026924 valid loss: 0.06132238575567802 best epoch: 722 ======= epoch: 723 train loss: 0.06834804563334694 valid loss: 0.06131606021275123 best epoch: 723 ======= epoch: 724 train loss: 0.06833598348829481 valid loss: 0.06131480478992065 best epoch: 724 ======= epoch: 725 train loss: 0.0683228581752858 valid loss: 0.06131152870754401 best epoch: 725 ======= epoch: 726 train loss: 0.06831209568513764 valid loss: 0.06131234671920538 best epoch: 725 ======= epoch: 727 train loss: 0.0683003106717894 valid loss: 0.06130741505573193 best epoch: 727 ======= epoch: 728 train loss: 0.06828989834743149 valid loss: 0.06130347897609075 best epoch: 728 ======= epoch: 729 train loss: 0.06827804673877028 valid loss: 0.06130277210225662 best epoch: 729 ======= epoch: 730 train loss: 0.06826756075944429 valid loss: 0.06129517064740261 best epoch: 730 ======= epoch: 731 train loss: 0.06825576294903402 valid loss: 0.06129607775559028 best epoch: 730 ======= epoch: 732 train loss: 0.0682446863502264 valid loss: 0.06128539082904657 best epoch: 732 ======= epoch: 733 train loss: 0.06823309381021025 valid loss: 0.06128743942826986 best epoch: 732 ======= epoch: 734 train loss: 0.06822149826926582 valid loss: 0.06127472190807263 best epoch: 734 ======= epoch: 735 train loss: 0.06820916927155153 valid loss: 0.061269551515579224 best epoch: 735 ======= epoch: 736 train loss: 0.06819735552517721 valid loss: 0.06126258118698994 best epoch: 736 ======= epoch: 737 train loss: 0.06818532782756252 valid loss: 0.06125695879260699 best epoch: 737 ======= epoch: 738 train loss: 0.06817467500352198 valid loss: 0.061253517866134644 best epoch: 738 ======= epoch: 739 train loss: 0.06816326209378464 valid loss: 0.061246924102306366 best epoch: 739 ======= epoch: 740 train loss: 0.06815206410110365 valid loss: 0.06124300044029951 best epoch: 740 ======= epoch: 741 train loss: 0.06814035973339169 valid loss: 0.061234596806267895 best epoch: 741 ======= epoch: 742 train loss: 0.06812911129982015 valid loss: 0.061228059542675815 best epoch: 742 ======= epoch: 743 train loss: 0.06811694595042939 valid loss: 0.06122249675293764 best epoch: 743 ======= epoch: 744 train loss: 0.06810534167114968 valid loss: 0.06121141898135344 best epoch: 744 ======= epoch: 745 train loss: 0.06809340804861283 valid loss: 0.0612075375393033 best epoch: 745 ======= epoch: 746 train loss: 0.06808304478540833 valid loss: 0.06119668080161015 best epoch: 746 ======= epoch: 747 train loss: 0.06807102912600026 valid loss: 0.06119329109787941 best epoch: 747 ======= epoch: 748 train loss: 0.06806162207812806 valid loss: 0.06118176691234112 best epoch: 748 ======= epoch: 749 train loss: 0.06804921900002676 valid loss: 0.061179593515892826 best epoch: 749 ======= epoch: 750 train loss: 0.06803912647574405 valid loss: 0.06116419875373443 best epoch: 750 ======= epoch: 751 train loss: 0.06802742928266525 valid loss: 0.06116085002819697 best epoch: 751 ======= epoch: 752 train loss: 0.06801658860741207 valid loss: 0.06115115371843179 best epoch: 752 ======= epoch: 753 train loss: 0.0680047920272674 valid loss: 0.06114794903745254 best epoch: 753 ======= epoch: 754 train loss: 0.06799327293525875 valid loss: 0.06113750208169222 best epoch: 754 ======= epoch: 755 train loss: 0.06798055493215352 valid loss: 0.061132844227055706 best epoch: 755 ======= epoch: 756 train loss: 0.0679679417890715 valid loss: 0.06112584720055262 best epoch: 756 ======= epoch: 757 train loss: 0.06795462282995383 valid loss: 0.06111620273441076 best epoch: 757 ======= epoch: 758 train loss: 0.06794349252487775 valid loss: 0.06111921525249878 best epoch: 757 ======= epoch: 759 train loss: 0.06793182974474297 valid loss: 0.06111337927480539 best epoch: 759 ======= epoch: 760 train loss: 0.06792159884432215 valid loss: 0.06111176467190186 best epoch: 760 ======= epoch: 761 train loss: 0.06791162910514775 valid loss: 0.06110721050451199 best epoch: 761 ======= epoch: 762 train loss: 0.06790130940714369 valid loss: 0.06110087192306916 best epoch: 762 ======= epoch: 763 train loss: 0.06789077872433412 valid loss: 0.0611013596256574 best epoch: 762 ======= epoch: 764 train loss: 0.0678799347722052 valid loss: 0.06109673188378414 best epoch: 764 ======= epoch: 765 train loss: 0.0678688030069073 valid loss: 0.06109987044086059 best epoch: 764 ======= epoch: 766 train loss: 0.0678593313758388 valid loss: 0.06109412697454294 best epoch: 766 ======= epoch: 767 train loss: 0.06784801465677626 valid loss: 0.061093767173588276 best epoch: 767 ======= epoch: 768 train loss: 0.06783820075891268 valid loss: 0.06108520179986954 best epoch: 768 ======= epoch: 769 train loss: 0.06782637099608962 valid loss: 0.06108578729132811 best epoch: 768 ======= epoch: 770 train loss: 0.06781566461525214 valid loss: 0.0610736341526111 best epoch: 770 ======= epoch: 771 train loss: 0.06780385994064955 valid loss: 0.06107343609134356 best epoch: 771 ======= epoch: 772 train loss: 0.06779270291466404 valid loss: 0.06106364199270805 best epoch: 772 ======= epoch: 773 train loss: 0.06778133631810362 valid loss: 0.06105933741976818 best epoch: 773 ======= epoch: 774 train loss: 0.06777053187244836 valid loss: 0.06104998787244161 best epoch: 774 ======= epoch: 775 train loss: 0.06775983015972155 valid loss: 0.061047473922371864 best epoch: 775 ======= epoch: 776 train loss: 0.06774921479554456 valid loss: 0.06103701423853636 best epoch: 776 ======= epoch: 777 train loss: 0.06773871085664004 valid loss: 0.061042335194846 best epoch: 776 ======= epoch: 778 train loss: 0.06772777960164311 valid loss: 0.06102990545332432 best epoch: 778 ======= epoch: 779 train loss: 0.06771695201089721 valid loss: 0.061031672482689224 best epoch: 778 ======= epoch: 780 train loss: 0.06770586750159661 valid loss: 0.0610210740317901 best epoch: 780 ======= epoch: 781 train loss: 0.06769442473204783 valid loss: 0.061020043678581715 best epoch: 781 ======= epoch: 782 train loss: 0.06768304442434951 valid loss: 0.06101654625187317 best epoch: 782 ======= epoch: 783 train loss: 0.06767115277456648 valid loss: 0.06100825313478708 best epoch: 783 ======= epoch: 784 train loss: 0.06765925428933567 valid loss: 0.06100423292567333 best epoch: 784 ======= epoch: 785 train loss: 0.06764838109827703 valid loss: 0.06099190811316172 best epoch: 785 ======= epoch: 786 train loss: 0.06763852128391097 valid loss: 0.06099305395036936 best epoch: 785 ======= epoch: 787 train loss: 0.06762689041019773 valid loss: 0.06098680756986141 best epoch: 787 ======= epoch: 788 train loss: 0.06761765116135832 valid loss: 0.06098525474468867 best epoch: 788 ======= epoch: 789 train loss: 0.06760824779853408 valid loss: 0.06098135902235905 best epoch: 789 ======= epoch: 790 train loss: 0.06759829497263756 valid loss: 0.060977305906514324 best epoch: 790 ======= epoch: 791 train loss: 0.06758775449944315 valid loss: 0.060977013781666756 best epoch: 791 ======= epoch: 792 train loss: 0.06757788464174043 valid loss: 0.06096711258093516 best epoch: 792 ======= epoch: 793 train loss: 0.0675669489428401 valid loss: 0.06096776481717825 best epoch: 792 ======= epoch: 794 train loss: 0.06755715962154446 valid loss: 0.060958465561270714 best epoch: 794 ======= epoch: 795 train loss: 0.06754565444763427 valid loss: 0.06095689721405506 best epoch: 795 ======= epoch: 796 train loss: 0.06753589680280399 valid loss: 0.06094662441561619 best epoch: 796 ======= epoch: 797 train loss: 0.06752432940274845 valid loss: 0.06094626554598411 best epoch: 797 ======= epoch: 798 train loss: 0.06751371944254195 valid loss: 0.06093395873904228 best epoch: 798 ======= epoch: 799 train loss: 0.06750201725941381 valid loss: 0.060933232928315796 best epoch: 799 ======= epoch: 800 train loss: 0.06749060290670138 valid loss: 0.06092233117669821 best epoch: 800 ======= epoch: 801 train loss: 0.06747829998601917 valid loss: 0.06092187793304523 best epoch: 801 ======= epoch: 802 train loss: 0.0674661756148217 valid loss: 0.06091421594222387 best epoch: 802 ======= epoch: 803 train loss: 0.0674531362939304 valid loss: 0.06090661883354187 best epoch: 803 ======= epoch: 804 train loss: 0.06744131580983967 valid loss: 0.06090044075002273 best epoch: 804 ======= epoch: 805 train loss: 0.06742851993207026 valid loss: 0.06089316246410211 best epoch: 805 ======= epoch: 806 train loss: 0.06741991627639458 valid loss: 0.0608930087958773 best epoch: 806 ======= epoch: 807 train loss: 0.06740726340263162 valid loss: 0.06088587921112776 best epoch: 807 ======= epoch: 808 train loss: 0.06739788310441339 valid loss: 0.0608826307579875 best epoch: 808 ======= epoch: 809 train loss: 0.06738828135263404 valid loss: 0.06087956173966328 best epoch: 809 ======= epoch: 810 train loss: 0.06737798665086796 valid loss: 0.06087142384300629 best epoch: 810 ======= epoch: 811 train loss: 0.0673677876913621 valid loss: 0.06086706711600224 best epoch: 811 ======= epoch: 812 train loss: 0.0673576093416799 valid loss: 0.060860316889981426 best epoch: 812 ======= epoch: 813 train loss: 0.06734670142723638 valid loss: 0.060859592631459236 best epoch: 813 ======= epoch: 814 train loss: 0.06733601773157716 valid loss: 0.060853069027264915 best epoch: 814 ======= epoch: 815 train loss: 0.06732486946692254 valid loss: 0.06084941700100899 best epoch: 815 ======= epoch: 816 train loss: 0.06731393152020043 valid loss: 0.06084388184050719 best epoch: 816 ======= epoch: 817 train loss: 0.06730276271470903 valid loss: 0.06084210549791654 best epoch: 817 ======= epoch: 818 train loss: 0.06729156235347927 valid loss: 0.06084000753859679 best epoch: 818 ======= epoch: 819 train loss: 0.067281337155972 valid loss: 0.06083394245555004 best epoch: 819 ======= epoch: 820 train loss: 0.06727084897481549 valid loss: 0.06082983811696371 best epoch: 820 ======= epoch: 821 train loss: 0.06725994760262562 valid loss: 0.060827539302408695 best epoch: 821 ======= epoch: 822 train loss: 0.0672498256324526 valid loss: 0.06082284605751435 best epoch: 822 ======= epoch: 823 train loss: 0.06723800081368765 valid loss: 0.060816879384219646 best epoch: 823 ======= epoch: 824 train loss: 0.06722711757354347 valid loss: 0.06081270053982735 best epoch: 824 ======= epoch: 825 train loss: 0.0672150559861351 valid loss: 0.06080672641595205 best epoch: 825 ======= epoch: 826 train loss: 0.0672045191347507 valid loss: 0.06079777143895626 best epoch: 826 ======= epoch: 827 train loss: 0.06719319261380552 valid loss: 0.060799592485030494 best epoch: 826 ======= epoch: 828 train loss: 0.067183384206146 valid loss: 0.06078966148197651 best epoch: 828 ======= epoch: 829 train loss: 0.06717309400002346 valid loss: 0.06079566447685162 best epoch: 828 ======= epoch: 830 train loss: 0.06716397006271614 valid loss: 0.06078535411506891 best epoch: 830 ======= epoch: 831 train loss: 0.06715436231482912 valid loss: 0.06078946186850468 best epoch: 830 ======= epoch: 832 train loss: 0.06714591456370221 valid loss: 0.06078297489633163 best epoch: 832 ======= epoch: 833 train loss: 0.06713553501959936 valid loss: 0.0607829000800848 best epoch: 833 ======= epoch: 834 train loss: 0.06712597077942373 valid loss: 0.060776338602105774 best epoch: 834 ======= epoch: 835 train loss: 0.06711605237796903 valid loss: 0.060772642803688846 best epoch: 835 ======= epoch: 836 train loss: 0.0671053169709113 valid loss: 0.06076945085078478 best epoch: 836 ======= epoch: 837 train loss: 0.06709523378859883 valid loss: 0.06076386943459511 best epoch: 837 ======= epoch: 838 train loss: 0.06708487669971806 valid loss: 0.06075884153445562 best epoch: 838 ======= epoch: 839 train loss: 0.06707412670768889 valid loss: 0.06075631454586983 best epoch: 839 ======= epoch: 840 train loss: 0.0670640164365371 valid loss: 0.06074722918371359 best epoch: 840 ======= epoch: 841 train loss: 0.06705355905801241 valid loss: 0.060745641899605594 best epoch: 841 ======= epoch: 842 train loss: 0.06704301234150742 valid loss: 0.060740395759542785 best epoch: 842 ======= epoch: 843 train loss: 0.06703277212211195 valid loss: 0.06073817145079374 best epoch: 843 ======= epoch: 844 train loss: 0.06702242956071357 valid loss: 0.06073049704233805 best epoch: 844 ======= epoch: 845 train loss: 0.0670126132885523 valid loss: 0.06073056751241287 best epoch: 844 ======= epoch: 846 train loss: 0.06700330819175751 valid loss: 0.0607194434851408 best epoch: 846 ======= epoch: 847 train loss: 0.06699317568384203 valid loss: 0.0607206883529822 best epoch: 846 ======= epoch: 848 train loss: 0.06698415779658122 valid loss: 0.06071337374548117 best epoch: 848 ======= epoch: 849 train loss: 0.0669741998081682 valid loss: 0.060711560460428395 best epoch: 849 ======= epoch: 850 train loss: 0.06696461511338934 valid loss: 0.060704740385214485 best epoch: 850 ======= epoch: 851 train loss: 0.06695449770784673 valid loss: 0.060702765360474586 best epoch: 851 ======= epoch: 852 train loss: 0.06694473764272751 valid loss: 0.06070149162163337 best epoch: 852 ======= epoch: 853 train loss: 0.06693384638861005 valid loss: 0.060697843631108604 best epoch: 853 ======= epoch: 854 train loss: 0.06692298701791852 valid loss: 0.06069543616225322 best epoch: 854 ======= epoch: 855 train loss: 0.06691170396444238 valid loss: 0.06069083698093891 best epoch: 855 ======= epoch: 856 train loss: 0.06690025188166786 valid loss: 0.06068210955709219 best epoch: 856 ======= epoch: 857 train loss: 0.0668890976841435 valid loss: 0.060679053887724876 best epoch: 857 ======= epoch: 858 train loss: 0.06687943444207863 valid loss: 0.06067252221206824 best epoch: 858 ======= epoch: 859 train loss: 0.06686792557162267 valid loss: 0.060669501312077045 best epoch: 859 ======= epoch: 860 train loss: 0.06685808438370809 valid loss: 0.0606657670189937 best epoch: 860 ======= epoch: 861 train loss: 0.06684759893328135 valid loss: 0.06066548855354389 best epoch: 861 ======= epoch: 862 train loss: 0.06683833590697175 valid loss: 0.06065381380418936 best epoch: 862 ======= epoch: 863 train loss: 0.0668291771887905 valid loss: 0.06066053050259749 best epoch: 862 ======= epoch: 864 train loss: 0.06681982587655018 valid loss: 0.0606491199384133 best epoch: 864 ======= epoch: 865 train loss: 0.06681125413304494 valid loss: 0.06065216505279144 best epoch: 864 ======= epoch: 866 train loss: 0.06680223492353603 valid loss: 0.06064726536472639 best epoch: 866 ======= epoch: 867 train loss: 0.06679319755898581 valid loss: 0.060645826160907745 best epoch: 867 ======= epoch: 868 train loss: 0.06678430004804223 valid loss: 0.060643687223394714 best epoch: 868 ======= epoch: 869 train loss: 0.06677463248280095 valid loss: 0.06064193012813727 best epoch: 869 ======= epoch: 870 train loss: 0.06676546368678962 valid loss: 0.06063796548793713 best epoch: 870 ======= epoch: 871 train loss: 0.06675509569223648 valid loss: 0.06063773576170206 best epoch: 871 ======= epoch: 872 train loss: 0.06674515366853204 valid loss: 0.06063296056042115 best epoch: 872 ======= epoch: 873 train loss: 0.06673392588502647 valid loss: 0.060630496901770435 best epoch: 873 ======= epoch: 874 train loss: 0.06672355038815259 valid loss: 0.06062057117621104 best epoch: 874 ======= epoch: 875 train loss: 0.06671268250908197 valid loss: 0.06061892087260882 best epoch: 875 ======= epoch: 876 train loss: 0.06670369148070429 valid loss: 0.06061092484742403 best epoch: 876 ======= epoch: 877 train loss: 0.06669194048937456 valid loss: 0.06061081557224194 best epoch: 877 ======= epoch: 878 train loss: 0.06668274530014139 valid loss: 0.06060178577899933 best epoch: 878 ======= epoch: 879 train loss: 0.06667199119189639 valid loss: 0.06060591650505861 best epoch: 878 ======= epoch: 880 train loss: 0.0666629466354663 valid loss: 0.06059501226991415 best epoch: 880 ======= epoch: 881 train loss: 0.06665253861703807 valid loss: 0.060595667300124965 best epoch: 880 ======= epoch: 882 train loss: 0.06664289400142467 valid loss: 0.06058648135513067 best epoch: 882 ======= epoch: 883 train loss: 0.06663237394834007 valid loss: 0.06058639753609896 best epoch: 883 ======= epoch: 884 train loss: 0.06662309382478764 valid loss: 0.06058199858913819 best epoch: 884 ======= epoch: 885 train loss: 0.06661244565161106 valid loss: 0.060577294789254665 best epoch: 885 ======= epoch: 886 train loss: 0.06660304034565702 valid loss: 0.060572441356877484 best epoch: 886 ======= epoch: 887 train loss: 0.06659306778951928 valid loss: 0.060567158895234265 best epoch: 887 ======= epoch: 888 train loss: 0.06658421286646231 valid loss: 0.060563089015583195 best epoch: 888 ======= epoch: 889 train loss: 0.06657418122308122 valid loss: 0.060557421296834946 best epoch: 889 ======= epoch: 890 train loss: 0.06656524047462477 valid loss: 0.06054847749571005 best epoch: 890 ======= epoch: 891 train loss: 0.06655511778008974 valid loss: 0.060545079720517 best epoch: 891 ======= epoch: 892 train loss: 0.06654509338607759 valid loss: 0.06053426116704941 best epoch: 892 ======= epoch: 893 train loss: 0.06653484598055114 valid loss: 0.06053066315750281 best epoch: 893 ======= epoch: 894 train loss: 0.06652492963720436 valid loss: 0.06052063498646021 best epoch: 894 ======= epoch: 895 train loss: 0.06651482629923174 valid loss: 0.060521069914102554 best epoch: 894 ======= epoch: 896 train loss: 0.06650535238009912 valid loss: 0.06051045500983795 best epoch: 896 ======= epoch: 897 train loss: 0.06649474532883844 valid loss: 0.06050808106859525 best epoch: 897 ======= epoch: 898 train loss: 0.06648551244403661 valid loss: 0.06050122405091921 best epoch: 898 ======= epoch: 899 train loss: 0.06647430818504942 valid loss: 0.06049824288735787 best epoch: 899 ======= epoch: 900 train loss: 0.06646465106556813 valid loss: 0.060492029413580894 best epoch: 900 ======= epoch: 901 train loss: 0.06645359942275617 valid loss: 0.06048766989260912 best epoch: 901 ======= epoch: 902 train loss: 0.06644360253725339 valid loss: 0.060480362425247826 best epoch: 902 ======= epoch: 903 train loss: 0.0664329562899599 valid loss: 0.060477055298785366 best epoch: 903 ======= epoch: 904 train loss: 0.06642309805654266 valid loss: 0.060468061516682305 best epoch: 904 ======= epoch: 905 train loss: 0.06641210431669965 valid loss: 0.06046760051200787 best epoch: 905 ======= epoch: 906 train loss: 0.06640221748248111 valid loss: 0.06045768782496452 best epoch: 906 ======= epoch: 907 train loss: 0.06639109660560887 valid loss: 0.060454148488740124 best epoch: 907 ======= epoch: 908 train loss: 0.06638169676111436 valid loss: 0.060448771653076015 best epoch: 908 ======= epoch: 909 train loss: 0.06637226856302147 valid loss: 0.060446636440853276 best epoch: 909 ======= epoch: 910 train loss: 0.06636338543583765 valid loss: 0.06044247684379419 best epoch: 910 ======= epoch: 911 train loss: 0.06635429640588017 valid loss: 0.060440401236216225 best epoch: 911 ======= epoch: 912 train loss: 0.06634585896253954 valid loss: 0.06043671661367019 best epoch: 912 ======= epoch: 913 train loss: 0.06633660669243079 valid loss: 0.060434541354576744 best epoch: 913 ======= epoch: 914 train loss: 0.06632736033794505 valid loss: 0.0604290459305048 best epoch: 914 ======= epoch: 915 train loss: 0.06631767057618242 valid loss: 0.06042580275485913 best epoch: 915 ======= epoch: 916 train loss: 0.06630869398330465 valid loss: 0.06041584877918164 best epoch: 916 ======= epoch: 917 train loss: 0.06629944966968011 valid loss: 0.06041877344250679 best epoch: 916 ======= epoch: 918 train loss: 0.06629117511005865 valid loss: 0.06040627943972746 best epoch: 918 ======= epoch: 919 train loss: 0.06628144559461946 valid loss: 0.06041353972007831 best epoch: 918 ======= epoch: 920 train loss: 0.06627433986207586 valid loss: 0.0604017653192083 best epoch: 920 ======= epoch: 921 train loss: 0.06626431751467379 valid loss: 0.06040283478796482 best epoch: 920 ======= epoch: 922 train loss: 0.06625652486274455 valid loss: 0.06039713012675444 best epoch: 922 ======= epoch: 923 train loss: 0.0662466148144485 valid loss: 0.06039393755296866 best epoch: 923 ======= epoch: 924 train loss: 0.06623789843424786 valid loss: 0.06038930515448252 best epoch: 924 ======= epoch: 925 train loss: 0.06622832396102173 valid loss: 0.06038642426331838 best epoch: 925 ======= epoch: 926 train loss: 0.06621927991049525 valid loss: 0.06038418567428986 best epoch: 926 ======= epoch: 927 train loss: 0.06621001789599289 valid loss: 0.0603820870940884 best epoch: 927 ======= epoch: 928 train loss: 0.0662014506481312 valid loss: 0.06037676054984331 best epoch: 928 ======= epoch: 929 train loss: 0.06619294393623684 valid loss: 0.06037639733403921 best epoch: 929 ======= epoch: 930 train loss: 0.0661842816474813 valid loss: 0.06037095313270887 best epoch: 930 ======= epoch: 931 train loss: 0.0661755279108974 valid loss: 0.060370461704830326 best epoch: 931 ======= epoch: 932 train loss: 0.06616640960176785 valid loss: 0.060365304350852966 best epoch: 932 ======= epoch: 933 train loss: 0.06615705868045305 valid loss: 0.060365491857131325 best epoch: 932 ======= epoch: 934 train loss: 0.06614858119024171 valid loss: 0.06035806145519018 best epoch: 934 ======= epoch: 935 train loss: 0.0661388289493819 valid loss: 0.0603582626208663 best epoch: 934 ======= epoch: 936 train loss: 0.0661309029918486 valid loss: 0.06034913597007593 best epoch: 936 ======= epoch: 937 train loss: 0.06612118115694618 valid loss: 0.06034986861050129 best epoch: 936 ======= epoch: 938 train loss: 0.06611308009352213 valid loss: 0.06034239816168944 best epoch: 938 ======= epoch: 939 train loss: 0.0661035301623705 valid loss: 0.06034634355455637 best epoch: 938 ======= epoch: 940 train loss: 0.0660952260936208 valid loss: 0.060334546491503716 best epoch: 940 ======= epoch: 941 train loss: 0.06608568625754596 valid loss: 0.060339536828299366 best epoch: 940 ======= epoch: 942 train loss: 0.06607723439595213 valid loss: 0.060329590613643326 best epoch: 942 ======= epoch: 943 train loss: 0.06606744702928412 valid loss: 0.06033221508065859 best epoch: 942 ======= epoch: 944 train loss: 0.06605938102268143 valid loss: 0.060323803996046386 best epoch: 944 ======= epoch: 945 train loss: 0.06605000696202487 valid loss: 0.060323505041499935 best epoch: 945 ======= epoch: 946 train loss: 0.0660421581317981 valid loss: 0.06031912099570036 best epoch: 946 ======= epoch: 947 train loss: 0.06603418997155479 valid loss: 0.06031551926086346 best epoch: 947 ======= epoch: 948 train loss: 0.06602575506348117 valid loss: 0.06031441316008568 best epoch: 948 ======= epoch: 949 train loss: 0.06601790235849259 valid loss: 0.06031270418316126 best epoch: 949 ======= epoch: 950 train loss: 0.06600964962171367 valid loss: 0.06031225776920716 best epoch: 950 ======= epoch: 951 train loss: 0.06600095252908858 valid loss: 0.06031102780252695 best epoch: 951 ======= epoch: 952 train loss: 0.06599299209537329 valid loss: 0.06030792991320292 best epoch: 952 ======= epoch: 953 train loss: 0.06598355903370696 valid loss: 0.06030790011088053 best epoch: 953 ======= epoch: 954 train loss: 0.06597432023903102 valid loss: 0.06030374517043432 best epoch: 954 ======= epoch: 955 train loss: 0.06596411993428145 valid loss: 0.060306718883415066 best epoch: 954 ======= epoch: 956 train loss: 0.06595456775530804 valid loss: 0.060299768733481564 best epoch: 956 ======= epoch: 957 train loss: 0.0659447909436292 valid loss: 0.06030713549504677 best epoch: 956 ======= epoch: 958 train loss: 0.06593619958486086 valid loss: 0.06029770306001107 best epoch: 958 ======= epoch: 959 train loss: 0.0659252868125928 valid loss: 0.06029789646466573 best epoch: 958 ======= epoch: 960 train loss: 0.06591622787240295 valid loss: 0.06029200895378987 best epoch: 960 ======= epoch: 961 train loss: 0.06590595342402841 valid loss: 0.06028915289789438 best epoch: 961 ======= epoch: 962 train loss: 0.0658962217412521 valid loss: 0.060286310190955796 best epoch: 962 ======= epoch: 963 train loss: 0.06588627470450269 valid loss: 0.06028529163450003 best epoch: 963 ======= epoch: 964 train loss: 0.06587756184838069 valid loss: 0.060279768581191696 best epoch: 964 ======= epoch: 965 train loss: 0.0658676039174567 valid loss: 0.06028054809818665 best epoch: 964 ======= epoch: 966 train loss: 0.06585911641840214 valid loss: 0.06027285195887089 best epoch: 966 ======= epoch: 967 train loss: 0.06585153766597311 valid loss: 0.06027234811335802 best epoch: 967 ======= epoch: 968 train loss: 0.06584173004591355 valid loss: 0.060266388269762196 best epoch: 968 ======= epoch: 969 train loss: 0.06583258226214919 valid loss: 0.060265858036776386 best epoch: 969 ======= epoch: 970 train loss: 0.06582414691715881 valid loss: 0.06026256798456112 best epoch: 970 ======= epoch: 971 train loss: 0.06581555912271142 valid loss: 0.06026164085293809 best epoch: 971 ======= epoch: 972 train loss: 0.06580686935419101 valid loss: 0.06025936066483458 best epoch: 972 ======= epoch: 973 train loss: 0.0657974185111622 valid loss: 0.06026080048953494 best epoch: 972 ======= epoch: 974 train loss: 0.06578932742208794 valid loss: 0.06026020537440976 best epoch: 972 ======= epoch: 975 train loss: 0.06578081686417629 valid loss: 0.06026016020526489 best epoch: 972 ======= epoch: 976 train loss: 0.06577266064001086 valid loss: 0.06025475698212782 best epoch: 976 ======= epoch: 977 train loss: 0.0657654681449963 valid loss: 0.060253189876675606 best epoch: 977 ======= epoch: 978 train loss: 0.0657571911305925 valid loss: 0.060249023915578924 best epoch: 978 ======= epoch: 979 train loss: 0.06574860093310292 valid loss: 0.060246215046693884 best epoch: 979 ======= epoch: 980 train loss: 0.0657410562371858 valid loss: 0.06023849888394276 best epoch: 980 ======= epoch: 981 train loss: 0.06573287929396754 valid loss: 0.06023669087638458 best epoch: 981 ======= epoch: 982 train loss: 0.06572460578639566 valid loss: 0.060230770303557314 best epoch: 982 ======= epoch: 983 train loss: 0.06571687221021195 valid loss: 0.06022799341008067 best epoch: 983 ======= epoch: 984 train loss: 0.0657087004582547 valid loss: 0.06022209580987692 best epoch: 984 ======= epoch: 985 train loss: 0.06569992286371595 valid loss: 0.060219825711101294 best epoch: 985 ======= epoch: 986 train loss: 0.06569153142311139 valid loss: 0.06021240617459019 best epoch: 986 ======= epoch: 987 train loss: 0.06568193217785454 valid loss: 0.060210438134769596 best epoch: 987 ======= epoch: 988 train loss: 0.06567312012438054 valid loss: 0.06019895104691386 best epoch: 988 ======= epoch: 989 train loss: 0.06566599346582352 valid loss: 0.06020304715881745 best epoch: 988 ======= epoch: 990 train loss: 0.06565729952359825 valid loss: 0.06019000678012768 best epoch: 990 ======= epoch: 991 train loss: 0.06564890132228166 valid loss: 0.06019275154297551 best epoch: 990 ======= epoch: 992 train loss: 0.06564128895025746 valid loss: 0.06018504422778884 best epoch: 992 ======= epoch: 993 train loss: 0.0656330074114656 valid loss: 0.06018346408382058 best epoch: 993 ======= epoch: 994 train loss: 0.06562625897335417 valid loss: 0.06017874088138342 best epoch: 994 ======= epoch: 995 train loss: 0.0656176209392279 valid loss: 0.06017322465777397 best epoch: 995 ======= epoch: 996 train loss: 0.065611431127943 valid loss: 0.060167350651075445 best epoch: 996 ======= epoch: 997 train loss: 0.06560295609826292 valid loss: 0.06016564772774776 best epoch: 997 ======= epoch: 998 train loss: 0.06559618721129717 valid loss: 0.06015802143762509 best epoch: 998 ======= epoch: 999 train loss: 0.06558761166201697 valid loss: 0.06015818721304337 best epoch: 998 ======= epoch: 1000 train loss: 0.06558035664766292 valid loss: 0.060152443746725716 best epoch: 1000 ======= epoch: 1001 train loss: 0.06557163190280582 valid loss: 0.06015130629142126 best epoch: 1001 ======= epoch: 1002 train loss: 0.06556434240077197 valid loss: 0.0601468162300686 best epoch: 1002 ======= epoch: 1003 train loss: 0.06555549796946623 valid loss: 0.060145258282621704 best epoch: 1003 ======= epoch: 1004 train loss: 0.06554736923656346 valid loss: 0.060139411905159555 best epoch: 1004 ======= epoch: 1005 train loss: 0.06553873100122552 valid loss: 0.06013754398251573 best epoch: 1005 ======= epoch: 1006 train loss: 0.06553124440948904 valid loss: 0.06013348946968714 best epoch: 1006 ======= epoch: 1007 train loss: 0.06552233520527313 valid loss: 0.060128556564450264 best epoch: 1007 ======= epoch: 1008 train loss: 0.06551447828058843 valid loss: 0.06012223024542133 best epoch: 1008 ======= epoch: 1009 train loss: 0.06550534351234451 valid loss: 0.06011826591566205 best epoch: 1009 ======= epoch: 1010 train loss: 0.06549771501464241 valid loss: 0.06010818233092626 best epoch: 1010 ======= epoch: 1011 train loss: 0.06548837090003086 valid loss: 0.06010659737512469 best epoch: 1011 ======= epoch: 1012 train loss: 0.06548128901394429 valid loss: 0.06009880763789018 best epoch: 1012 ======= epoch: 1013 train loss: 0.06547245925959246 valid loss: 0.060097725596278906 best epoch: 1013 ======= epoch: 1014 train loss: 0.06546589307326042 valid loss: 0.06009569950401783 best epoch: 1014 ======= epoch: 1015 train loss: 0.0654569069257029 valid loss: 0.06009394877279798 best epoch: 1015 ======= epoch: 1016 train loss: 0.06544997413746184 valid loss: 0.06009196862578392 best epoch: 1016 ======= epoch: 1017 train loss: 0.06544105327446703 valid loss: 0.06008842928955952 best epoch: 1017 ======= epoch: 1018 train loss: 0.06543270543930524 valid loss: 0.06008128790805737 best epoch: 1018 ======= epoch: 1019 train loss: 0.0654237375330226 valid loss: 0.06008594095086058 best epoch: 1018 ======= epoch: 1020 train loss: 0.06541592110937208 valid loss: 0.06007482623681426 best epoch: 1020 ======= epoch: 1021 train loss: 0.06540767244856667 valid loss: 0.06007896907006701 best epoch: 1020 ======= epoch: 1022 train loss: 0.06539963131172606 valid loss: 0.06007081891099612 best epoch: 1022 ======= epoch: 1023 train loss: 0.06539248383638484 valid loss: 0.060070942621678114 best epoch: 1022 ======= epoch: 1024 train loss: 0.06538499360559163 valid loss: 0.06006552123775085 best epoch: 1024 ======= epoch: 1025 train loss: 0.06537695776349232 valid loss: 0.06006345820302764 best epoch: 1025 ======= epoch: 1026 train loss: 0.06536980795984466 valid loss: 0.060059069811056055 best epoch: 1026 ======= epoch: 1027 train loss: 0.06536128470835126 valid loss: 0.0600596919345359 best epoch: 1026 ======= epoch: 1028 train loss: 0.06535436679252687 valid loss: 0.06005227736507853 best epoch: 1028 ======= epoch: 1029 train loss: 0.06534593953624551 valid loss: 0.06005439721047878 best epoch: 1028 ======= epoch: 1030 train loss: 0.06533848578800205 valid loss: 0.060050646774470806 best epoch: 1030 ======= epoch: 1031 train loss: 0.06533049126903032 valid loss: 0.06005142039308945 best epoch: 1030 ======= epoch: 1032 train loss: 0.06532267264354928 valid loss: 0.06004941323772073 best epoch: 1032 ======= epoch: 1033 train loss: 0.06531385710049006 valid loss: 0.060051527339965105 best epoch: 1032 ======= epoch: 1034 train loss: 0.0653063667949611 valid loss: 0.06004952732473612 best epoch: 1032 ======= epoch: 1035 train loss: 0.06529720451245889 valid loss: 0.06005421622345845 best epoch: 1032 ======= epoch: 1036 train loss: 0.06528978593026598 valid loss: 0.060053315168867506 best epoch: 1032 ======= epoch: 1037 train loss: 0.06528209591353382 valid loss: 0.060053871013224125 best epoch: 1032 ======= epoch: 1038 train loss: 0.0652743283038338 valid loss: 0.060052937207122646 best epoch: 1032 ======= epoch: 1039 train loss: 0.06526614860114124 valid loss: 0.060053994258244835 best epoch: 1032 ======= epoch: 1040 train loss: 0.06525931833311915 valid loss: 0.060051560401916504 best epoch: 1032 ======= epoch: 1041 train loss: 0.06525157828941758 valid loss: 0.06005492495993773 best epoch: 1032 ======= epoch: 1042 train loss: 0.06524487124916947 valid loss: 0.06005213735625148 best epoch: 1032 ======= epoch: 1043 train loss: 0.06523673017911705 valid loss: 0.06005055379743377 best epoch: 1032 ======= epoch: 1044 train loss: 0.06522996536812481 valid loss: 0.060047178994864225 best epoch: 1044 ======= epoch: 1045 train loss: 0.06522209555439559 valid loss: 0.06004938343539834 best epoch: 1044 ======= epoch: 1046 train loss: 0.06521558524741802 valid loss: 0.060044913863142334 best epoch: 1046 ======= epoch: 1047 train loss: 0.06520747878002348 valid loss: 0.060049425500134625 best epoch: 1046 ======= epoch: 1048 train loss: 0.06520097914876209 valid loss: 0.06004053199042877 best epoch: 1048 ======= epoch: 1049 train loss: 0.06519276584372108 valid loss: 0.06004377563173572 best epoch: 1048 ======= epoch: 1050 train loss: 0.06518701611109722 valid loss: 0.060035726986825466 best epoch: 1050 ======= epoch: 1051 train loss: 0.06517855612630094 valid loss: 0.06003725357974569 best epoch: 1050 ======= epoch: 1052 train loss: 0.06517214296431637 valid loss: 0.060029791202396154 best epoch: 1052 ======= epoch: 1053 train loss: 0.06516438132772844 valid loss: 0.060029954804728426 best epoch: 1052 ======= epoch: 1054 train loss: 0.06515705884997676 valid loss: 0.06002853329603871 best epoch: 1054 ======= epoch: 1055 train loss: 0.06514975726374506 valid loss: 0.06002573467170199 best epoch: 1055 ======= epoch: 1056 train loss: 0.06514272908392696 valid loss: 0.06002166587859392 best epoch: 1056 ======= epoch: 1057 train loss: 0.06513527704310822 valid loss: 0.060020283330231905 best epoch: 1057 ======= epoch: 1058 train loss: 0.065128477273026 valid loss: 0.060013381919513144 best epoch: 1058 ======= epoch: 1059 train loss: 0.06512081679416659 valid loss: 0.06001292805497845 best epoch: 1059 ======= epoch: 1060 train loss: 0.06511426258855212 valid loss: 0.06000691885128617 best epoch: 1060 ======= epoch: 1061 train loss: 0.06510612339839156 valid loss: 0.06000601810713609 best epoch: 1061 ======= epoch: 1062 train loss: 0.06509972282650846 valid loss: 0.06000253170107802 best epoch: 1062 ======= epoch: 1063 train loss: 0.06509103101712686 valid loss: 0.060001630491266646 best epoch: 1063 ======= epoch: 1064 train loss: 0.06508443650019205 valid loss: 0.05999788253878554 best epoch: 1064 ======= epoch: 1065 train loss: 0.06507583445995863 valid loss: 0.05999693460762501 best epoch: 1065 ======= epoch: 1066 train loss: 0.06506876672767563 valid loss: 0.05998971142495672 best epoch: 1066 ======= epoch: 1067 train loss: 0.0650607233027709 valid loss: 0.059991339376817145 best epoch: 1066 ======= epoch: 1068 train loss: 0.06505338410720413 valid loss: 0.05998445923129717 best epoch: 1068 ======= epoch: 1069 train loss: 0.06504599257360821 valid loss: 0.05998295250659188 best epoch: 1069 ======= epoch: 1070 train loss: 0.06503987196189018 valid loss: 0.059980084809164204 best epoch: 1070 ======= epoch: 1071 train loss: 0.06503212677866772 valid loss: 0.05997774206722776 best epoch: 1071 ======= epoch: 1072 train loss: 0.06502631922355956 valid loss: 0.059970458038151264 best epoch: 1072 ======= epoch: 1073 train loss: 0.06501866913872001 valid loss: 0.059969756907473006 best epoch: 1073 ======= epoch: 1074 train loss: 0.06501341979651723 valid loss: 0.0599625064060092 best epoch: 1074 ======= epoch: 1075 train loss: 0.06500621492411067 valid loss: 0.059959655006726585 best epoch: 1075 ======= epoch: 1076 train loss: 0.06500097697048828 valid loss: 0.059950855715821184 best epoch: 1076 ======= epoch: 1077 train loss: 0.06499369966763037 valid loss: 0.05995110313718518 best epoch: 1076 ======= epoch: 1078 train loss: 0.06498816467384681 valid loss: 0.0599412526935339 best epoch: 1078 ======= epoch: 1079 train loss: 0.0649806882576718 valid loss: 0.059940795258929334 best epoch: 1079 ======= epoch: 1080 train loss: 0.0649743987718758 valid loss: 0.05993285635486245 best epoch: 1080 ======= epoch: 1081 train loss: 0.06496659206962328 valid loss: 0.059930623043328524 best epoch: 1081 ======= epoch: 1082 train loss: 0.06495924081860317 valid loss: 0.05992633601029714 best epoch: 1082 ======= epoch: 1083 train loss: 0.06495188411187243 valid loss: 0.05992302360634009 best epoch: 1083 ======= epoch: 1084 train loss: 0.06494488997706845 valid loss: 0.05992134908835093 best epoch: 1084 ======= epoch: 1085 train loss: 0.06493776833730532 valid loss: 0.059920345122615494 best epoch: 1085 ======= epoch: 1086 train loss: 0.06493159027103289 valid loss: 0.05991443137948712 best epoch: 1086 ======= epoch: 1087 train loss: 0.06492404775969961 valid loss: 0.05991571101670464 best epoch: 1086 ======= epoch: 1088 train loss: 0.06491790292209682 valid loss: 0.059911667524526514 best epoch: 1088 ======= epoch: 1089 train loss: 0.06491021246269897 valid loss: 0.05991322407498956 best epoch: 1088 ======= epoch: 1090 train loss: 0.06490422316746396 valid loss: 0.05990873618672291 best epoch: 1090 ======= epoch: 1091 train loss: 0.06489603996943728 valid loss: 0.05990568005169431 best epoch: 1091 ======= epoch: 1092 train loss: 0.06488995171830426 valid loss: 0.059900914784520864 best epoch: 1092 ======= epoch: 1093 train loss: 0.06488154033474901 valid loss: 0.05989804010217389 best epoch: 1093 ======= epoch: 1094 train loss: 0.06487521878731103 valid loss: 0.05989359552040696 best epoch: 1094 ======= epoch: 1095 train loss: 0.06486752933397154 valid loss: 0.05989430130769809 best epoch: 1094 ======= epoch: 1096 train loss: 0.0648615087646945 valid loss: 0.05988978439321121 best epoch: 1096 ======= epoch: 1097 train loss: 0.06485427014046798 valid loss: 0.05988955280433098 best epoch: 1097 ======= epoch: 1098 train loss: 0.06484779714963135 valid loss: 0.05988323533286651 best epoch: 1098 ======= epoch: 1099 train loss: 0.0648404498193643 valid loss: 0.059882191475480795 best epoch: 1099 ======= epoch: 1100 train loss: 0.06483383617582328 valid loss: 0.05987726617604494 best epoch: 1100 ======= epoch: 1101 train loss: 0.0648263412769194 valid loss: 0.059880598758657776 best epoch: 1100 ======= epoch: 1102 train loss: 0.06482037126925992 valid loss: 0.0598770414168636 best epoch: 1102 ======= epoch: 1103 train loss: 0.06481312503348347 valid loss: 0.05987725577627619 best epoch: 1102 ======= epoch: 1104 train loss: 0.06480734744166702 valid loss: 0.0598706832776467 best epoch: 1104 ======= epoch: 1105 train loss: 0.06479970355989573 valid loss: 0.059873262885957956 best epoch: 1104 ======= epoch: 1106 train loss: 0.06479415660839022 valid loss: 0.059867642975101866 best epoch: 1106 ======= epoch: 1107 train loss: 0.06478635759242339 valid loss: 0.05986700455347697 best epoch: 1107 ======= epoch: 1108 train loss: 0.06478095380014476 valid loss: 0.05986562961091598 best epoch: 1108 ======= epoch: 1109 train loss: 0.0647734256323289 valid loss: 0.059864841712017856 best epoch: 1109 ======= epoch: 1110 train loss: 0.06476821495725969 valid loss: 0.05985954745362202 best epoch: 1110 ======= epoch: 1111 train loss: 0.06476120799090024 valid loss: 0.05985976414134105 best epoch: 1110 ======= epoch: 1112 train loss: 0.06475575275167271 valid loss: 0.05985657963901758 best epoch: 1112 ======= epoch: 1113 train loss: 0.06474882842007058 valid loss: 0.05985551172246536 best epoch: 1113 ======= epoch: 1114 train loss: 0.06474326607874697 valid loss: 0.05985002887124816 best epoch: 1114 ======= epoch: 1115 train loss: 0.06473601534157808 valid loss: 0.05985280421252052 best epoch: 1114 ======= epoch: 1116 train loss: 0.06473051839404637 valid loss: 0.0598423082071046 best epoch: 1116 ======= epoch: 1117 train loss: 0.06472352769133853 valid loss: 0.059844103952248894 best epoch: 1116 ======= epoch: 1118 train loss: 0.06471802961127257 valid loss: 0.05983380740508437 best epoch: 1118 ======= epoch: 1119 train loss: 0.0647114875530939 valid loss: 0.05983504606410861 best epoch: 1118 ======= epoch: 1120 train loss: 0.06470586170361918 valid loss: 0.05983085464686155 best epoch: 1120 ======= epoch: 1121 train loss: 0.06469925985108187 valid loss: 0.059828192771722875 best epoch: 1121 ======= epoch: 1122 train loss: 0.0646937486577641 valid loss: 0.05982207149888078 best epoch: 1122 ======= epoch: 1123 train loss: 0.0646869855599455 valid loss: 0.05982227871815363 best epoch: 1122 ======= epoch: 1124 train loss: 0.0646816179972648 valid loss: 0.059820235862086214 best epoch: 1124 ======= epoch: 1125 train loss: 0.06467433632523925 valid loss: 0.05981857189908624 best epoch: 1125 ======= epoch: 1126 train loss: 0.064669465025266 valid loss: 0.059813713654875755 best epoch: 1126 ======= epoch: 1127 train loss: 0.06466198383750003 valid loss: 0.05981442037348946 best epoch: 1126 ======= epoch: 1128 train loss: 0.06465691531559935 valid loss: 0.05980668434252342 best epoch: 1128 ======= epoch: 1129 train loss: 0.06464962120869278 valid loss: 0.059809175630410515 best epoch: 1128 ======= epoch: 1130 train loss: 0.06464443502970683 valid loss: 0.05980025123183926 best epoch: 1130 ======= epoch: 1131 train loss: 0.06463735397146256 valid loss: 0.05980238039046526 best epoch: 1130 ======= epoch: 1132 train loss: 0.06463185267775882 valid loss: 0.0597962678099672 best epoch: 1132 ======= epoch: 1133 train loss: 0.06462537490383342 valid loss: 0.05979487579315901 best epoch: 1133 ======= epoch: 1134 train loss: 0.06461932952228336 valid loss: 0.05979208927601576 best epoch: 1134 ======= epoch: 1135 train loss: 0.06461313812430074 valid loss: 0.059790199622511864 best epoch: 1135 ======= epoch: 1136 train loss: 0.06460696950922777 valid loss: 0.05978615923474232 best epoch: 1136 ======= epoch: 1137 train loss: 0.06460047719057327 valid loss: 0.05978756599749128 best epoch: 1136 ======= epoch: 1138 train loss: 0.06459444626751873 valid loss: 0.059784484871973596 best epoch: 1138 ======= epoch: 1139 train loss: 0.06458773416248553 valid loss: 0.05978588263193766 best epoch: 1138 ======= epoch: 1140 train loss: 0.0645824427900399 valid loss: 0.05978218171124657 best epoch: 1140 ======= epoch: 1141 train loss: 0.06457510251942793 valid loss: 0.05978223324442903 best epoch: 1140 ======= epoch: 1142 train loss: 0.06456903702820893 valid loss: 0.059778514628609024 best epoch: 1142 ======= epoch: 1143 train loss: 0.0645621732843143 valid loss: 0.05977789141858617 best epoch: 1143 ======= epoch: 1144 train loss: 0.0645561760670885 valid loss: 0.05977635628854235 best epoch: 1144 ======= epoch: 1145 train loss: 0.06454970917583984 valid loss: 0.05977498833090067 best epoch: 1145 ======= epoch: 1146 train loss: 0.06454393153803216 valid loss: 0.0597715323480467 best epoch: 1146 ======= epoch: 1147 train loss: 0.06453773524773157 valid loss: 0.05977214376131693 best epoch: 1146 ======= epoch: 1148 train loss: 0.06453244036270513 valid loss: 0.059766663859287895 best epoch: 1148 ======= epoch: 1149 train loss: 0.0645261109392676 valid loss: 0.059767017140984535 best epoch: 1148 ======= epoch: 1150 train loss: 0.06452135045518294 valid loss: 0.05976053777461251 best epoch: 1150 ======= epoch: 1151 train loss: 0.06451485276705136 valid loss: 0.05976091930642724 best epoch: 1150 ======= epoch: 1152 train loss: 0.0645097706777354 valid loss: 0.05975789235283931 best epoch: 1152 ======= epoch: 1153 train loss: 0.06450338461033908 valid loss: 0.059757235925644636 best epoch: 1153 ======= epoch: 1154 train loss: 0.0644980383588484 valid loss: 0.059754036677380405 best epoch: 1154 ======= epoch: 1155 train loss: 0.06449097873252115 valid loss: 0.059756383299827576 best epoch: 1154 ======= epoch: 1156 train loss: 0.06448580448251263 valid loss: 0.05975111744677027 best epoch: 1156 ======= epoch: 1157 train loss: 0.06447822068829412 valid loss: 0.05975166413312157 best epoch: 1156 ======= epoch: 1158 train loss: 0.0644725015685882 valid loss: 0.059747131230930485 best epoch: 1158 ======= epoch: 1159 train loss: 0.06446546630611942 valid loss: 0.05974695645272732 best epoch: 1159 ======= epoch: 1160 train loss: 0.06445921268199145 valid loss: 0.05974588248257836 best epoch: 1160 ======= epoch: 1161 train loss: 0.06445263116333036 valid loss: 0.05974435977016886 best epoch: 1161 ======= epoch: 1162 train loss: 0.0644469503198325 valid loss: 0.05974179568390051 best epoch: 1162 ======= epoch: 1163 train loss: 0.06444086705299991 valid loss: 0.059742743615061045 best epoch: 1162 ======= epoch: 1164 train loss: 0.06443560781693201 valid loss: 0.05973937828093767 best epoch: 1164 ======= epoch: 1165 train loss: 0.06443028242246789 valid loss: 0.05974023090675473 best epoch: 1164 ======= epoch: 1166 train loss: 0.06442531695543433 valid loss: 0.059734787015865244 best epoch: 1166 ======= epoch: 1167 train loss: 0.0644201501445086 valid loss: 0.05973623956864079 best epoch: 1166 ======= epoch: 1168 train loss: 0.0644149694442289 valid loss: 0.059731853970636926 best epoch: 1168 ======= epoch: 1169 train loss: 0.06440925835574299 valid loss: 0.05973494673768679 best epoch: 1168 ======= epoch: 1170 train loss: 0.0644046584788112 valid loss: 0.0597288622520864 best epoch: 1170 ======= epoch: 1171 train loss: 0.0643984987055538 valid loss: 0.0597330373711884 best epoch: 1170 ======= epoch: 1172 train loss: 0.06439388958438311 valid loss: 0.059729337226599455 best epoch: 1170 ======= epoch: 1173 train loss: 0.06438766955685468 valid loss: 0.05973258117834727 best epoch: 1170 ======= epoch: 1174 train loss: 0.0643828923434571 valid loss: 0.059730414456377424 best epoch: 1170 ======= epoch: 1175 train loss: 0.06437659463672726 valid loss: 0.059731747644642987 best epoch: 1170 ======= epoch: 1176 train loss: 0.06437175804864109 valid loss: 0.05972968755910794 best epoch: 1170 ======= epoch: 1177 train loss: 0.06436612317338586 valid loss: 0.05972939496859908 best epoch: 1170 ======= epoch: 1178 train loss: 0.06436153807113935 valid loss: 0.05972789196918408 best epoch: 1178 ======= epoch: 1179 train loss: 0.06435584763271941 valid loss: 0.05972929314399759 best epoch: 1178 ======= epoch: 1180 train loss: 0.06435124742810006 valid loss: 0.05972565927853187 best epoch: 1180 ======= epoch: 1181 train loss: 0.06434508176221523 valid loss: 0.05972602916881442 best epoch: 1180 ======= epoch: 1182 train loss: 0.06434018958796874 valid loss: 0.05972230279197296 best epoch: 1182 ======= epoch: 1183 train loss: 0.06433328922929955 valid loss: 0.05972293997183442 best epoch: 1182 ======= epoch: 1184 train loss: 0.06432779569086837 valid loss: 0.059716340620070696 best epoch: 1184 ======= epoch: 1185 train loss: 0.0643213385210177 valid loss: 0.05971914871285359 best epoch: 1184 ======= epoch: 1186 train loss: 0.06431650110508924 valid loss: 0.059712556501229606 best epoch: 1186 ======= epoch: 1187 train loss: 0.06431033296142648 valid loss: 0.05971367579574386 best epoch: 1186 ======= epoch: 1188 train loss: 0.0643062445588042 valid loss: 0.05970820148165027 best epoch: 1188 ======= epoch: 1189 train loss: 0.0643002863134039 valid loss: 0.05970848072320223 best epoch: 1188 ======= epoch: 1190 train loss: 0.06429610063356382 valid loss: 0.05970264614249269 best epoch: 1190 ======= epoch: 1191 train loss: 0.06428986328458529 valid loss: 0.05970489621783296 best epoch: 1190 ======= epoch: 1192 train loss: 0.06428570727484277 valid loss: 0.05969950836151838 best epoch: 1192 ======= epoch: 1193 train loss: 0.06427899558947962 valid loss: 0.05969964060932398 best epoch: 1192 ======= epoch: 1194 train loss: 0.06427409832170349 valid loss: 0.059695316944271326 best epoch: 1194 ======= epoch: 1195 train loss: 0.06426694455706043 valid loss: 0.05969778401777148 best epoch: 1194 ======= epoch: 1196 train loss: 0.06426094777675138 valid loss: 0.059694585390388966 best epoch: 1196 ======= epoch: 1197 train loss: 0.06425495122064963 valid loss: 0.05969345352301995 best epoch: 1197 ======= epoch: 1198 train loss: 0.06424960250862771 valid loss: 0.059690989864369236 best epoch: 1198 ======= epoch: 1199 train loss: 0.0642435555173843 valid loss: 0.05969000793993473 best epoch: 1199 ======= epoch: 1200 train loss: 0.06423875731473536 valid loss: 0.05968570973103245 best epoch: 1200 ======= epoch: 1201 train loss: 0.0642325479514253 valid loss: 0.05968630484615763 best epoch: 1200 ======= epoch: 1202 train loss: 0.06422802678849777 valid loss: 0.05967909221847852 best epoch: 1202 ======= epoch: 1203 train loss: 0.06422125404976585 valid loss: 0.05968206158528725 best epoch: 1202 ======= epoch: 1204 train loss: 0.06421703941844127 valid loss: 0.05967552059640487 best epoch: 1204 ======= epoch: 1205 train loss: 0.06421024549499522 valid loss: 0.05967856772864858 best epoch: 1204 ======= epoch: 1206 train loss: 0.06420642611812111 valid loss: 0.05967111621672908 best epoch: 1206 ======= epoch: 1207 train loss: 0.06419970545296867 valid loss: 0.05967481201514602 best epoch: 1206 ======= epoch: 1208 train loss: 0.06419584927735505 valid loss: 0.0596678884079059 best epoch: 1208 ======= epoch: 1209 train loss: 0.06418966230602911 valid loss: 0.059667728996525206 best epoch: 1209 ======= epoch: 1210 train loss: 0.06418534855783721 valid loss: 0.059662421233952045 best epoch: 1210 ======= epoch: 1211 train loss: 0.0641790731223645 valid loss: 0.05966528862093886 best epoch: 1210 ======= epoch: 1212 train loss: 0.06417461873871493 valid loss: 0.05966100996981064 best epoch: 1212 ======= epoch: 1213 train loss: 0.06416806231402321 valid loss: 0.059661226657529674 best epoch: 1212 ======= epoch: 1214 train loss: 0.06416403910973006 valid loss: 0.05965800862759352 best epoch: 1214 ======= epoch: 1215 train loss: 0.06415769140477534 valid loss: 0.05965869206314286 best epoch: 1214 ======= epoch: 1216 train loss: 0.06415427043275149 valid loss: 0.05965387728065252 best epoch: 1216 ======= epoch: 1217 train loss: 0.06414807253850648 valid loss: 0.05965336443235477 best epoch: 1217 ======= epoch: 1218 train loss: 0.06414402082539451 valid loss: 0.05964633993183573 best epoch: 1218 ======= epoch: 1219 train loss: 0.06413802080270317 valid loss: 0.05964689034347733 best epoch: 1218 ======= epoch: 1220 train loss: 0.06413371618952096 valid loss: 0.05964053003117442 best epoch: 1220 ======= epoch: 1221 train loss: 0.06412769917306708 valid loss: 0.059642382276554905 best epoch: 1220 ======= epoch: 1222 train loss: 0.06412292289674099 valid loss: 0.059637223525593676 best epoch: 1222 ======= epoch: 1223 train loss: 0.06411663119186774 valid loss: 0.0596381265980502 best epoch: 1222 ======= epoch: 1224 train loss: 0.06411196546698059 valid loss: 0.059632939752191305 best epoch: 1224 ======= epoch: 1225 train loss: 0.06410554883265752 valid loss: 0.05963689958055814 best epoch: 1224 ======= epoch: 1226 train loss: 0.06410103056142911 valid loss: 0.05963019017751018 best epoch: 1226 ======= epoch: 1227 train loss: 0.06409483962910778 valid loss: 0.05963430336366097 best epoch: 1226 ======= epoch: 1228 train loss: 0.06409076961723191 valid loss: 0.059626882119725146 best epoch: 1228 ======= epoch: 1229 train loss: 0.06408453096325199 valid loss: 0.05962756276130676 best epoch: 1228 ======= epoch: 1230 train loss: 0.06408055229572418 valid loss: 0.059624445314208664 best epoch: 1230 ======= epoch: 1231 train loss: 0.06407423819311792 valid loss: 0.05962426013623675 best epoch: 1231 ======= epoch: 1232 train loss: 0.06407034845539818 valid loss: 0.05962091994782289 best epoch: 1232 ======= epoch: 1233 train loss: 0.0640641854857497 valid loss: 0.05962316303824385 best epoch: 1232 ======= epoch: 1234 train loss: 0.06406054352582605 valid loss: 0.059620605781674385 best epoch: 1234 ======= epoch: 1235 train loss: 0.06405453977784441 valid loss: 0.05962266974772016 best epoch: 1234 ======= epoch: 1236 train loss: 0.06405063115497246 valid loss: 0.05962040089070797 best epoch: 1236 ======= epoch: 1237 train loss: 0.06404459367241756 valid loss: 0.05962578160688281 best epoch: 1236 ======= epoch: 1238 train loss: 0.06404095953100442 valid loss: 0.05962121471141776 best epoch: 1236 ======= epoch: 1239 train loss: 0.06403427590029659 valid loss: 0.05962472843627135 best epoch: 1236 ======= epoch: 1240 train loss: 0.06403108434406696 valid loss: 0.05961958039551973 best epoch: 1240 ======= epoch: 1241 train loss: 0.06402413852880766 valid loss: 0.05962188309058547 best epoch: 1240 ======= epoch: 1242 train loss: 0.06402039691161962 valid loss: 0.05961731200416883 best epoch: 1242 ======= epoch: 1243 train loss: 0.06401400987265839 valid loss: 0.059618445578962564 best epoch: 1242 ======= epoch: 1244 train loss: 0.06401082712375088 valid loss: 0.05961559076483051 best epoch: 1244 ======= epoch: 1245 train loss: 0.06400414918445879 valid loss: 0.059616539627313614 best epoch: 1244 ======= epoch: 1246 train loss: 0.0640004195364905 valid loss: 0.05961269186809659 best epoch: 1246 ======= epoch: 1247 train loss: 0.06399429634351421 valid loss: 0.05961370002478361 best epoch: 1246 ======= epoch: 1248 train loss: 0.06399020963681885 valid loss: 0.05960999211917321 best epoch: 1248 ======= epoch: 1249 train loss: 0.06398423910026013 valid loss: 0.059611343002567686 best epoch: 1248 ======= epoch: 1250 train loss: 0.06398055527106296 valid loss: 0.05960815896590551 best epoch: 1250 ======= epoch: 1251 train loss: 0.0639743778946591 valid loss: 0.05960613116621971 best epoch: 1251 ======= epoch: 1252 train loss: 0.06397063455279962 valid loss: 0.0596015932969749 best epoch: 1252 ======= epoch: 1253 train loss: 0.06396470342498319 valid loss: 0.0596052206431826 best epoch: 1252 ======= epoch: 1254 train loss: 0.06396101124837626 valid loss: 0.05960054276511073 best epoch: 1254 ======= epoch: 1255 train loss: 0.06395478324711691 valid loss: 0.059600663681825004 best epoch: 1254 ======= epoch: 1256 train loss: 0.06395096021393935 valid loss: 0.059596216616531215 best epoch: 1256 ======= epoch: 1257 train loss: 0.06394457297951535 valid loss: 0.0595992241675655 best epoch: 1256 ======= epoch: 1258 train loss: 0.06394098484658717 valid loss: 0.05959127812335888 best epoch: 1258 ======= epoch: 1259 train loss: 0.0639343224408358 valid loss: 0.05959359851355354 best epoch: 1258 ======= epoch: 1260 train loss: 0.063930721786793 valid loss: 0.05958608786265055 best epoch: 1260 ======= epoch: 1261 train loss: 0.06392414395317987 valid loss: 0.05958560574799776 best epoch: 1261 ======= epoch: 1262 train loss: 0.06392034462273673 valid loss: 0.05958083194370071 best epoch: 1262 ======= epoch: 1263 train loss: 0.06391369185214978 valid loss: 0.05958130614211162 best epoch: 1262 ======= epoch: 1264 train loss: 0.06390960957211109 valid loss: 0.05957791224742929 best epoch: 1264 ======= epoch: 1265 train loss: 0.0639029963194956 valid loss: 0.05957772862166166 best epoch: 1265 ======= epoch: 1266 train loss: 0.06390007788798323 valid loss: 0.059574401626984276 best epoch: 1266 ======= epoch: 1267 train loss: 0.06389312861763217 valid loss: 0.05957452688987056 best epoch: 1266 ======= epoch: 1268 train loss: 0.06388987704858552 valid loss: 0.059570814638088145 best epoch: 1268 ======= epoch: 1269 train loss: 0.06388353277572695 valid loss: 0.05957211724792918 best epoch: 1268 ======= epoch: 1270 train loss: 0.06387947696939479 valid loss: 0.05956937357162436 best epoch: 1270 ======= epoch: 1271 train loss: 0.06387319523887133 valid loss: 0.059568418034662805 best epoch: 1271 ======= epoch: 1272 train loss: 0.06386922654569333 valid loss: 0.05956584044421712 best epoch: 1272 ======= epoch: 1273 train loss: 0.0638636021391937 valid loss: 0.059566316505273185 best epoch: 1272 ======= epoch: 1274 train loss: 0.06385936300028805 valid loss: 0.05956402001902461 best epoch: 1274 ======= epoch: 1275 train loss: 0.06385383902140605 valid loss: 0.05956577059502403 best epoch: 1274 ======= epoch: 1276 train loss: 0.06384956987321745 valid loss: 0.05956342335169514 best epoch: 1276 ======= epoch: 1277 train loss: 0.06384408751065716 valid loss: 0.059565260695914425 best epoch: 1276 ======= epoch: 1278 train loss: 0.06383979511964652 valid loss: 0.05956016496444742 best epoch: 1278 ======= epoch: 1279 train loss: 0.06383361507575085 valid loss: 0.05956340705355009 best epoch: 1278 ======= epoch: 1280 train loss: 0.06382965782289336 valid loss: 0.05955972243100405 best epoch: 1280 ======= epoch: 1281 train loss: 0.06382309275184885 valid loss: 0.059561988494048514 best epoch: 1280 ======= epoch: 1282 train loss: 0.06381921456367881 valid loss: 0.05955619256322583 best epoch: 1282 ======= epoch: 1283 train loss: 0.06381286000602959 valid loss: 0.05955850053578615 best epoch: 1282 ======= epoch: 1284 train loss: 0.06380917747608489 valid loss: 0.05955589531610409 best epoch: 1284 ======= epoch: 1285 train loss: 0.06380307994621956 valid loss: 0.059555897023528814 best epoch: 1284 ======= epoch: 1286 train loss: 0.06379984873404473 valid loss: 0.05955348567416271 best epoch: 1286 ======= epoch: 1287 train loss: 0.06379347805646651 valid loss: 0.05955444679905971 best epoch: 1286 ======= epoch: 1288 train loss: 0.06379035357845786 valid loss: 0.05954902432858944 best epoch: 1288 ======= epoch: 1289 train loss: 0.0637831236983155 valid loss: 0.059549686343719564 best epoch: 1288 ======= epoch: 1290 train loss: 0.06377920759611844 valid loss: 0.059543774308015905 best epoch: 1290 ======= epoch: 1291 train loss: 0.06377136663039341 valid loss: 0.059545665979385376 best epoch: 1290 ======= epoch: 1292 train loss: 0.06376719966323839 valid loss: 0.059542106464505196 best epoch: 1292 ======= epoch: 1293 train loss: 0.0637595240475127 valid loss: 0.0595440372514228 best epoch: 1292 ======= epoch: 1294 train loss: 0.06375504298894494 valid loss: 0.0595383468704919 best epoch: 1294 ======= epoch: 1295 train loss: 0.06374824628877787 valid loss: 0.059539970165739454 best epoch: 1294 ======= epoch: 1296 train loss: 0.06374382416798192 valid loss: 0.05953430167088906 best epoch: 1296 ======= epoch: 1297 train loss: 0.06373755013524199 valid loss: 0.05953550447399417 best epoch: 1296 ======= epoch: 1298 train loss: 0.06373290222621074 valid loss: 0.059530407190322876 best epoch: 1298 ======= epoch: 1299 train loss: 0.0637262762742646 valid loss: 0.059530488370607294 best epoch: 1298 ======= epoch: 1300 train loss: 0.06372214259326826 valid loss: 0.05952697262788812 best epoch: 1300 ======= epoch: 1301 train loss: 0.06371522829350498 valid loss: 0.05952810340871414 best epoch: 1300 ======= epoch: 1302 train loss: 0.06371072054277231 valid loss: 0.059524388828625284 best epoch: 1302 ======= epoch: 1303 train loss: 0.06370473187991682 valid loss: 0.05952451657503843 best epoch: 1302 ======= epoch: 1304 train loss: 0.06370009716462206 valid loss: 0.059521118799845375 best epoch: 1304 ======= epoch: 1305 train loss: 0.06369393290147002 valid loss: 0.05952159175649285 best epoch: 1304 ======= epoch: 1306 train loss: 0.06368949809284122 valid loss: 0.05951730549956361 best epoch: 1306 ======= epoch: 1307 train loss: 0.06368279705444972 valid loss: 0.05951972305774689 best epoch: 1306 ======= epoch: 1308 train loss: 0.0636777678148154 valid loss: 0.05951544471705953 best epoch: 1308 ======= epoch: 1309 train loss: 0.06367099529454186 valid loss: 0.05951711582019925 best epoch: 1308 ======= epoch: 1310 train loss: 0.06366629593281282 valid loss: 0.05951196296761433 best epoch: 1310 ======= epoch: 1311 train loss: 0.06365898753987786 valid loss: 0.059513711370527744 best epoch: 1310 ======= epoch: 1312 train loss: 0.06365492062091276 valid loss: 0.05950869247317314 best epoch: 1312 ======= epoch: 1313 train loss: 0.0636483101680139 valid loss: 0.05951201062028607 best epoch: 1312 ======= epoch: 1314 train loss: 0.06364426003585075 valid loss: 0.059507413456837334 best epoch: 1314 ======= epoch: 1315 train loss: 0.06363725905410117 valid loss: 0.05950905242934823 best epoch: 1314 ======= epoch: 1316 train loss: 0.06363363202983215 valid loss: 0.059507422149181366 best epoch: 1314 ======= epoch: 1317 train loss: 0.06362635653787925 valid loss: 0.059506832311550774 best epoch: 1317 ======= epoch: 1318 train loss: 0.06362284644058457 valid loss: 0.05950377508997917 best epoch: 1318 ======= epoch: 1319 train loss: 0.063616530302866 valid loss: 0.059506529631714024 best epoch: 1318 ======= epoch: 1320 train loss: 0.06361307492939594 valid loss: 0.059501624355713524 best epoch: 1320 ======= epoch: 1321 train loss: 0.06360611555247027 valid loss: 0.059504792715112366 best epoch: 1320 ======= epoch: 1322 train loss: 0.06360274993665056 valid loss: 0.0594988662439088 best epoch: 1322 ======= epoch: 1323 train loss: 0.06359581434869288 valid loss: 0.059501186634103455 best epoch: 1322 ======= epoch: 1324 train loss: 0.06359170499131267 valid loss: 0.05949487381925186 best epoch: 1324 ======= epoch: 1325 train loss: 0.06358502711525854 valid loss: 0.05949785296494762 best epoch: 1324 ======= epoch: 1326 train loss: 0.06358033073146219 valid loss: 0.05949472775682807 best epoch: 1326 ======= epoch: 1327 train loss: 0.0635738521411923 valid loss: 0.059493014899392925 best epoch: 1327 ======= epoch: 1328 train loss: 0.06356837640711922 valid loss: 0.05948963978638252 best epoch: 1328 ======= epoch: 1329 train loss: 0.06356228284974709 valid loss: 0.05949117600296935 best epoch: 1328 ======= epoch: 1330 train loss: 0.06355641908184798 valid loss: 0.05948802533869942 best epoch: 1330 ======= epoch: 1331 train loss: 0.06355022115885843 valid loss: 0.05948978398616115 best epoch: 1330 ======= epoch: 1332 train loss: 0.06354526371154705 valid loss: 0.05948656890541315 best epoch: 1332 ======= epoch: 1333 train loss: 0.0635382248962551 valid loss: 0.059486257998893656 best epoch: 1333 ======= epoch: 1334 train loss: 0.06353302570749764 valid loss: 0.05948370757202307 best epoch: 1334 ======= epoch: 1335 train loss: 0.0635263288313132 valid loss: 0.059484194964170456 best epoch: 1334 ======= epoch: 1336 train loss: 0.06352087947321527 valid loss: 0.059482564528783165 best epoch: 1336 ======= epoch: 1337 train loss: 0.06351410683221471 valid loss: 0.05948238323132197 best epoch: 1337 ======= epoch: 1338 train loss: 0.0635086131328142 valid loss: 0.05948092555627227 best epoch: 1338 ======= epoch: 1339 train loss: 0.0635010243140529 valid loss: 0.059482659213244915 best epoch: 1338 ======= epoch: 1340 train loss: 0.06349601667880277 valid loss: 0.05947801315536102 best epoch: 1340 ======= epoch: 1341 train loss: 0.06348892683623197 valid loss: 0.059484123873213925 best epoch: 1340 ======= epoch: 1342 train loss: 0.06348461488744726 valid loss: 0.0594773485014836 best epoch: 1342 ======= epoch: 1343 train loss: 0.0634771296294567 valid loss: 0.05948139183844129 best epoch: 1342 ======= epoch: 1344 train loss: 0.06347340663872016 valid loss: 0.059476591336230435 best epoch: 1344 ======= epoch: 1345 train loss: 0.06346606109636249 valid loss: 0.05947706134368976 best epoch: 1344 ======= epoch: 1346 train loss: 0.06346302056386147 valid loss: 0.059472321843107544 best epoch: 1346 ======= epoch: 1347 train loss: 0.06345533570964579 valid loss: 0.05947381071746349 best epoch: 1346 ======= epoch: 1348 train loss: 0.06345166939161258 valid loss: 0.05946919834241271 best epoch: 1348 ======= epoch: 1349 train loss: 0.06344551520054172 valid loss: 0.0594685492105782 best epoch: 1349 ======= epoch: 1350 train loss: 0.06344151227263573 valid loss: 0.059463908430188894 best epoch: 1350 ======= epoch: 1351 train loss: 0.06343470995578869 valid loss: 0.05946529982611537 best epoch: 1350 ======= epoch: 1352 train loss: 0.06343043120118019 valid loss: 0.05946058174595237 best epoch: 1352 ======= epoch: 1353 train loss: 0.06342342284933468 valid loss: 0.059461611323058605 best epoch: 1352 ======= epoch: 1354 train loss: 0.06341878423628247 valid loss: 0.05945706755543748 best epoch: 1354 ======= epoch: 1355 train loss: 0.0634121307240868 valid loss: 0.05945866756762067 best epoch: 1354 ======= epoch: 1356 train loss: 0.06340743769765084 valid loss: 0.05945628027742108 best epoch: 1356 ======= epoch: 1357 train loss: 0.06340027897745187 valid loss: 0.05945804622024298 best epoch: 1356 ======= epoch: 1358 train loss: 0.06339576680581142 valid loss: 0.05945398549859723 best epoch: 1358 ======= epoch: 1359 train loss: 0.06338822478888395 valid loss: 0.059456171933561563 best epoch: 1358 ======= epoch: 1360 train loss: 0.06338364610816409 valid loss: 0.059451339300721884 best epoch: 1360 ======= epoch: 1361 train loss: 0.0633762274109931 valid loss: 0.059452183078974485 best epoch: 1360 ======= epoch: 1362 train loss: 0.06337177518893171 valid loss: 0.05944601601610581 best epoch: 1362 ======= epoch: 1363 train loss: 0.06336462519557019 valid loss: 0.05944930606832107 best epoch: 1362 ======= epoch: 1364 train loss: 0.06336042984607226 valid loss: 0.05944274847085277 best epoch: 1364 ======= epoch: 1365 train loss: 0.06335353276427881 valid loss: 0.05944464448839426 best epoch: 1364 ======= epoch: 1366 train loss: 0.0633496208012932 valid loss: 0.05943945096805692 best epoch: 1366 ======= epoch: 1367 train loss: 0.06334292386187079 valid loss: 0.05944069754332304 best epoch: 1366 ======= epoch: 1368 train loss: 0.0633394317758948 valid loss: 0.05943484868233403 best epoch: 1368 ======= epoch: 1369 train loss: 0.0633326126779947 valid loss: 0.05943495221436024 best epoch: 1368 ======= epoch: 1370 train loss: 0.0633293920725492 valid loss: 0.05942889473711451 best epoch: 1370 ======= epoch: 1371 train loss: 0.0633222612057939 valid loss: 0.059429640881717205 best epoch: 1370 ======= epoch: 1372 train loss: 0.06331844692819832 valid loss: 0.05942610092461109 best epoch: 1372 ======= epoch: 1373 train loss: 0.06331123627814246 valid loss: 0.059428115064899124 best epoch: 1372 ======= epoch: 1374 train loss: 0.0633062876439021 valid loss: 0.05942488803217808 best epoch: 1374 ======= epoch: 1375 train loss: 0.06329940921155575 valid loss: 0.05942594834292928 best epoch: 1374 ======= epoch: 1376 train loss: 0.06329411028504923 valid loss: 0.05942359194159508 best epoch: 1376 ======= epoch: 1377 train loss: 0.0632875115196738 valid loss: 0.05942580212528507 best epoch: 1376 ======= epoch: 1378 train loss: 0.06328233128505541 valid loss: 0.059422875909755625 best epoch: 1378 ======= epoch: 1379 train loss: 0.06327568696535847 valid loss: 0.059423296401898064 best epoch: 1378 ======= epoch: 1380 train loss: 0.06327127376111386 valid loss: 0.05941901262849569 best epoch: 1380 ======= epoch: 1381 train loss: 0.06326403217620136 valid loss: 0.05942340319355329 best epoch: 1380 ======= epoch: 1382 train loss: 0.06325948591355557 valid loss: 0.059418433966735996 best epoch: 1382 ======= epoch: 1383 train loss: 0.0632520152117919 valid loss: 0.059419117867946625 best epoch: 1382 ======= epoch: 1384 train loss: 0.06324710023738904 valid loss: 0.05941483875115713 best epoch: 1384 ======= epoch: 1385 train loss: 0.06323951387341008 valid loss: 0.05941522757833203 best epoch: 1384 ======= epoch: 1386 train loss: 0.06323508879767707 valid loss: 0.05941101567198833 best epoch: 1386 ======= epoch: 1387 train loss: 0.06322817203738255 valid loss: 0.059413246189554535 best epoch: 1386 ======= epoch: 1388 train loss: 0.06322415980774863 valid loss: 0.05940827106436094 best epoch: 1388 ======= epoch: 1389 train loss: 0.0632175539539736 valid loss: 0.05941003328189254 best epoch: 1388 ======= epoch: 1390 train loss: 0.0632136674011471 valid loss: 0.05940484612559279 best epoch: 1390 ======= epoch: 1391 train loss: 0.06320682172606021 valid loss: 0.059408375062048435 best epoch: 1390 ======= epoch: 1392 train loss: 0.0632034014036626 valid loss: 0.05940225596229235 best epoch: 1392 ======= epoch: 1393 train loss: 0.06319620448199136 valid loss: 0.059405934531241655 best epoch: 1392 ======= epoch: 1394 train loss: 0.06319253119053664 valid loss: 0.059400986724843584 best epoch: 1394 ======= epoch: 1395 train loss: 0.06318556316725818 valid loss: 0.05940272690107425 best epoch: 1394 ======= epoch: 1396 train loss: 0.0631815450909881 valid loss: 0.059397015099724136 best epoch: 1396 ======= epoch: 1397 train loss: 0.06317441358035546 valid loss: 0.05939999362453818 best epoch: 1396 ======= epoch: 1398 train loss: 0.06316988887610259 valid loss: 0.059394525519261755 best epoch: 1398 ======= epoch: 1399 train loss: 0.06316319309795897 valid loss: 0.059396490920335054 best epoch: 1398 ======= epoch: 1400 train loss: 0.06315920807789137 valid loss: 0.05938961201657852 best epoch: 1400 ======= epoch: 1401 train loss: 0.06315262122999554 valid loss: 0.059392671566456556 best epoch: 1400 ======= epoch: 1402 train loss: 0.06314916466075329 valid loss: 0.05938895558938384 best epoch: 1402 ======= epoch: 1403 train loss: 0.06314215335971964 valid loss: 0.05939115475242337 best epoch: 1402 ======= epoch: 1404 train loss: 0.06313813401294159 valid loss: 0.05938474709788958 best epoch: 1404 ======= epoch: 1405 train loss: 0.0631306904345107 valid loss: 0.05938895558938384 best epoch: 1404 ======= epoch: 1406 train loss: 0.063126117600427 valid loss: 0.05938290664926171 best epoch: 1406 ======= epoch: 1407 train loss: 0.063118477443946 valid loss: 0.05938593220586578 best epoch: 1406 ======= epoch: 1408 train loss: 0.063113273397364 valid loss: 0.05938161785403887 best epoch: 1408 ======= epoch: 1409 train loss: 0.063106398506343 valid loss: 0.05938161304220557 best epoch: 1409 ======= epoch: 1410 train loss: 0.0631010735143022 valid loss: 0.0593754251797994 best epoch: 1410 ======= epoch: 1411 train loss: 0.06309462683620276 valid loss: 0.05937653717895349 best epoch: 1410 ======= epoch: 1412 train loss: 0.06309063170381167 valid loss: 0.05937063414603472 best epoch: 1412 ======= epoch: 1413 train loss: 0.06308359461319116 valid loss: 0.05937350650007526 best epoch: 1412 ======= epoch: 1414 train loss: 0.06308019066367436 valid loss: 0.05936677490050594 best epoch: 1414 ======= epoch: 1415 train loss: 0.06307267574708403 valid loss: 0.05936996157591542 best epoch: 1414 ======= epoch: 1416 train loss: 0.06306867115774456 valid loss: 0.059363070564965405 best epoch: 1416 ======= epoch: 1417 train loss: 0.06306114616332414 valid loss: 0.05936586298048496 best epoch: 1416 ======= epoch: 1418 train loss: 0.06305641824103615 valid loss: 0.0593596895535787 best epoch: 1418 ======= epoch: 1419 train loss: 0.06304921761132133 valid loss: 0.05936358806987604 best epoch: 1418 ======= epoch: 1420 train loss: 0.06304420415243064 valid loss: 0.059356359454492726 best epoch: 1420 ======= epoch: 1421 train loss: 0.06303746913801189 valid loss: 0.05935910949483514 best epoch: 1420 ======= epoch: 1422 train loss: 0.06303323914561375 valid loss: 0.05935204277435938 best epoch: 1422 ======= epoch: 1423 train loss: 0.06302632820321086 valid loss: 0.059356458174685635 best epoch: 1422 ======= epoch: 1424 train loss: 0.06302268893377465 valid loss: 0.059350437174240746 best epoch: 1424 ======= epoch: 1425 train loss: 0.06301543914517503 valid loss: 0.05935538746416569 best epoch: 1424 ======= epoch: 1426 train loss: 0.06301130201483582 valid loss: 0.05935123500724634 best epoch: 1424 ======= epoch: 1427 train loss: 0.06300394840101585 valid loss: 0.059354197389135756 best epoch: 1424 ======= epoch: 1428 train loss: 0.06299937690067807 valid loss: 0.059350865272184215 best epoch: 1424 ======= epoch: 1429 train loss: 0.06299302128523036 valid loss: 0.05935127489889661 best epoch: 1424 ======= epoch: 1430 train loss: 0.06298850590570106 valid loss: 0.05934304449086388 best epoch: 1430 ======= epoch: 1431 train loss: 0.06298210076343866 valid loss: 0.05934690761690339 best epoch: 1430 ======= epoch: 1432 train loss: 0.0629780492917807 valid loss: 0.05934043554589152 best epoch: 1432 ======= epoch: 1433 train loss: 0.06297088901937743 valid loss: 0.05934294033795595 best epoch: 1432 ======= epoch: 1434 train loss: 0.06296666811024886 valid loss: 0.05933561719333132 best epoch: 1434 ======= epoch: 1435 train loss: 0.06295935650367612 valid loss: 0.05933828807125489 best epoch: 1434 ======= epoch: 1436 train loss: 0.06295468953127663 valid loss: 0.059330907029410206 best epoch: 1436 ======= epoch: 1437 train loss: 0.06294749074696022 valid loss: 0.0593336073992153 best epoch: 1436 ======= epoch: 1438 train loss: 0.06294294357598748 valid loss: 0.0593258758696417 best epoch: 1438 ======= epoch: 1439 train loss: 0.06293673488529927 valid loss: 0.059329644311219454 best epoch: 1438 ======= epoch: 1440 train loss: 0.06293241826265498 valid loss: 0.059324221685528755 best epoch: 1440 ======= epoch: 1441 train loss: 0.06292696598411342 valid loss: 0.05932482766608397 best epoch: 1440 ======= epoch: 1442 train loss: 0.06292306972138675 valid loss: 0.059318295369545616 best epoch: 1442 ======= epoch: 1443 train loss: 0.06291686422134071 valid loss: 0.0593231525272131 best epoch: 1442 ======= epoch: 1444 train loss: 0.06291343823627189 valid loss: 0.05931652834018072 best epoch: 1444 ======= epoch: 1445 train loss: 0.06290662493901672 valid loss: 0.05931930554409822 best epoch: 1444 ======= epoch: 1446 train loss: 0.06290310017626595 valid loss: 0.05931197556977471 best epoch: 1446 ======= epoch: 1447 train loss: 0.06289611049111427 valid loss: 0.0593129430587093 best epoch: 1446 ======= epoch: 1448 train loss: 0.06289196261607202 valid loss: 0.05930339094872276 best epoch: 1448 ======= epoch: 1449 train loss: 0.0628849249103187 valid loss: 0.0593090100834767 best epoch: 1448 ======= epoch: 1450 train loss: 0.06288026199664598 valid loss: 0.05930380895733833 best epoch: 1448 ======= epoch: 1451 train loss: 0.06287443879301902 valid loss: 0.05930569193636378 best epoch: 1448 ======= epoch: 1452 train loss: 0.06287015708922236 valid loss: 0.05929915762195984 best epoch: 1452 ======= epoch: 1453 train loss: 0.06286391167453041 valid loss: 0.05930498708039522 best epoch: 1452 ======= epoch: 1454 train loss: 0.06285983871921529 valid loss: 0.05929741356521845 best epoch: 1454 ======= epoch: 1455 train loss: 0.06285321332491291 valid loss: 0.05930293848117193 best epoch: 1454 ======= epoch: 1456 train loss: 0.06284983049687227 valid loss: 0.05929687603687247 best epoch: 1456 ======= epoch: 1457 train loss: 0.06284291176470341 valid loss: 0.05930139760797223 best epoch: 1456 ======= epoch: 1458 train loss: 0.06283923877794066 valid loss: 0.059292987920343876 best epoch: 1458 ======= epoch: 1459 train loss: 0.06283225685381043 valid loss: 0.05929763397822777 best epoch: 1458 ======= epoch: 1460 train loss: 0.06282813476835504 valid loss: 0.05929046977932254 best epoch: 1460 ======= epoch: 1461 train loss: 0.06282084120184551 valid loss: 0.059295827988535166 best epoch: 1460 ======= epoch: 1462 train loss: 0.06281610178749686 valid loss: 0.05928964742148916 best epoch: 1462 ======= epoch: 1463 train loss: 0.06280977284696734 valid loss: 0.05929184518754482 best epoch: 1462 ======= epoch: 1464 train loss: 0.0628044473087806 valid loss: 0.05928663071244955 best epoch: 1464 ======= epoch: 1465 train loss: 0.06279783196931268 valid loss: 0.0592944483893613 best epoch: 1464 ======= epoch: 1466 train loss: 0.06279269231621314 valid loss: 0.059288800694048405 best epoch: 1464 ======= epoch: 1467 train loss: 0.06278571449105202 valid loss: 0.05929504272838434 best epoch: 1464 ======= epoch: 1468 train loss: 0.06278096552202363 valid loss: 0.0592866400256753 best epoch: 1464 ======= epoch: 1469 train loss: 0.0627752711341061 valid loss: 0.059291442235310875 best epoch: 1464 ======= epoch: 1470 train loss: 0.06277070773972405 valid loss: 0.05928329130013784 best epoch: 1470 ======= epoch: 1471 train loss: 0.0627648125310647 valid loss: 0.05928816506639123 best epoch: 1470 ======= epoch: 1472 train loss: 0.0627619728308033 valid loss: 0.05928053551663955 best epoch: 1472 ======= epoch: 1473 train loss: 0.06275509899758079 valid loss: 0.059285367062936224 best epoch: 1472 ======= epoch: 1474 train loss: 0.06275302233795325 valid loss: 0.05927776110668977 best epoch: 1474 ======= epoch: 1475 train loss: 0.06274616021524977 valid loss: 0.05928659159690142 best epoch: 1474 ======= epoch: 1476 train loss: 0.06274341614227052 valid loss: 0.059279422430942454 best epoch: 1474 ======= epoch: 1477 train loss: 0.062737277024828 valid loss: 0.05928483931347728 best epoch: 1474 ======= epoch: 1478 train loss: 0.06273402641009954 valid loss: 0.05927837438260516 best epoch: 1474 ======= epoch: 1479 train loss: 0.06272866448134552 valid loss: 0.0592839311187466 best epoch: 1474 ======= epoch: 1480 train loss: 0.06272489203278113 valid loss: 0.05927771019438902 best epoch: 1480 ======= epoch: 1481 train loss: 0.0627192222904184 valid loss: 0.059283449314534664 best epoch: 1480 ======= epoch: 1482 train loss: 0.06271516739816209 valid loss: 0.05927583295851946 best epoch: 1482 ======= epoch: 1483 train loss: 0.0627089308137879 valid loss: 0.05928048382823666 best epoch: 1482 ======= epoch: 1484 train loss: 0.06270414783998772 valid loss: 0.05927380220964551 best epoch: 1484 ======= epoch: 1485 train loss: 0.06269790985288075 valid loss: 0.05927358102053404 best epoch: 1485 ======= epoch: 1486 train loss: 0.06269281627725304 valid loss: 0.05926857749000192 best epoch: 1486 ======= epoch: 1487 train loss: 0.06268643430307691 valid loss: 0.0592758854230245 best epoch: 1486 ======= epoch: 1488 train loss: 0.06268283846661632 valid loss: 0.059267477287600435 best epoch: 1488 ======= epoch: 1489 train loss: 0.0626763284528329 valid loss: 0.059273255368073784 best epoch: 1488 ======= epoch: 1490 train loss: 0.06267350025038108 valid loss: 0.05926560595010718 best epoch: 1490 ======= epoch: 1491 train loss: 0.06266738194397387 valid loss: 0.05926908878609538 best epoch: 1490 ======= epoch: 1492 train loss: 0.06266484553868204 valid loss: 0.05926217030112942 best epoch: 1492 ======= epoch: 1493 train loss: 0.06265833521445777 valid loss: 0.059269494377076626 best epoch: 1492 ======= epoch: 1494 train loss: 0.06265566154831538 valid loss: 0.059261545383681856 best epoch: 1494 ======= epoch: 1495 train loss: 0.06264894555702254 valid loss: 0.05926712229847908 best epoch: 1494 ======= epoch: 1496 train loss: 0.06264554618363395 valid loss: 0.059258469846099615 best epoch: 1496 ======= epoch: 1497 train loss: 0.06263909897663528 valid loss: 0.059264397248625755 best epoch: 1496 ======= epoch: 1498 train loss: 0.06263487574494915 valid loss: 0.05925584708650907 best epoch: 1498 ======= epoch: 1499 train loss: 0.06262838622026237 valid loss: 0.05926056997850537 best epoch: 1498 =======
In [41]:
autoencoder.save('conv_only_autoencoder_rprop.pkl')
In [42]:
fig_loss = plt.figure()
ax_loss = fig_loss.add_subplot(1, 1, 1)
ax_loss.plot(train_result.train_losses, linewidth=2, label='Training')
ax_loss.plot(train_result.valid_losses, linewidth=2, label='Validation')
ax_loss.scatter(
(train_result.best_epoch,), (train_result.best_valid_loss,),
linewidth=3, s=200, marker='x', color='red', label='Best')
ax_loss.legend()
ax_loss.autoscale(tight=True)
ax_loss.grid();
ax_loss.set_ylim((0.0, 1.0))
Out[42]:
(0.0, 1.0)
In [43]:
first_segment = valid_data[0]
valid_data.plot_trace(first_segment)
In [44]:
recon = autoencoder.recon(first_segment[None, ...])
valid_data.plot_trace(recon[0])
an encoded segment
In [45]:
encoded_flat = autoencoder.encode(first_segment[None, ...])
encoded = encoded_flat.reshape(autoencoder.conv_specs[-1][0], -1)
# need to make plot_trace free standing
def plot_trace(segment, samp_rate, chan_names=None, scale_factor=0.75):
segment = segment.T
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
nsec = segment.shape[0] / samp_rate
nchan = segment.shape[1]
scale = th.max(th.abs(segment))
sep = -th.arange(segment.shape[1]) * scale_factor * scale
segment = segment + sep
time = th.linspace(0, nsec, segment.shape[0])
ax.plot(time, segment)
ax.set_xlabel(r'Time ($s$)')
ax.set_ylabel(r'Signal ($\mu V$) Across Channels')
ax.set_yticks(sep)
if chan_names:
ax.set_yticklabels([c for i, c in enumerate(chan_names)])
ax.autoscale(tight=True)
# samp rate assumes 2-second segments
plot_trace(encoded, samp_rate=encoded.shape[1] / 2.0)
the residuals
In [46]:
plt.imshow(encoded, aspect='auto', interpolation='none', cmap=plt.cm.hot);
plt.colorbar();
In [47]:
valid_data.plot_trace(first_segment - recon[0])
In [48]:
valid_data.plot_psd(first_segment)
In [49]:
valid_data.plot_psd(recon[0])
In [50]:
valid_data.plot_psd(first_segment - recon[0])
In [51]:
plt.plot(autoencoder.encoder.conv[0].conv.weight[0].detach().T);
In [52]:
plt.plot(autoencoder.encoder.conv[1].conv.weight[0].detach().T);
In [53]:
plt.plot(autoencoder.encoder.conv[2].conv.weight[0].detach().T);
P300 Classification¶
In [54]:
class P3Dataset(EEGDataset):
def __init__(self, subject, word, day=1, width=0.8, samp_rate=1024.0, autoencoder=None,
low_freq=0.5, high_freq=100.0, downsample_factor=4, clamp=60.0,
full_chan_names=chan_names_biosemi_32, select_chan_names=select_chan_names,
means=None, stdevs=None):
orig_samp_rate = samp_rate
new_samp_rate = samp_rate / downsample_factor
del samp_rate
data_mats = []
bandpass_filter = self._make_bandpass(orig_samp_rate, low_freq, high_freq)
notch_filter = self._make_notch(orig_samp_rate, 60.0)
filenames = sorted(glob(f'data/p300grid/s{subject:02}-d{day}-*-{word}.pkl'))
print(filenames)
assert len(filenames) == 1
filename = filenames[0]
with open(filename, mode='rb') as fh:
# FIXME: convert to py3 pickle
raw_data = th.as_tensor(pickle.load(fh, encoding='bytes'), dtype=th.float32)
raw_data, markers = raw_data[:, :-1], raw_data[:, -1]
data = self._prep(
data=raw_data,
full_chan_names=full_chan_names,
select_chan_names=select_chan_names,
bandpass_filter=bandpass_filter,
notch_filter=notch_filter,
clamp=clamp,
downsample_factor=downsample_factor)
markers = markers[::downsample_factor]
diffs = np.diff(np.abs(markers.numpy()))
starts = np.where(diffs > 0.0)[0] + 1
width_steps = int(width * new_samp_rate)
data = th.stack([data[start:start+width_steps] for start in starts])
self.labels = (markers[starts] > 0.0).type(th.int64)
#data = np.ascontiguousarray(segments)
assert len(select_chan_names) == data.shape[-1]
nchan = data.shape[-1]
self.means = data.reshape((-1, nchan)).mean(0) if means is None else means
assert self.means.shape == (nchan,)
data -= self.means
self.stdevs = data.reshape((-1, nchan)).std(0) if stdevs is None else stdevs
assert self.stdevs.shape == (nchan,)
data /= self.stdevs
# we switch to shape (nseg, nchan, nobs) here since pytorch
# convolution operators expect channel first
data = data.permute(0, 2, 1)
if autoencoder is not None:
data = autoencoder.encode(data)
super().__init__(data, new_samp_rate, select_chan_names)
def __getitem__(self, idx):
return self.data[idx], self.labels[idx]
In [55]:
p3_train_data = P3Dataset(
subject=1, word='colorado',
means=train_data.means, stdevs=train_data.stdevs,
autoencoder=autoencoder)
['data/p300grid/s01-d1-p300grid-train-colorado.pkl']
In [56]:
p3_train_data[:10]
Out[56]:
(tensor([[ 1.2761, 3.2872, 3.8582, ..., -4.5459, -4.6114, -4.9648],
[ 1.8314, 3.6358, 3.4345, ..., -4.1457, -5.0480, -5.9346],
[ 2.1586, 3.4201, 4.0658, ..., -4.7186, -4.2923, -5.2723],
...,
[ 1.7998, 3.2497, 3.9659, ..., -4.3860, -3.9816, -4.9794],
[ 1.6292, 4.1978, 3.9045, ..., -4.6903, -5.2618, -6.0108],
[ 2.2849, 3.8555, 3.5685, ..., -3.7679, -3.8736, -4.8487]]),
tensor([1, 0, 0, 0, 0, 0, 0, 0, 0, 1]))
In [57]:
p3_train_data[0][0].shape
Out[57]:
torch.Size([624])
In [58]:
p3_train_data[0][0].min()
Out[58]:
tensor(-10.1820)
In [59]:
p3_train_data[0][0].max()
Out[59]:
tensor(12.1692)
In [60]:
#p3_train_data[0][0].mean(1)
In [61]:
#p3_train_data[0][0].std(1)
In [62]:
#plt.bar(p3_train_data[0][0]);
In [63]:
len(p3_train_data)
Out[63]:
960
In [64]:
8*32*0.8
Out[64]:
204.8