In [0]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
In [0]:
!pip install tensorflow-gpu==2.0.0-alpha0
Collecting tensorflow-gpu==2.0.0-alpha0
  Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)
     |████████████████████████████████| 332.1MB 56kB/s 
Collecting tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 (from tensorflow-gpu==2.0.0-alpha0)
  Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)
     |████████████████████████████████| 419kB 49.7MB/s 
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)
Collecting tb-nightly<1.14.0a20190302,>=1.14.0a20190301 (from tensorflow-gpu==2.0.0-alpha0)
  Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)
     |████████████████████████████████| 3.0MB 53.8MB/s 
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.4)
Collecting google-pasta>=0.1.2 (from tensorflow-gpu==2.0.0-alpha0)
  Downloading https://files.pythonhosted.org/packages/d0/33/376510eb8d6246f3c30545f416b2263eee461e40940c2a4413c711bdf62d/google_pasta-0.1.7-py3-none-any.whl (52kB)
     |████████████████████████████████| 61kB 32.0MB/s 
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.12.0)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.4)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.8.0)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (41.0.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.4)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1.1)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)
Installing collected packages: tf-estimator-nightly, tb-nightly, google-pasta, tensorflow-gpu
Successfully installed google-pasta-0.1.7 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115
In [0]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-alpha0
GPU Enabled: True
In [0]:
def data_generator(f_paths, params):
  for f_path in f_paths:
    with open(f_path) as f:
      print('Reading', f_path)
      for line in f:
        line = line.rstrip()
        label, text = line.split('\t')
        text = text.split(' ')
        x = [params['word2idx'].get(w, len(word2idx)) for w in text]
        if len(x) >= params['max_len']:
          x = x[:params['max_len']]
        else:
          x += [0] * (params['max_len'] - len(x))
        y = int(label)
        yield x, y


def dataset(is_training, params):
  _shapes = ([params['max_len']], ())
  _types = (tf.int32, tf.int32)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.batch(params['batch_size'])
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.batch(params['batch_size'])
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    
    self.embedding = tf.Variable(np.load('../vocab/word.npy'),
                                 dtype=tf.float32,
                                 name='pretrained_embedding',
                                 trainable=False,)
    
    self.drop1 = tf.keras.layers.Dropout(params['dropout_rate'])
    self.drop2 = tf.keras.layers.Dropout(params['dropout_rate'])
    self.drop3 = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.rnn1 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(params['rnn_units'], return_sequences=True))
    self.rnn2 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(params['rnn_units'], return_sequences=True))
    self.rnn3 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(params['rnn_units'], return_sequences=True))
    
    self.drop_fc = tf.keras.layers.Dropout(params['dropout_rate'])
    self.fc = tf.keras.layers.Dense(2*params['rnn_units'], tf.nn.elu)
    
    self.out_linear = tf.keras.layers.Dense(2)
  
  
  def call(self, inputs, training=False):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    
    batch_sz = tf.shape(inputs)[0]
    rnn_units = 2*params['rnn_units']
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    
    x = tf.reshape(x, (batch_sz*10*10, 10, 300))
    x = self.drop1(x, training=training)
    x = self.rnn1(x)
    x = tf.reduce_max(x, 1)
    
    x = tf.reshape(x, (batch_sz*10, 10, rnn_units))
    x = self.drop2(x, training=training)
    x = self.rnn2(x)
    x = tf.reduce_max(x, 1)
    
    x = tf.reshape(x, (batch_sz, 10, rnn_units))
    x = self.drop3(x, training=training)
    x = self.rnn3(x)
    x = tf.reduce_max(x, 1)
    
    x = self.drop_fc(x, training=training)
    x = self.fc(x)
    
    x = self.out_linear(x)
    
    return x
In [0]:
params = {
  'vocab_path': '../vocab/word.txt',
  'train_paths': [
    '../data/train_bt_part1.txt',
    '../data/train_bt_part2.txt',
    '../data/train_bt_part3.txt',
    '../data/train_bt_part4.txt',
    '../data/train_bt_part5.txt',
    '../data/train_bt_part6.txt',
  ],
  'test_paths': [
    '../data/test.txt',
  ],
  'num_samples': 25000*2,
  'num_labels': 2,
  'batch_size': 64,
  'max_len': 1000,
  'rnn_units': 200,
  'dropout_rate': 0.2,
  'clip_norm': 10.,
  'num_patience': 3,
  'lr': 3e-4,
}
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [0]:
word2idx = {}
with open(params['vocab_path']) as f:
  for i, line in enumerate(f):
    line = line.rstrip()
    word2idx[line] = i
params['word2idx'] = word2idx
params['vocab_size'] = len(word2idx) + 1


model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.95)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for texts, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model(texts, training=True)
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
      loss = tf.reduce_mean(loss)
  
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1

  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for texts, labels in dataset(is_training=False, params=params):
    logits = model(texts, training=False)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))
  history_acc.append(acc)
  
  if acc > best_acc:
    best_acc = acc
    # you can save model here
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0604 05:08:04.144721 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c042d7f98>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.154609 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c042e6828>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.162103 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c042e6e80>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.166756 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c042a5668>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.169664 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c042a5d68>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.176014 139760335886208 tf_logging.py:161] <tensorflow.python.keras.layers.recurrent.UnifiedLSTM object at 0x7f1c0422e518>: Note that this layer is not optimized for performance. Please use tf.keras.layers.CuDNNLSTM for better performance on GPU.
W0604 05:08:04.736387 139760335886208 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:410: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('bidirectional/forward_unified_lstm/kernel:0', TensorShape([300, 800])),
 ('bidirectional/forward_unified_lstm/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional/forward_unified_lstm/bias:0', TensorShape([800])),
 ('bidirectional/backward_unified_lstm/kernel:0', TensorShape([300, 800])),
 ('bidirectional/backward_unified_lstm/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional/backward_unified_lstm/bias:0', TensorShape([800])),
 ('bidirectional_1/forward_unified_lstm_1/kernel:0', TensorShape([400, 800])),
 ('bidirectional_1/forward_unified_lstm_1/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_1/forward_unified_lstm_1/bias:0', TensorShape([800])),
 ('bidirectional_1/backward_unified_lstm_1/kernel:0', TensorShape([400, 800])),
 ('bidirectional_1/backward_unified_lstm_1/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_1/backward_unified_lstm_1/bias:0', TensorShape([800])),
 ('bidirectional_2/forward_unified_lstm_2/kernel:0', TensorShape([400, 800])),
 ('bidirectional_2/forward_unified_lstm_2/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_2/forward_unified_lstm_2/bias:0', TensorShape([800])),
 ('bidirectional_2/backward_unified_lstm_2/kernel:0', TensorShape([400, 800])),
 ('bidirectional_2/backward_unified_lstm_2/recurrent_kernel:0',
  TensorShape([200, 800])),
 ('bidirectional_2/backward_unified_lstm_2/bias:0', TensorShape([800])),
 ('dense/kernel:0', TensorShape([400, 400])),
 ('dense/bias:0', TensorShape([400])),
 ('dense_1/kernel:0', TensorShape([400, 2])),
 ('dense_1/bias:0', TensorShape([2]))]
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:08:27.043282 139760335886208 interactiveshell.py:2882] Step 0 | Loss: 0.6915 | Spent: 22.3 secs | LR: 0.000300
I0604 05:08:38.346872 139760335886208 interactiveshell.py:2882] Step 50 | Loss: 0.6155 | Spent: 11.3 secs | LR: 0.000299
I0604 05:08:49.525030 139760335886208 interactiveshell.py:2882] Step 100 | Loss: 0.6370 | Spent: 11.2 secs | LR: 0.000298
I0604 05:09:00.728225 139760335886208 interactiveshell.py:2882] Step 150 | Loss: 0.3955 | Spent: 11.2 secs | LR: 0.000298
I0604 05:09:11.964376 139760335886208 interactiveshell.py:2882] Step 200 | Loss: 0.4248 | Spent: 11.2 secs | LR: 0.000297
I0604 05:09:23.318131 139760335886208 interactiveshell.py:2882] Step 250 | Loss: 0.3979 | Spent: 11.4 secs | LR: 0.000296
I0604 05:09:34.908226 139760335886208 interactiveshell.py:2882] Step 300 | Loss: 0.2888 | Spent: 11.6 secs | LR: 0.000295
I0604 05:09:46.415040 139760335886208 interactiveshell.py:2882] Step 350 | Loss: 0.3135 | Spent: 11.5 secs | LR: 0.000295
I0604 05:09:57.880719 139760335886208 interactiveshell.py:2882] Step 400 | Loss: 0.3881 | Spent: 11.5 secs | LR: 0.000294
I0604 05:10:09.297485 139760335886208 interactiveshell.py:2882] Step 450 | Loss: 0.3650 | Spent: 11.4 secs | LR: 0.000293
I0604 05:10:20.668198 139760335886208 interactiveshell.py:2882] Step 500 | Loss: 0.2195 | Spent: 11.4 secs | LR: 0.000292
I0604 05:10:32.058044 139760335886208 interactiveshell.py:2882] Step 550 | Loss: 0.4970 | Spent: 11.4 secs | LR: 0.000292
I0604 05:10:43.393068 139760335886208 interactiveshell.py:2882] Step 600 | Loss: 0.2887 | Spent: 11.3 secs | LR: 0.000291
I0604 05:10:54.902850 139760335886208 interactiveshell.py:2882] Step 650 | Loss: 0.2948 | Spent: 11.5 secs | LR: 0.000290
I0604 05:11:06.416477 139760335886208 interactiveshell.py:2882] Step 700 | Loss: 0.2473 | Spent: 11.5 secs | LR: 0.000289
I0604 05:11:17.827234 139760335886208 interactiveshell.py:2882] Step 750 | Loss: 0.3707 | Spent: 11.4 secs | LR: 0.000289
Reading ../data/test.txt
I0604 05:12:52.996667 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.873
I0604 05:12:52.997944 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.873
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:13:11.716400 139760335886208 interactiveshell.py:2882] Step 800 | Loss: 0.2907 | Spent: 113.9 secs | LR: 0.000288
I0604 05:13:23.303611 139760335886208 interactiveshell.py:2882] Step 850 | Loss: 0.3658 | Spent: 11.6 secs | LR: 0.000287
I0604 05:13:34.861685 139760335886208 interactiveshell.py:2882] Step 900 | Loss: 0.4076 | Spent: 11.6 secs | LR: 0.000286
I0604 05:13:46.363067 139760335886208 interactiveshell.py:2882] Step 950 | Loss: 0.2683 | Spent: 11.5 secs | LR: 0.000286
I0604 05:13:57.822511 139760335886208 interactiveshell.py:2882] Step 1000 | Loss: 0.2831 | Spent: 11.5 secs | LR: 0.000285
I0604 05:14:09.203094 139760335886208 interactiveshell.py:2882] Step 1050 | Loss: 0.2960 | Spent: 11.4 secs | LR: 0.000284
I0604 05:14:20.558709 139760335886208 interactiveshell.py:2882] Step 1100 | Loss: 0.2493 | Spent: 11.4 secs | LR: 0.000284
I0604 05:14:31.975912 139760335886208 interactiveshell.py:2882] Step 1150 | Loss: 0.2207 | Spent: 11.4 secs | LR: 0.000283
I0604 05:14:43.376853 139760335886208 interactiveshell.py:2882] Step 1200 | Loss: 0.2316 | Spent: 11.4 secs | LR: 0.000282
I0604 05:14:54.878766 139760335886208 interactiveshell.py:2882] Step 1250 | Loss: 0.2261 | Spent: 11.5 secs | LR: 0.000281
I0604 05:15:06.393621 139760335886208 interactiveshell.py:2882] Step 1300 | Loss: 0.4215 | Spent: 11.5 secs | LR: 0.000281
I0604 05:15:17.790007 139760335886208 interactiveshell.py:2882] Step 1350 | Loss: 0.2899 | Spent: 11.4 secs | LR: 0.000280
I0604 05:15:29.204187 139760335886208 interactiveshell.py:2882] Step 1400 | Loss: 0.2799 | Spent: 11.4 secs | LR: 0.000279
I0604 05:15:40.583214 139760335886208 interactiveshell.py:2882] Step 1450 | Loss: 0.2024 | Spent: 11.4 secs | LR: 0.000278
I0604 05:15:52.015429 139760335886208 interactiveshell.py:2882] Step 1500 | Loss: 0.1664 | Spent: 11.4 secs | LR: 0.000278
I0604 05:16:03.436983 139760335886208 interactiveshell.py:2882] Step 1550 | Loss: 0.3086 | Spent: 11.4 secs | LR: 0.000277
Reading ../data/test.txt
I0604 05:17:33.978852 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.897
I0604 05:17:33.979910 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.897
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:17:56.692689 139760335886208 interactiveshell.py:2882] Step 1600 | Loss: 0.2670 | Spent: 113.3 secs | LR: 0.000276
I0604 05:18:08.165771 139760335886208 interactiveshell.py:2882] Step 1650 | Loss: 0.3470 | Spent: 11.5 secs | LR: 0.000276
I0604 05:18:19.737285 139760335886208 interactiveshell.py:2882] Step 1700 | Loss: 0.1912 | Spent: 11.6 secs | LR: 0.000275
I0604 05:18:31.476118 139760335886208 interactiveshell.py:2882] Step 1750 | Loss: 0.2250 | Spent: 11.7 secs | LR: 0.000274
I0604 05:18:42.886483 139760335886208 interactiveshell.py:2882] Step 1800 | Loss: 0.2474 | Spent: 11.4 secs | LR: 0.000274
I0604 05:18:54.284980 139760335886208 interactiveshell.py:2882] Step 1850 | Loss: 0.2597 | Spent: 11.4 secs | LR: 0.000273
I0604 05:19:05.790857 139760335886208 interactiveshell.py:2882] Step 1900 | Loss: 0.3855 | Spent: 11.5 secs | LR: 0.000272
I0604 05:19:17.184355 139760335886208 interactiveshell.py:2882] Step 1950 | Loss: 0.1796 | Spent: 11.4 secs | LR: 0.000271
I0604 05:19:28.589839 139760335886208 interactiveshell.py:2882] Step 2000 | Loss: 0.2500 | Spent: 11.4 secs | LR: 0.000271
I0604 05:19:40.125723 139760335886208 interactiveshell.py:2882] Step 2050 | Loss: 0.3211 | Spent: 11.5 secs | LR: 0.000270
I0604 05:19:51.559666 139760335886208 interactiveshell.py:2882] Step 2100 | Loss: 0.1893 | Spent: 11.4 secs | LR: 0.000269
I0604 05:20:03.030630 139760335886208 interactiveshell.py:2882] Step 2150 | Loss: 0.2859 | Spent: 11.5 secs | LR: 0.000269
I0604 05:20:14.509434 139760335886208 interactiveshell.py:2882] Step 2200 | Loss: 0.1851 | Spent: 11.5 secs | LR: 0.000268
I0604 05:20:25.885906 139760335886208 interactiveshell.py:2882] Step 2250 | Loss: 0.2557 | Spent: 11.4 secs | LR: 0.000267
I0604 05:20:37.326089 139760335886208 interactiveshell.py:2882] Step 2300 | Loss: 0.2559 | Spent: 11.4 secs | LR: 0.000267
Reading ../data/test.txt
I0604 05:22:15.264368 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.903
I0604 05:22:15.265667 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.903
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:22:30.757441 139760335886208 interactiveshell.py:2882] Step 2350 | Loss: 0.2147 | Spent: 113.4 secs | LR: 0.000266
I0604 05:22:42.207552 139760335886208 interactiveshell.py:2882] Step 2400 | Loss: 0.2479 | Spent: 11.4 secs | LR: 0.000265
I0604 05:22:53.805418 139760335886208 interactiveshell.py:2882] Step 2450 | Loss: 0.1796 | Spent: 11.6 secs | LR: 0.000265
I0604 05:23:05.369930 139760335886208 interactiveshell.py:2882] Step 2500 | Loss: 0.2235 | Spent: 11.6 secs | LR: 0.000264
I0604 05:23:16.918413 139760335886208 interactiveshell.py:2882] Step 2550 | Loss: 0.2357 | Spent: 11.5 secs | LR: 0.000263
I0604 05:23:28.293157 139760335886208 interactiveshell.py:2882] Step 2600 | Loss: 0.2977 | Spent: 11.4 secs | LR: 0.000263
I0604 05:23:40.053388 139760335886208 interactiveshell.py:2882] Step 2650 | Loss: 0.2057 | Spent: 11.8 secs | LR: 0.000262
I0604 05:23:51.439643 139760335886208 interactiveshell.py:2882] Step 2700 | Loss: 0.2181 | Spent: 11.4 secs | LR: 0.000261
I0604 05:24:02.881897 139760335886208 interactiveshell.py:2882] Step 2750 | Loss: 0.1900 | Spent: 11.4 secs | LR: 0.000261
I0604 05:24:14.400581 139760335886208 interactiveshell.py:2882] Step 2800 | Loss: 0.2725 | Spent: 11.5 secs | LR: 0.000260
I0604 05:24:25.891650 139760335886208 interactiveshell.py:2882] Step 2850 | Loss: 0.2193 | Spent: 11.5 secs | LR: 0.000259
I0604 05:24:37.361024 139760335886208 interactiveshell.py:2882] Step 2900 | Loss: 0.3091 | Spent: 11.5 secs | LR: 0.000259
I0604 05:24:48.867970 139760335886208 interactiveshell.py:2882] Step 2950 | Loss: 0.2555 | Spent: 11.5 secs | LR: 0.000258
I0604 05:25:00.309714 139760335886208 interactiveshell.py:2882] Step 3000 | Loss: 0.2786 | Spent: 11.4 secs | LR: 0.000257
I0604 05:25:11.722701 139760335886208 interactiveshell.py:2882] Step 3050 | Loss: 0.1207 | Spent: 11.4 secs | LR: 0.000257
I0604 05:25:23.134300 139760335886208 interactiveshell.py:2882] Step 3100 | Loss: 0.2085 | Spent: 11.4 secs | LR: 0.000256
Reading ../data/test.txt
I0604 05:26:57.167933 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.898
I0604 05:26:57.169248 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.903
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:27:16.857199 139760335886208 interactiveshell.py:2882] Step 3150 | Loss: 0.2144 | Spent: 113.7 secs | LR: 0.000255
I0604 05:27:28.397136 139760335886208 interactiveshell.py:2882] Step 3200 | Loss: 0.1385 | Spent: 11.5 secs | LR: 0.000255
I0604 05:27:39.957666 139760335886208 interactiveshell.py:2882] Step 3250 | Loss: 0.1346 | Spent: 11.6 secs | LR: 0.000254
I0604 05:27:51.470806 139760335886208 interactiveshell.py:2882] Step 3300 | Loss: 0.1309 | Spent: 11.5 secs | LR: 0.000253
I0604 05:28:02.929177 139760335886208 interactiveshell.py:2882] Step 3350 | Loss: 0.1309 | Spent: 11.5 secs | LR: 0.000253
I0604 05:28:14.361644 139760335886208 interactiveshell.py:2882] Step 3400 | Loss: 0.1936 | Spent: 11.4 secs | LR: 0.000252
I0604 05:28:25.735171 139760335886208 interactiveshell.py:2882] Step 3450 | Loss: 0.2198 | Spent: 11.4 secs | LR: 0.000251
I0604 05:28:37.098662 139760335886208 interactiveshell.py:2882] Step 3500 | Loss: 0.1045 | Spent: 11.4 secs | LR: 0.000251
I0604 05:28:48.691666 139760335886208 interactiveshell.py:2882] Step 3550 | Loss: 0.2727 | Spent: 11.6 secs | LR: 0.000250
I0604 05:29:00.119660 139760335886208 interactiveshell.py:2882] Step 3600 | Loss: 0.1479 | Spent: 11.4 secs | LR: 0.000249
I0604 05:29:11.580726 139760335886208 interactiveshell.py:2882] Step 3650 | Loss: 0.2768 | Spent: 11.5 secs | LR: 0.000249
I0604 05:29:23.022918 139760335886208 interactiveshell.py:2882] Step 3700 | Loss: 0.2691 | Spent: 11.4 secs | LR: 0.000248
I0604 05:29:34.414606 139760335886208 interactiveshell.py:2882] Step 3750 | Loss: 0.1624 | Spent: 11.4 secs | LR: 0.000248
I0604 05:29:45.894632 139760335886208 interactiveshell.py:2882] Step 3800 | Loss: 0.2367 | Spent: 11.5 secs | LR: 0.000247
I0604 05:29:57.466183 139760335886208 interactiveshell.py:2882] Step 3850 | Loss: 0.3080 | Spent: 11.6 secs | LR: 0.000246
I0604 05:30:08.888422 139760335886208 interactiveshell.py:2882] Step 3900 | Loss: 0.2706 | Spent: 11.4 secs | LR: 0.000246
Reading ../data/test.txt
I0604 05:31:38.541024 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.911
I0604 05:31:38.542165 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.911
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:32:02.130124 139760335886208 interactiveshell.py:2882] Step 3950 | Loss: 0.1420 | Spent: 113.2 secs | LR: 0.000245
I0604 05:32:13.637567 139760335886208 interactiveshell.py:2882] Step 4000 | Loss: 0.1471 | Spent: 11.5 secs | LR: 0.000244
I0604 05:32:25.262587 139760335886208 interactiveshell.py:2882] Step 4050 | Loss: 0.2207 | Spent: 11.6 secs | LR: 0.000244
I0604 05:32:36.761795 139760335886208 interactiveshell.py:2882] Step 4100 | Loss: 0.1562 | Spent: 11.5 secs | LR: 0.000243
I0604 05:32:48.216577 139760335886208 interactiveshell.py:2882] Step 4150 | Loss: 0.2409 | Spent: 11.5 secs | LR: 0.000242
I0604 05:32:59.612482 139760335886208 interactiveshell.py:2882] Step 4200 | Loss: 0.1687 | Spent: 11.4 secs | LR: 0.000242
I0604 05:33:11.033323 139760335886208 interactiveshell.py:2882] Step 4250 | Loss: 0.1355 | Spent: 11.4 secs | LR: 0.000241
I0604 05:33:22.402960 139760335886208 interactiveshell.py:2882] Step 4300 | Loss: 0.1620 | Spent: 11.4 secs | LR: 0.000241
I0604 05:33:33.863770 139760335886208 interactiveshell.py:2882] Step 4350 | Loss: 0.2737 | Spent: 11.5 secs | LR: 0.000240
I0604 05:33:45.393763 139760335886208 interactiveshell.py:2882] Step 4400 | Loss: 0.1929 | Spent: 11.5 secs | LR: 0.000239
I0604 05:33:57.009071 139760335886208 interactiveshell.py:2882] Step 4450 | Loss: 0.3249 | Spent: 11.6 secs | LR: 0.000239
I0604 05:34:08.668536 139760335886208 interactiveshell.py:2882] Step 4500 | Loss: 0.1776 | Spent: 11.7 secs | LR: 0.000238
I0604 05:34:20.086093 139760335886208 interactiveshell.py:2882] Step 4550 | Loss: 0.1540 | Spent: 11.4 secs | LR: 0.000238
I0604 05:34:31.506853 139760335886208 interactiveshell.py:2882] Step 4600 | Loss: 0.3185 | Spent: 11.4 secs | LR: 0.000237
I0604 05:34:42.877658 139760335886208 interactiveshell.py:2882] Step 4650 | Loss: 0.3172 | Spent: 11.4 secs | LR: 0.000236
Reading ../data/test.txt
I0604 05:36:20.334700 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.911
I0604 05:36:20.335933 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.911
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:36:36.866846 139760335886208 interactiveshell.py:2882] Step 4700 | Loss: 0.0979 | Spent: 114.0 secs | LR: 0.000236
I0604 05:36:48.246677 139760335886208 interactiveshell.py:2882] Step 4750 | Loss: 0.1233 | Spent: 11.4 secs | LR: 0.000235
I0604 05:36:59.739756 139760335886208 interactiveshell.py:2882] Step 4800 | Loss: 0.0942 | Spent: 11.5 secs | LR: 0.000235
I0604 05:37:11.272686 139760335886208 interactiveshell.py:2882] Step 4850 | Loss: 0.1288 | Spent: 11.5 secs | LR: 0.000234
I0604 05:37:22.768908 139760335886208 interactiveshell.py:2882] Step 4900 | Loss: 0.1853 | Spent: 11.5 secs | LR: 0.000233
I0604 05:37:34.204087 139760335886208 interactiveshell.py:2882] Step 4950 | Loss: 0.1321 | Spent: 11.4 secs | LR: 0.000233
I0604 05:37:45.672522 139760335886208 interactiveshell.py:2882] Step 5000 | Loss: 0.2014 | Spent: 11.5 secs | LR: 0.000232
I0604 05:37:57.020244 139760335886208 interactiveshell.py:2882] Step 5050 | Loss: 0.0498 | Spent: 11.3 secs | LR: 0.000232
I0604 05:38:08.387158 139760335886208 interactiveshell.py:2882] Step 5100 | Loss: 0.2427 | Spent: 11.4 secs | LR: 0.000231
I0604 05:38:19.906675 139760335886208 interactiveshell.py:2882] Step 5150 | Loss: 0.1184 | Spent: 11.5 secs | LR: 0.000230
I0604 05:38:31.415204 139760335886208 interactiveshell.py:2882] Step 5200 | Loss: 0.1480 | Spent: 11.5 secs | LR: 0.000230
I0604 05:38:42.864474 139760335886208 interactiveshell.py:2882] Step 5250 | Loss: 0.1828 | Spent: 11.4 secs | LR: 0.000229
I0604 05:38:54.255676 139760335886208 interactiveshell.py:2882] Step 5300 | Loss: 0.1904 | Spent: 11.4 secs | LR: 0.000229
I0604 05:39:05.836040 139760335886208 interactiveshell.py:2882] Step 5350 | Loss: 0.2021 | Spent: 11.6 secs | LR: 0.000228
I0604 05:39:17.377594 139760335886208 interactiveshell.py:2882] Step 5400 | Loss: 0.1726 | Spent: 11.5 secs | LR: 0.000227
I0604 05:39:28.821479 139760335886208 interactiveshell.py:2882] Step 5450 | Loss: 0.0950 | Spent: 11.4 secs | LR: 0.000227
Reading ../data/test.txt
I0604 05:41:02.064751 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.916
I0604 05:41:02.065931 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.916
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:41:22.762428 139760335886208 interactiveshell.py:2882] Step 5500 | Loss: 0.1017 | Spent: 113.9 secs | LR: 0.000226
I0604 05:41:34.266127 139760335886208 interactiveshell.py:2882] Step 5550 | Loss: 0.2113 | Spent: 11.5 secs | LR: 0.000226
I0604 05:41:45.890175 139760335886208 interactiveshell.py:2882] Step 5600 | Loss: 0.1718 | Spent: 11.6 secs | LR: 0.000225
I0604 05:41:57.476666 139760335886208 interactiveshell.py:2882] Step 5650 | Loss: 0.1757 | Spent: 11.6 secs | LR: 0.000225
I0604 05:42:08.956524 139760335886208 interactiveshell.py:2882] Step 5700 | Loss: 0.1019 | Spent: 11.5 secs | LR: 0.000224
I0604 05:42:20.362220 139760335886208 interactiveshell.py:2882] Step 5750 | Loss: 0.2889 | Spent: 11.4 secs | LR: 0.000223
I0604 05:42:31.895283 139760335886208 interactiveshell.py:2882] Step 5800 | Loss: 0.1807 | Spent: 11.5 secs | LR: 0.000223
I0604 05:42:43.352706 139760335886208 interactiveshell.py:2882] Step 5850 | Loss: 0.0945 | Spent: 11.5 secs | LR: 0.000222
I0604 05:42:54.847708 139760335886208 interactiveshell.py:2882] Step 5900 | Loss: 0.3550 | Spent: 11.5 secs | LR: 0.000222
I0604 05:43:06.445735 139760335886208 interactiveshell.py:2882] Step 5950 | Loss: 0.1641 | Spent: 11.6 secs | LR: 0.000221
I0604 05:43:17.982021 139760335886208 interactiveshell.py:2882] Step 6000 | Loss: 0.1136 | Spent: 11.5 secs | LR: 0.000221
I0604 05:43:29.472474 139760335886208 interactiveshell.py:2882] Step 6050 | Loss: 0.1521 | Spent: 11.5 secs | LR: 0.000220
I0604 05:43:40.963078 139760335886208 interactiveshell.py:2882] Step 6100 | Loss: 0.1175 | Spent: 11.5 secs | LR: 0.000219
I0604 05:43:52.469952 139760335886208 interactiveshell.py:2882] Step 6150 | Loss: 0.0637 | Spent: 11.5 secs | LR: 0.000219
I0604 05:44:04.038683 139760335886208 interactiveshell.py:2882] Step 6200 | Loss: 0.2009 | Spent: 11.6 secs | LR: 0.000218
I0604 05:44:15.711004 139760335886208 interactiveshell.py:2882] Step 6250 | Loss: 0.1996 | Spent: 11.7 secs | LR: 0.000218
Reading ../data/test.txt
I0604 05:45:45.351789 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.914
I0604 05:45:45.352833 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.916
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:46:10.037213 139760335886208 interactiveshell.py:2882] Step 6300 | Loss: 0.0787 | Spent: 114.3 secs | LR: 0.000217
I0604 05:46:21.581485 139760335886208 interactiveshell.py:2882] Step 6350 | Loss: 0.2469 | Spent: 11.5 secs | LR: 0.000217
I0604 05:46:33.209584 139760335886208 interactiveshell.py:2882] Step 6400 | Loss: 0.1884 | Spent: 11.6 secs | LR: 0.000216
I0604 05:46:44.764897 139760335886208 interactiveshell.py:2882] Step 6450 | Loss: 0.1692 | Spent: 11.6 secs | LR: 0.000215
I0604 05:46:56.310832 139760335886208 interactiveshell.py:2882] Step 6500 | Loss: 0.1277 | Spent: 11.5 secs | LR: 0.000215
I0604 05:47:07.790755 139760335886208 interactiveshell.py:2882] Step 6550 | Loss: 0.1997 | Spent: 11.5 secs | LR: 0.000214
I0604 05:47:19.238046 139760335886208 interactiveshell.py:2882] Step 6600 | Loss: 0.0806 | Spent: 11.4 secs | LR: 0.000214
I0604 05:47:30.719805 139760335886208 interactiveshell.py:2882] Step 6650 | Loss: 0.0886 | Spent: 11.5 secs | LR: 0.000213
I0604 05:47:42.224004 139760335886208 interactiveshell.py:2882] Step 6700 | Loss: 0.2933 | Spent: 11.5 secs | LR: 0.000213
I0604 05:47:53.752958 139760335886208 interactiveshell.py:2882] Step 6750 | Loss: 0.0890 | Spent: 11.5 secs | LR: 0.000212
I0604 05:48:05.236981 139760335886208 interactiveshell.py:2882] Step 6800 | Loss: 0.0583 | Spent: 11.5 secs | LR: 0.000212
I0604 05:48:16.704712 139760335886208 interactiveshell.py:2882] Step 6850 | Loss: 0.0995 | Spent: 11.5 secs | LR: 0.000211
I0604 05:48:28.220817 139760335886208 interactiveshell.py:2882] Step 6900 | Loss: 0.1410 | Spent: 11.5 secs | LR: 0.000211
I0604 05:48:39.731920 139760335886208 interactiveshell.py:2882] Step 6950 | Loss: 0.2293 | Spent: 11.5 secs | LR: 0.000210
I0604 05:48:51.188437 139760335886208 interactiveshell.py:2882] Step 7000 | Loss: 0.1415 | Spent: 11.5 secs | LR: 0.000210
Reading ../data/test.txt
I0604 05:50:27.824304 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.907
I0604 05:50:27.825577 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.916
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:50:45.147420 139760335886208 interactiveshell.py:2882] Step 7050 | Loss: 0.1476 | Spent: 114.0 secs | LR: 0.000209
I0604 05:50:56.563912 139760335886208 interactiveshell.py:2882] Step 7100 | Loss: 0.0780 | Spent: 11.4 secs | LR: 0.000208
I0604 05:51:08.227834 139760335886208 interactiveshell.py:2882] Step 7150 | Loss: 0.1261 | Spent: 11.7 secs | LR: 0.000208
I0604 05:51:19.913246 139760335886208 interactiveshell.py:2882] Step 7200 | Loss: 0.1807 | Spent: 11.7 secs | LR: 0.000207
I0604 05:51:31.415824 139760335886208 interactiveshell.py:2882] Step 7250 | Loss: 0.1752 | Spent: 11.5 secs | LR: 0.000207
I0604 05:51:42.902441 139760335886208 interactiveshell.py:2882] Step 7300 | Loss: 0.2218 | Spent: 11.5 secs | LR: 0.000206
I0604 05:51:54.330959 139760335886208 interactiveshell.py:2882] Step 7350 | Loss: 0.2602 | Spent: 11.4 secs | LR: 0.000206
I0604 05:52:05.749526 139760335886208 interactiveshell.py:2882] Step 7400 | Loss: 0.1229 | Spent: 11.4 secs | LR: 0.000205
I0604 05:52:17.170059 139760335886208 interactiveshell.py:2882] Step 7450 | Loss: 0.0952 | Spent: 11.4 secs | LR: 0.000205
I0604 05:52:28.728710 139760335886208 interactiveshell.py:2882] Step 7500 | Loss: 0.1807 | Spent: 11.6 secs | LR: 0.000204
I0604 05:52:40.263424 139760335886208 interactiveshell.py:2882] Step 7550 | Loss: 0.1268 | Spent: 11.5 secs | LR: 0.000204
I0604 05:52:51.744956 139760335886208 interactiveshell.py:2882] Step 7600 | Loss: 0.1236 | Spent: 11.5 secs | LR: 0.000203
I0604 05:53:03.255841 139760335886208 interactiveshell.py:2882] Step 7650 | Loss: 0.2078 | Spent: 11.5 secs | LR: 0.000203
I0604 05:53:14.738575 139760335886208 interactiveshell.py:2882] Step 7700 | Loss: 0.1527 | Spent: 11.5 secs | LR: 0.000202
I0604 05:53:26.199120 139760335886208 interactiveshell.py:2882] Step 7750 | Loss: 0.0614 | Spent: 11.5 secs | LR: 0.000202
I0604 05:53:37.673645 139760335886208 interactiveshell.py:2882] Step 7800 | Loss: 0.1161 | Spent: 11.5 secs | LR: 0.000201
Reading ../data/test.txt
I0604 05:55:10.561406 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.916
I0604 05:55:10.562538 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.916
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 05:55:32.148681 139760335886208 interactiveshell.py:2882] Step 7850 | Loss: 0.1708 | Spent: 114.5 secs | LR: 0.000201
I0604 05:55:43.863220 139760335886208 interactiveshell.py:2882] Step 7900 | Loss: 0.1640 | Spent: 11.7 secs | LR: 0.000200
I0604 05:55:55.508426 139760335886208 interactiveshell.py:2882] Step 7950 | Loss: 0.2501 | Spent: 11.6 secs | LR: 0.000200
I0604 05:56:07.025819 139760335886208 interactiveshell.py:2882] Step 8000 | Loss: 0.1083 | Spent: 11.5 secs | LR: 0.000199
I0604 05:56:18.542153 139760335886208 interactiveshell.py:2882] Step 8050 | Loss: 0.0755 | Spent: 11.5 secs | LR: 0.000199
I0604 05:56:30.060383 139760335886208 interactiveshell.py:2882] Step 8100 | Loss: 0.0914 | Spent: 11.5 secs | LR: 0.000198
I0604 05:56:41.507568 139760335886208 interactiveshell.py:2882] Step 8150 | Loss: 0.2969 | Spent: 11.4 secs | LR: 0.000198
I0604 05:56:52.992695 139760335886208 interactiveshell.py:2882] Step 8200 | Loss: 0.0599 | Spent: 11.5 secs | LR: 0.000197
I0604 05:57:04.444954 139760335886208 interactiveshell.py:2882] Step 8250 | Loss: 0.1861 | Spent: 11.5 secs | LR: 0.000196
I0604 05:57:15.845170 139760335886208 interactiveshell.py:2882] Step 8300 | Loss: 0.1379 | Spent: 11.4 secs | LR: 0.000196
I0604 05:57:27.311816 139760335886208 interactiveshell.py:2882] Step 8350 | Loss: 0.1109 | Spent: 11.5 secs | LR: 0.000195
I0604 05:57:38.743787 139760335886208 interactiveshell.py:2882] Step 8400 | Loss: 0.2401 | Spent: 11.4 secs | LR: 0.000195
I0604 05:57:50.279259 139760335886208 interactiveshell.py:2882] Step 8450 | Loss: 0.2631 | Spent: 11.5 secs | LR: 0.000194
I0604 05:58:01.758955 139760335886208 interactiveshell.py:2882] Step 8500 | Loss: 0.1924 | Spent: 11.5 secs | LR: 0.000194
I0604 05:58:13.221278 139760335886208 interactiveshell.py:2882] Step 8550 | Loss: 0.0966 | Spent: 11.5 secs | LR: 0.000193
I0604 05:58:24.761089 139760335886208 interactiveshell.py:2882] Step 8600 | Loss: 0.1186 | Spent: 11.5 secs | LR: 0.000193
Reading ../data/test.txt
I0604 05:59:53.156516 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.917
I0604 05:59:53.157708 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.917
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 06:00:18.677695 139760335886208 interactiveshell.py:2882] Step 8650 | Loss: 0.0504 | Spent: 113.9 secs | LR: 0.000192
I0604 06:00:30.371270 139760335886208 interactiveshell.py:2882] Step 8700 | Loss: 0.1217 | Spent: 11.7 secs | LR: 0.000192
I0604 06:00:41.999482 139760335886208 interactiveshell.py:2882] Step 8750 | Loss: 0.0645 | Spent: 11.6 secs | LR: 0.000192
I0604 06:00:53.549116 139760335886208 interactiveshell.py:2882] Step 8800 | Loss: 0.1509 | Spent: 11.5 secs | LR: 0.000191
I0604 06:01:05.053719 139760335886208 interactiveshell.py:2882] Step 8850 | Loss: 0.1946 | Spent: 11.5 secs | LR: 0.000191
I0604 06:01:16.560001 139760335886208 interactiveshell.py:2882] Step 8900 | Loss: 0.1837 | Spent: 11.5 secs | LR: 0.000190
I0604 06:01:27.975829 139760335886208 interactiveshell.py:2882] Step 8950 | Loss: 0.1321 | Spent: 11.4 secs | LR: 0.000190
I0604 06:01:39.422101 139760335886208 interactiveshell.py:2882] Step 9000 | Loss: 0.0732 | Spent: 11.4 secs | LR: 0.000189
I0604 06:01:51.045317 139760335886208 interactiveshell.py:2882] Step 9050 | Loss: 0.1379 | Spent: 11.6 secs | LR: 0.000189
I0604 06:02:02.592805 139760335886208 interactiveshell.py:2882] Step 9100 | Loss: 0.0611 | Spent: 11.5 secs | LR: 0.000188
I0604 06:02:14.139413 139760335886208 interactiveshell.py:2882] Step 9150 | Loss: 0.2390 | Spent: 11.5 secs | LR: 0.000188
I0604 06:02:25.682177 139760335886208 interactiveshell.py:2882] Step 9200 | Loss: 0.1008 | Spent: 11.5 secs | LR: 0.000187
I0604 06:02:37.215114 139760335886208 interactiveshell.py:2882] Step 9250 | Loss: 0.1177 | Spent: 11.5 secs | LR: 0.000187
I0604 06:02:48.824466 139760335886208 interactiveshell.py:2882] Step 9300 | Loss: 0.1556 | Spent: 11.6 secs | LR: 0.000186
I0604 06:03:00.265330 139760335886208 interactiveshell.py:2882] Step 9350 | Loss: 0.1206 | Spent: 11.4 secs | LR: 0.000186
Reading ../data/test.txt
I0604 06:04:36.046642 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.916
I0604 06:04:36.047896 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.917
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 06:04:54.419764 139760335886208 interactiveshell.py:2882] Step 9400 | Loss: 0.0893 | Spent: 114.2 secs | LR: 0.000185
I0604 06:05:06.220837 139760335886208 interactiveshell.py:2882] Step 9450 | Loss: 0.0176 | Spent: 11.8 secs | LR: 0.000185
I0604 06:05:17.796020 139760335886208 interactiveshell.py:2882] Step 9500 | Loss: 0.1154 | Spent: 11.6 secs | LR: 0.000184
I0604 06:05:29.533416 139760335886208 interactiveshell.py:2882] Step 9550 | Loss: 0.1472 | Spent: 11.7 secs | LR: 0.000184
I0604 06:05:41.059415 139760335886208 interactiveshell.py:2882] Step 9600 | Loss: 0.0339 | Spent: 11.5 secs | LR: 0.000183
I0604 06:05:52.629210 139760335886208 interactiveshell.py:2882] Step 9650 | Loss: 0.0836 | Spent: 11.6 secs | LR: 0.000183
I0604 06:06:04.041951 139760335886208 interactiveshell.py:2882] Step 9700 | Loss: 0.0936 | Spent: 11.4 secs | LR: 0.000182
I0604 06:06:15.414686 139760335886208 interactiveshell.py:2882] Step 9750 | Loss: 0.1640 | Spent: 11.4 secs | LR: 0.000182
I0604 06:06:26.866932 139760335886208 interactiveshell.py:2882] Step 9800 | Loss: 0.1053 | Spent: 11.5 secs | LR: 0.000181
I0604 06:06:38.304646 139760335886208 interactiveshell.py:2882] Step 9850 | Loss: 0.1479 | Spent: 11.4 secs | LR: 0.000181
I0604 06:06:49.757967 139760335886208 interactiveshell.py:2882] Step 9900 | Loss: 0.0698 | Spent: 11.5 secs | LR: 0.000181
I0604 06:07:01.188644 139760335886208 interactiveshell.py:2882] Step 9950 | Loss: 0.0606 | Spent: 11.4 secs | LR: 0.000180
I0604 06:07:12.768653 139760335886208 interactiveshell.py:2882] Step 10000 | Loss: 0.0401 | Spent: 11.6 secs | LR: 0.000180
I0604 06:07:24.161149 139760335886208 interactiveshell.py:2882] Step 10050 | Loss: 0.0395 | Spent: 11.4 secs | LR: 0.000179
I0604 06:07:35.568012 139760335886208 interactiveshell.py:2882] Step 10100 | Loss: 0.0631 | Spent: 11.4 secs | LR: 0.000179
I0604 06:07:47.028149 139760335886208 interactiveshell.py:2882] Step 10150 | Loss: 0.1839 | Spent: 11.5 secs | LR: 0.000178
Reading ../data/test.txt
I0604 06:09:18.020703 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.913
I0604 06:09:18.021788 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.917
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
I0604 06:09:40.311201 139760335886208 interactiveshell.py:2882] Step 10200 | Loss: 0.0696 | Spent: 113.3 secs | LR: 0.000178
I0604 06:09:51.865028 139760335886208 interactiveshell.py:2882] Step 10250 | Loss: 0.1023 | Spent: 11.6 secs | LR: 0.000177
I0604 06:10:03.506661 139760335886208 interactiveshell.py:2882] Step 10300 | Loss: 0.0439 | Spent: 11.6 secs | LR: 0.000177
I0604 06:10:15.313686 139760335886208 interactiveshell.py:2882] Step 10350 | Loss: 0.1518 | Spent: 11.8 secs | LR: 0.000176
I0604 06:10:26.762205 139760335886208 interactiveshell.py:2882] Step 10400 | Loss: 0.0862 | Spent: 11.4 secs | LR: 0.000176
I0604 06:10:38.290188 139760335886208 interactiveshell.py:2882] Step 10450 | Loss: 0.0407 | Spent: 11.5 secs | LR: 0.000176
I0604 06:10:49.670680 139760335886208 interactiveshell.py:2882] Step 10500 | Loss: 0.0511 | Spent: 11.4 secs | LR: 0.000175
I0604 06:11:01.067734 139760335886208 interactiveshell.py:2882] Step 10550 | Loss: 0.1261 | Spent: 11.4 secs | LR: 0.000175
I0604 06:11:12.532267 139760335886208 interactiveshell.py:2882] Step 10600 | Loss: 0.0926 | Spent: 11.5 secs | LR: 0.000174
I0604 06:11:24.007652 139760335886208 interactiveshell.py:2882] Step 10650 | Loss: 0.1600 | Spent: 11.5 secs | LR: 0.000174
I0604 06:11:35.565228 139760335886208 interactiveshell.py:2882] Step 10700 | Loss: 0.0313 | Spent: 11.6 secs | LR: 0.000173
I0604 06:11:47.023925 139760335886208 interactiveshell.py:2882] Step 10750 | Loss: 0.0157 | Spent: 11.5 secs | LR: 0.000173
I0604 06:11:58.470937 139760335886208 interactiveshell.py:2882] Step 10800 | Loss: 0.1307 | Spent: 11.4 secs | LR: 0.000172
I0604 06:12:09.843670 139760335886208 interactiveshell.py:2882] Step 10850 | Loss: 0.0588 | Spent: 11.4 secs | LR: 0.000172
I0604 06:12:21.296119 139760335886208 interactiveshell.py:2882] Step 10900 | Loss: 0.0888 | Spent: 11.5 secs | LR: 0.000172
Reading ../data/test.txt
I0604 06:13:59.575398 139760335886208 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.899
I0604 06:13:59.576631 139760335886208 interactiveshell.py:2882] Best Accuracy: 0.917
I0604 06:13:59.582616 139760335886208 interactiveshell.py:2882] Testing Accuracy not improved over 3 epochs, Early Stop