In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tensorflow-gpu==2.0.0-alpha0
Requirement already satisfied: tensorflow-gpu==2.0.0-alpha0 in /usr/local/lib/python3.6/dist-packages (2.0.0a0)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.11.0)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.2)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)
Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0.dev2019030115)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)
Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.1.5)
Requirement already satisfied: tb-nightly<1.14.0a20190302,>=1.14.0a20190301 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0a20190301)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (40.9.0)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)
In [3]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import array
import hashlib
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-alpha0
GPU Enabled: True
In [0]:
def hashed_ngrams(unigrams, unigram_vocab_size, n_buckets, n):
    """hashed n-gram features."""

    def get_ngrams(n):
        return list(zip(*[unigrams[i:] for i in range(n)]))

    def hash_ngram(ngram):
        bytes_ = array.array('L', ngram).tobytes()
        hash_ = int(hashlib.sha256(bytes_).hexdigest(), 16)
        return unigram_vocab_size + hash_ % n_buckets

    return [hash_ngram(ngram) for ngram in get_ngrams(n)]


def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      label, text = line.split('\t')
      text = text.split(' ')
      x = [params['word2idx'].get(w, len(word2idx)) for w in text]
      if len(x) > params['max_len']:
        x = x[:params['max_len']]
      x += hashed_ngrams(x,
                         params['vocab_size'],
                         params['num_buckets'],
                         n=2,)
      y = int(label)
      yield x, y


def dataset(is_training, params):
  _shapes = ([None], ())
  _types = (tf.int32, tf.int32)
  _pads = (0, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    #ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class FastText(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.unigram = tf.Variable(np.load('../vocab/word.npy'), dtype=tf.float32, name='embedding_unigram')
    self.bigram = tf.Variable(np.random.uniform(size=(params['num_buckets'], 300)), dtype=tf.float32, name='embedding_bigram')
    self.out_linear = tf.keras.layers.Dense(2)
    
  def call(self, inputs):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.cast(tf.sign(inputs), tf.float32)
    
    x = tf.nn.embedding_lookup(tf.concat((self.unigram, self.bigram), axis=0), inputs)
    
    masked_mean = lambda x, mask: tf.reduce_sum(x * tf.expand_dims(mask, -1), 1) / tf.reduce_sum(mask, 1, keepdims=True)
    x = masked_mean(x, masks)
    
    x = self.out_linear(x)
    
    return x
In [0]:
params = {
  'vocab_path': '../vocab/word.txt',
  'train_path': '../data/train.txt',
  'test_path': '../data/test.txt',
  'num_samples': 25000,
  'num_labels': 2,
  'batch_size': 32,
  'max_len': 500,
  'num_buckets': int(8E5),
  'num_patience': 3,
  'lr': 3e-4,
}
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [8]:
word2idx = {}
with open(params['vocab_path']) as f:
  for i, line in enumerate(f):
    line = line.rstrip()
    word2idx[line] = i
params['word2idx'] = word2idx
params['vocab_size'] = len(word2idx) + 1


model = FastText(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.95)
optim = tf.keras.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = 0.

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for texts, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model(texts)
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
      loss = tf.reduce_mean(loss)

    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for texts, labels in dataset(is_training=False, params=params):
    logits = model(texts)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))  
  history_acc.append(acc)
  
  if acc > best_acc:
    best_acc = acc
    # you can save model here
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0415 01:20:15.735604 139745329543040 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:410: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('dense/kernel:0', TensorShape([300, 2])),
 ('dense/bias:0', TensorShape([2])),
 ('embedding_unigram:0', TensorShape([20599, 300])),
 ('embedding_bigram:0', TensorShape([800000, 300]))]
Reading ../data/train.txt
W0415 01:20:15.996830 139745329543040 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.cast` instead.
I0415 01:20:16.188757 139745329543040 interactiveshell.py:2882] Step 0 | Loss: 0.6812 | Spent: 0.5 secs | LR: 0.000300
I0415 01:20:26.224740 139745329543040 interactiveshell.py:2882] Step 50 | Loss: 0.6753 | Spent: 10.0 secs | LR: 0.000299
I0415 01:20:36.381464 139745329543040 interactiveshell.py:2882] Step 100 | Loss: 0.6946 | Spent: 10.2 secs | LR: 0.000298
I0415 01:20:46.588542 139745329543040 interactiveshell.py:2882] Step 150 | Loss: 0.6757 | Spent: 10.2 secs | LR: 0.000298
I0415 01:20:56.824025 139745329543040 interactiveshell.py:2882] Step 200 | Loss: 0.6717 | Spent: 10.2 secs | LR: 0.000297
I0415 01:21:07.103020 139745329543040 interactiveshell.py:2882] Step 250 | Loss: 0.6699 | Spent: 10.3 secs | LR: 0.000296
I0415 01:21:17.404358 139745329543040 interactiveshell.py:2882] Step 300 | Loss: 0.6511 | Spent: 10.3 secs | LR: 0.000295
I0415 01:21:27.743236 139745329543040 interactiveshell.py:2882] Step 350 | Loss: 0.6530 | Spent: 10.3 secs | LR: 0.000295
I0415 01:21:38.142692 139745329543040 interactiveshell.py:2882] Step 400 | Loss: 0.6607 | Spent: 10.4 secs | LR: 0.000294
I0415 01:21:48.616468 139745329543040 interactiveshell.py:2882] Step 450 | Loss: 0.6463 | Spent: 10.5 secs | LR: 0.000293
I0415 01:21:59.127490 139745329543040 interactiveshell.py:2882] Step 500 | Loss: 0.6664 | Spent: 10.5 secs | LR: 0.000292
I0415 01:22:09.721337 139745329543040 interactiveshell.py:2882] Step 550 | Loss: 0.6332 | Spent: 10.6 secs | LR: 0.000292
I0415 01:22:20.445174 139745329543040 interactiveshell.py:2882] Step 600 | Loss: 0.6342 | Spent: 10.7 secs | LR: 0.000291
I0415 01:22:31.329882 139745329543040 interactiveshell.py:2882] Step 650 | Loss: 0.6668 | Spent: 10.9 secs | LR: 0.000290
I0415 01:22:42.437232 139745329543040 interactiveshell.py:2882] Step 700 | Loss: 0.6326 | Spent: 11.1 secs | LR: 0.000289
I0415 01:22:53.844873 139745329543040 interactiveshell.py:2882] Step 750 | Loss: 0.6277 | Spent: 11.4 secs | LR: 0.000289
Reading ../data/test.txt
I0415 01:23:33.981194 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.766
I0415 01:23:33.982703 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.766
Reading ../data/train.txt
I0415 01:23:37.864746 139745329543040 interactiveshell.py:2882] Step 800 | Loss: 0.5530 | Spent: 44.0 secs | LR: 0.000288
I0415 01:23:47.911203 139745329543040 interactiveshell.py:2882] Step 850 | Loss: 0.5427 | Spent: 10.0 secs | LR: 0.000287
I0415 01:23:58.022640 139745329543040 interactiveshell.py:2882] Step 900 | Loss: 0.5377 | Spent: 10.1 secs | LR: 0.000286
I0415 01:24:08.215434 139745329543040 interactiveshell.py:2882] Step 950 | Loss: 0.5984 | Spent: 10.2 secs | LR: 0.000286
I0415 01:24:18.416720 139745329543040 interactiveshell.py:2882] Step 1000 | Loss: 0.5643 | Spent: 10.2 secs | LR: 0.000285
I0415 01:24:28.635050 139745329543040 interactiveshell.py:2882] Step 1050 | Loss: 0.5557 | Spent: 10.2 secs | LR: 0.000284
I0415 01:24:39.036653 139745329543040 interactiveshell.py:2882] Step 1100 | Loss: 0.5401 | Spent: 10.4 secs | LR: 0.000284
I0415 01:24:49.413336 139745329543040 interactiveshell.py:2882] Step 1150 | Loss: 0.5988 | Spent: 10.4 secs | LR: 0.000283
I0415 01:24:59.825424 139745329543040 interactiveshell.py:2882] Step 1200 | Loss: 0.5432 | Spent: 10.4 secs | LR: 0.000282
I0415 01:25:10.263591 139745329543040 interactiveshell.py:2882] Step 1250 | Loss: 0.5706 | Spent: 10.4 secs | LR: 0.000281
I0415 01:25:20.717916 139745329543040 interactiveshell.py:2882] Step 1300 | Loss: 0.5191 | Spent: 10.5 secs | LR: 0.000281
I0415 01:25:31.283183 139745329543040 interactiveshell.py:2882] Step 1350 | Loss: 0.5540 | Spent: 10.6 secs | LR: 0.000280
I0415 01:25:42.057296 139745329543040 interactiveshell.py:2882] Step 1400 | Loss: 0.5741 | Spent: 10.8 secs | LR: 0.000279
I0415 01:25:53.014499 139745329543040 interactiveshell.py:2882] Step 1450 | Loss: 0.5769 | Spent: 11.0 secs | LR: 0.000278
I0415 01:26:04.237364 139745329543040 interactiveshell.py:2882] Step 1500 | Loss: 0.5236 | Spent: 11.2 secs | LR: 0.000278
I0415 01:26:15.653532 139745329543040 interactiveshell.py:2882] Step 1550 | Loss: 0.5526 | Spent: 11.4 secs | LR: 0.000277
Reading ../data/test.txt
I0415 01:26:52.413290 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.824
I0415 01:26:52.414806 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.824
Reading ../data/train.txt
I0415 01:26:59.900328 139745329543040 interactiveshell.py:2882] Step 1600 | Loss: 0.3533 | Spent: 44.2 secs | LR: 0.000276
I0415 01:27:09.967741 139745329543040 interactiveshell.py:2882] Step 1650 | Loss: 0.4634 | Spent: 10.1 secs | LR: 0.000276
I0415 01:27:20.100201 139745329543040 interactiveshell.py:2882] Step 1700 | Loss: 0.4806 | Spent: 10.1 secs | LR: 0.000275
I0415 01:27:30.279241 139745329543040 interactiveshell.py:2882] Step 1750 | Loss: 0.4857 | Spent: 10.2 secs | LR: 0.000274
I0415 01:27:40.542280 139745329543040 interactiveshell.py:2882] Step 1800 | Loss: 0.4455 | Spent: 10.3 secs | LR: 0.000274
I0415 01:27:50.813371 139745329543040 interactiveshell.py:2882] Step 1850 | Loss: 0.4506 | Spent: 10.3 secs | LR: 0.000273
I0415 01:28:01.131809 139745329543040 interactiveshell.py:2882] Step 1900 | Loss: 0.4554 | Spent: 10.3 secs | LR: 0.000272
I0415 01:28:11.533240 139745329543040 interactiveshell.py:2882] Step 1950 | Loss: 0.3997 | Spent: 10.4 secs | LR: 0.000271
I0415 01:28:21.967097 139745329543040 interactiveshell.py:2882] Step 2000 | Loss: 0.4399 | Spent: 10.4 secs | LR: 0.000271
I0415 01:28:32.461143 139745329543040 interactiveshell.py:2882] Step 2050 | Loss: 0.4350 | Spent: 10.5 secs | LR: 0.000270
I0415 01:28:43.012799 139745329543040 interactiveshell.py:2882] Step 2100 | Loss: 0.4179 | Spent: 10.6 secs | LR: 0.000269
I0415 01:28:53.683593 139745329543040 interactiveshell.py:2882] Step 2150 | Loss: 0.4352 | Spent: 10.7 secs | LR: 0.000269
I0415 01:29:04.508596 139745329543040 interactiveshell.py:2882] Step 2200 | Loss: 0.4368 | Spent: 10.8 secs | LR: 0.000268
I0415 01:29:15.545304 139745329543040 interactiveshell.py:2882] Step 2250 | Loss: 0.4399 | Spent: 11.0 secs | LR: 0.000267
I0415 01:29:26.919458 139745329543040 interactiveshell.py:2882] Step 2300 | Loss: 0.4291 | Spent: 11.4 secs | LR: 0.000267
Reading ../data/test.txt
I0415 01:30:11.267449 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.853
I0415 01:30:11.269586 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.853
Reading ../data/train.txt
I0415 01:30:12.357179 139745329543040 interactiveshell.py:2882] Step 2350 | Loss: 0.1598 | Spent: 45.4 secs | LR: 0.000266
I0415 01:30:22.346391 139745329543040 interactiveshell.py:2882] Step 2400 | Loss: 0.2625 | Spent: 10.0 secs | LR: 0.000265
I0415 01:30:32.452352 139745329543040 interactiveshell.py:2882] Step 2450 | Loss: 0.3647 | Spent: 10.1 secs | LR: 0.000265
I0415 01:30:42.586920 139745329543040 interactiveshell.py:2882] Step 2500 | Loss: 0.2241 | Spent: 10.1 secs | LR: 0.000264
I0415 01:30:52.811489 139745329543040 interactiveshell.py:2882] Step 2550 | Loss: 0.3362 | Spent: 10.2 secs | LR: 0.000263
I0415 01:31:03.061976 139745329543040 interactiveshell.py:2882] Step 2600 | Loss: 0.3077 | Spent: 10.2 secs | LR: 0.000263
I0415 01:31:13.362631 139745329543040 interactiveshell.py:2882] Step 2650 | Loss: 0.3035 | Spent: 10.3 secs | LR: 0.000262
I0415 01:31:23.693254 139745329543040 interactiveshell.py:2882] Step 2700 | Loss: 0.2936 | Spent: 10.3 secs | LR: 0.000261
I0415 01:31:34.077312 139745329543040 interactiveshell.py:2882] Step 2750 | Loss: 0.3001 | Spent: 10.4 secs | LR: 0.000261
I0415 01:31:44.518343 139745329543040 interactiveshell.py:2882] Step 2800 | Loss: 0.2908 | Spent: 10.4 secs | LR: 0.000260
I0415 01:31:55.057559 139745329543040 interactiveshell.py:2882] Step 2850 | Loss: 0.4034 | Spent: 10.5 secs | LR: 0.000259
I0415 01:32:05.664100 139745329543040 interactiveshell.py:2882] Step 2900 | Loss: 0.3290 | Spent: 10.6 secs | LR: 0.000259
I0415 01:32:16.385904 139745329543040 interactiveshell.py:2882] Step 2950 | Loss: 0.3830 | Spent: 10.7 secs | LR: 0.000258
I0415 01:32:27.243322 139745329543040 interactiveshell.py:2882] Step 3000 | Loss: 0.3712 | Spent: 10.9 secs | LR: 0.000257
I0415 01:32:38.354990 139745329543040 interactiveshell.py:2882] Step 3050 | Loss: 0.4493 | Spent: 11.1 secs | LR: 0.000257
I0415 01:32:49.732590 139745329543040 interactiveshell.py:2882] Step 3100 | Loss: 0.3827 | Spent: 11.4 secs | LR: 0.000256
Reading ../data/test.txt
I0415 01:33:29.134066 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.870
I0415 01:33:29.135443 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.870
Reading ../data/train.txt
I0415 01:33:33.810279 139745329543040 interactiveshell.py:2882] Step 3150 | Loss: 0.2615 | Spent: 44.1 secs | LR: 0.000255
I0415 01:33:43.826864 139745329543040 interactiveshell.py:2882] Step 3200 | Loss: 0.1579 | Spent: 10.0 secs | LR: 0.000255
I0415 01:33:54.029944 139745329543040 interactiveshell.py:2882] Step 3250 | Loss: 0.1565 | Spent: 10.2 secs | LR: 0.000254
I0415 01:34:04.291844 139745329543040 interactiveshell.py:2882] Step 3300 | Loss: 0.2032 | Spent: 10.3 secs | LR: 0.000253
I0415 01:34:14.546670 139745329543040 interactiveshell.py:2882] Step 3350 | Loss: 0.2651 | Spent: 10.3 secs | LR: 0.000253
I0415 01:34:24.818355 139745329543040 interactiveshell.py:2882] Step 3400 | Loss: 0.2184 | Spent: 10.3 secs | LR: 0.000252
I0415 01:34:35.105801 139745329543040 interactiveshell.py:2882] Step 3450 | Loss: 0.1947 | Spent: 10.3 secs | LR: 0.000251
I0415 01:34:45.472296 139745329543040 interactiveshell.py:2882] Step 3500 | Loss: 0.2948 | Spent: 10.4 secs | LR: 0.000251
I0415 01:34:55.914911 139745329543040 interactiveshell.py:2882] Step 3550 | Loss: 0.1904 | Spent: 10.4 secs | LR: 0.000250
I0415 01:35:06.425357 139745329543040 interactiveshell.py:2882] Step 3600 | Loss: 0.3719 | Spent: 10.5 secs | LR: 0.000249
I0415 01:35:16.960734 139745329543040 interactiveshell.py:2882] Step 3650 | Loss: 0.2836 | Spent: 10.5 secs | LR: 0.000249
I0415 01:35:27.586854 139745329543040 interactiveshell.py:2882] Step 3700 | Loss: 0.2505 | Spent: 10.6 secs | LR: 0.000248
I0415 01:35:38.354557 139745329543040 interactiveshell.py:2882] Step 3750 | Loss: 0.2216 | Spent: 10.8 secs | LR: 0.000248
I0415 01:35:49.306425 139745329543040 interactiveshell.py:2882] Step 3800 | Loss: 0.3111 | Spent: 10.9 secs | LR: 0.000247
I0415 01:36:00.545531 139745329543040 interactiveshell.py:2882] Step 3850 | Loss: 0.3719 | Spent: 11.2 secs | LR: 0.000246
I0415 01:36:11.943248 139745329543040 interactiveshell.py:2882] Step 3900 | Loss: 0.3490 | Spent: 11.4 secs | LR: 0.000246
Reading ../data/test.txt
I0415 01:36:47.728570 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.879
I0415 01:36:47.729991 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.879
Reading ../data/train.txt
I0415 01:36:56.072151 139745329543040 interactiveshell.py:2882] Step 3950 | Loss: 0.0989 | Spent: 44.1 secs | LR: 0.000245
I0415 01:37:06.279090 139745329543040 interactiveshell.py:2882] Step 4000 | Loss: 0.2196 | Spent: 10.2 secs | LR: 0.000244
I0415 01:37:16.506770 139745329543040 interactiveshell.py:2882] Step 4050 | Loss: 0.1820 | Spent: 10.2 secs | LR: 0.000244
I0415 01:37:26.772241 139745329543040 interactiveshell.py:2882] Step 4100 | Loss: 0.2969 | Spent: 10.3 secs | LR: 0.000243
I0415 01:37:37.069669 139745329543040 interactiveshell.py:2882] Step 4150 | Loss: 0.1457 | Spent: 10.3 secs | LR: 0.000242
I0415 01:37:47.388544 139745329543040 interactiveshell.py:2882] Step 4200 | Loss: 0.2315 | Spent: 10.3 secs | LR: 0.000242
I0415 01:37:57.747011 139745329543040 interactiveshell.py:2882] Step 4250 | Loss: 0.1347 | Spent: 10.4 secs | LR: 0.000241
I0415 01:38:08.119165 139745329543040 interactiveshell.py:2882] Step 4300 | Loss: 0.1676 | Spent: 10.4 secs | LR: 0.000241
I0415 01:38:18.564817 139745329543040 interactiveshell.py:2882] Step 4350 | Loss: 0.2224 | Spent: 10.4 secs | LR: 0.000240
I0415 01:38:29.056092 139745329543040 interactiveshell.py:2882] Step 4400 | Loss: 0.1698 | Spent: 10.5 secs | LR: 0.000239
I0415 01:38:39.598960 139745329543040 interactiveshell.py:2882] Step 4450 | Loss: 0.1891 | Spent: 10.5 secs | LR: 0.000239
I0415 01:38:50.273959 139745329543040 interactiveshell.py:2882] Step 4500 | Loss: 0.2357 | Spent: 10.7 secs | LR: 0.000238
I0415 01:39:01.100708 139745329543040 interactiveshell.py:2882] Step 4550 | Loss: 0.2245 | Spent: 10.8 secs | LR: 0.000238
I0415 01:39:12.148861 139745329543040 interactiveshell.py:2882] Step 4600 | Loss: 0.3252 | Spent: 11.0 secs | LR: 0.000237
I0415 01:39:23.484169 139745329543040 interactiveshell.py:2882] Step 4650 | Loss: 0.2685 | Spent: 11.3 secs | LR: 0.000236
Reading ../data/test.txt
I0415 01:40:06.852572 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.885
I0415 01:40:06.854040 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.885
Reading ../data/train.txt
I0415 01:40:08.765081 139745329543040 interactiveshell.py:2882] Step 4700 | Loss: 0.0938 | Spent: 45.3 secs | LR: 0.000236
I0415 01:40:18.910866 139745329543040 interactiveshell.py:2882] Step 4750 | Loss: 0.1524 | Spent: 10.1 secs | LR: 0.000235
I0415 01:40:29.060257 139745329543040 interactiveshell.py:2882] Step 4800 | Loss: 0.1162 | Spent: 10.1 secs | LR: 0.000235
I0415 01:40:39.270718 139745329543040 interactiveshell.py:2882] Step 4850 | Loss: 0.0882 | Spent: 10.2 secs | LR: 0.000234
I0415 01:40:49.522821 139745329543040 interactiveshell.py:2882] Step 4900 | Loss: 0.1100 | Spent: 10.2 secs | LR: 0.000233
I0415 01:40:59.778528 139745329543040 interactiveshell.py:2882] Step 4950 | Loss: 0.1914 | Spent: 10.3 secs | LR: 0.000233
I0415 01:41:10.070171 139745329543040 interactiveshell.py:2882] Step 5000 | Loss: 0.1333 | Spent: 10.3 secs | LR: 0.000232
I0415 01:41:20.414767 139745329543040 interactiveshell.py:2882] Step 5050 | Loss: 0.1729 | Spent: 10.3 secs | LR: 0.000232
I0415 01:41:30.775951 139745329543040 interactiveshell.py:2882] Step 5100 | Loss: 0.3836 | Spent: 10.4 secs | LR: 0.000231
I0415 01:41:41.206710 139745329543040 interactiveshell.py:2882] Step 5150 | Loss: 0.1852 | Spent: 10.4 secs | LR: 0.000230
I0415 01:41:51.715557 139745329543040 interactiveshell.py:2882] Step 5200 | Loss: 0.1696 | Spent: 10.5 secs | LR: 0.000230
I0415 01:42:02.343198 139745329543040 interactiveshell.py:2882] Step 5250 | Loss: 0.1234 | Spent: 10.6 secs | LR: 0.000229
I0415 01:42:13.107508 139745329543040 interactiveshell.py:2882] Step 5300 | Loss: 0.2139 | Spent: 10.8 secs | LR: 0.000229
I0415 01:42:24.005746 139745329543040 interactiveshell.py:2882] Step 5350 | Loss: 0.1783 | Spent: 10.9 secs | LR: 0.000228
I0415 01:42:35.146737 139745329543040 interactiveshell.py:2882] Step 5400 | Loss: 0.3027 | Spent: 11.1 secs | LR: 0.000227
I0415 01:42:46.528259 139745329543040 interactiveshell.py:2882] Step 5450 | Loss: 0.2494 | Spent: 11.4 secs | LR: 0.000227
Reading ../data/test.txt
I0415 01:43:25.430454 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0415 01:43:25.431770 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.889
Reading ../data/train.txt
I0415 01:43:30.957347 139745329543040 interactiveshell.py:2882] Step 5500 | Loss: 0.0643 | Spent: 44.4 secs | LR: 0.000226
I0415 01:43:41.072390 139745329543040 interactiveshell.py:2882] Step 5550 | Loss: 0.0723 | Spent: 10.1 secs | LR: 0.000226
I0415 01:43:51.274345 139745329543040 interactiveshell.py:2882] Step 5600 | Loss: 0.1131 | Spent: 10.2 secs | LR: 0.000225
I0415 01:44:01.509601 139745329543040 interactiveshell.py:2882] Step 5650 | Loss: 0.0838 | Spent: 10.2 secs | LR: 0.000225
I0415 01:44:11.778080 139745329543040 interactiveshell.py:2882] Step 5700 | Loss: 0.1016 | Spent: 10.3 secs | LR: 0.000224
I0415 01:44:22.046715 139745329543040 interactiveshell.py:2882] Step 5750 | Loss: 0.1861 | Spent: 10.3 secs | LR: 0.000223
I0415 01:44:32.349428 139745329543040 interactiveshell.py:2882] Step 5800 | Loss: 0.1513 | Spent: 10.3 secs | LR: 0.000223
I0415 01:44:42.723931 139745329543040 interactiveshell.py:2882] Step 5850 | Loss: 0.1360 | Spent: 10.4 secs | LR: 0.000222
I0415 01:44:53.113706 139745329543040 interactiveshell.py:2882] Step 5900 | Loss: 0.1502 | Spent: 10.4 secs | LR: 0.000222
I0415 01:45:03.571163 139745329543040 interactiveshell.py:2882] Step 5950 | Loss: 0.1601 | Spent: 10.5 secs | LR: 0.000221
I0415 01:45:14.104643 139745329543040 interactiveshell.py:2882] Step 6000 | Loss: 0.1851 | Spent: 10.5 secs | LR: 0.000221
I0415 01:45:24.832377 139745329543040 interactiveshell.py:2882] Step 6050 | Loss: 0.2386 | Spent: 10.7 secs | LR: 0.000220
I0415 01:45:35.611841 139745329543040 interactiveshell.py:2882] Step 6100 | Loss: 0.1360 | Spent: 10.8 secs | LR: 0.000219
I0415 01:45:46.590933 139745329543040 interactiveshell.py:2882] Step 6150 | Loss: 0.1878 | Spent: 11.0 secs | LR: 0.000219
I0415 01:45:57.860543 139745329543040 interactiveshell.py:2882] Step 6200 | Loss: 0.1658 | Spent: 11.3 secs | LR: 0.000218
I0415 01:46:09.291716 139745329543040 interactiveshell.py:2882] Step 6250 | Loss: 0.2134 | Spent: 11.4 secs | LR: 0.000218
Reading ../data/test.txt
I0415 01:46:44.128950 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.893
I0415 01:46:44.130201 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.893
Reading ../data/train.txt
I0415 01:46:53.264286 139745329543040 interactiveshell.py:2882] Step 6300 | Loss: 0.0923 | Spent: 44.0 secs | LR: 0.000217
I0415 01:47:03.399487 139745329543040 interactiveshell.py:2882] Step 6350 | Loss: 0.0580 | Spent: 10.1 secs | LR: 0.000217
I0415 01:47:13.665977 139745329543040 interactiveshell.py:2882] Step 6400 | Loss: 0.1363 | Spent: 10.3 secs | LR: 0.000216
I0415 01:47:23.905976 139745329543040 interactiveshell.py:2882] Step 6450 | Loss: 0.0984 | Spent: 10.2 secs | LR: 0.000215
I0415 01:47:34.171512 139745329543040 interactiveshell.py:2882] Step 6500 | Loss: 0.0859 | Spent: 10.3 secs | LR: 0.000215
I0415 01:47:44.461750 139745329543040 interactiveshell.py:2882] Step 6550 | Loss: 0.0840 | Spent: 10.3 secs | LR: 0.000214
I0415 01:47:54.776402 139745329543040 interactiveshell.py:2882] Step 6600 | Loss: 0.1099 | Spent: 10.3 secs | LR: 0.000214
I0415 01:48:05.146180 139745329543040 interactiveshell.py:2882] Step 6650 | Loss: 0.1106 | Spent: 10.4 secs | LR: 0.000213
I0415 01:48:15.557303 139745329543040 interactiveshell.py:2882] Step 6700 | Loss: 0.1646 | Spent: 10.4 secs | LR: 0.000213
I0415 01:48:26.037533 139745329543040 interactiveshell.py:2882] Step 6750 | Loss: 0.0732 | Spent: 10.5 secs | LR: 0.000212
I0415 01:48:36.596440 139745329543040 interactiveshell.py:2882] Step 6800 | Loss: 0.1379 | Spent: 10.6 secs | LR: 0.000212
I0415 01:48:47.277818 139745329543040 interactiveshell.py:2882] Step 6850 | Loss: 0.1170 | Spent: 10.7 secs | LR: 0.000211
I0415 01:48:58.123697 139745329543040 interactiveshell.py:2882] Step 6900 | Loss: 0.1159 | Spent: 10.8 secs | LR: 0.000211
I0415 01:49:09.185041 139745329543040 interactiveshell.py:2882] Step 6950 | Loss: 0.2053 | Spent: 11.1 secs | LR: 0.000210
I0415 01:49:20.541540 139745329543040 interactiveshell.py:2882] Step 7000 | Loss: 0.1555 | Spent: 11.4 secs | LR: 0.000210
Reading ../data/test.txt
I0415 01:50:02.823018 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.895
I0415 01:50:02.824579 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0415 01:50:05.484863 139745329543040 interactiveshell.py:2882] Step 7050 | Loss: 0.0286 | Spent: 44.9 secs | LR: 0.000209
I0415 01:50:15.489789 139745329543040 interactiveshell.py:2882] Step 7100 | Loss: 0.0633 | Spent: 10.0 secs | LR: 0.000208
I0415 01:50:25.601087 139745329543040 interactiveshell.py:2882] Step 7150 | Loss: 0.0513 | Spent: 10.1 secs | LR: 0.000208
I0415 01:50:35.819119 139745329543040 interactiveshell.py:2882] Step 7200 | Loss: 0.0815 | Spent: 10.2 secs | LR: 0.000207
I0415 01:50:46.035642 139745329543040 interactiveshell.py:2882] Step 7250 | Loss: 0.0516 | Spent: 10.2 secs | LR: 0.000207
I0415 01:50:56.313188 139745329543040 interactiveshell.py:2882] Step 7300 | Loss: 0.0786 | Spent: 10.3 secs | LR: 0.000206
I0415 01:51:06.642698 139745329543040 interactiveshell.py:2882] Step 7350 | Loss: 0.0934 | Spent: 10.3 secs | LR: 0.000206
I0415 01:51:16.969647 139745329543040 interactiveshell.py:2882] Step 7400 | Loss: 0.0656 | Spent: 10.3 secs | LR: 0.000205
I0415 01:51:27.364239 139745329543040 interactiveshell.py:2882] Step 7450 | Loss: 0.0857 | Spent: 10.4 secs | LR: 0.000205
I0415 01:51:37.811794 139745329543040 interactiveshell.py:2882] Step 7500 | Loss: 0.1147 | Spent: 10.4 secs | LR: 0.000204
I0415 01:51:48.315237 139745329543040 interactiveshell.py:2882] Step 7550 | Loss: 0.0591 | Spent: 10.5 secs | LR: 0.000204
I0415 01:51:58.927058 139745329543040 interactiveshell.py:2882] Step 7600 | Loss: 0.1246 | Spent: 10.6 secs | LR: 0.000203
I0415 01:52:09.661368 139745329543040 interactiveshell.py:2882] Step 7650 | Loss: 0.1227 | Spent: 10.7 secs | LR: 0.000203
I0415 01:52:20.634384 139745329543040 interactiveshell.py:2882] Step 7700 | Loss: 0.1292 | Spent: 11.0 secs | LR: 0.000202
I0415 01:52:31.815685 139745329543040 interactiveshell.py:2882] Step 7750 | Loss: 0.1066 | Spent: 11.2 secs | LR: 0.000202
I0415 01:52:43.204978 139745329543040 interactiveshell.py:2882] Step 7800 | Loss: 0.1161 | Spent: 11.4 secs | LR: 0.000201
Reading ../data/test.txt
I0415 01:53:21.054434 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.896
I0415 01:53:21.055746 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.896
Reading ../data/train.txt
I0415 01:53:27.370863 139745329543040 interactiveshell.py:2882] Step 7850 | Loss: 0.0237 | Spent: 44.2 secs | LR: 0.000201
I0415 01:53:37.509232 139745329543040 interactiveshell.py:2882] Step 7900 | Loss: 0.0149 | Spent: 10.1 secs | LR: 0.000200
I0415 01:53:47.700642 139745329543040 interactiveshell.py:2882] Step 7950 | Loss: 0.0350 | Spent: 10.2 secs | LR: 0.000200
I0415 01:53:57.934563 139745329543040 interactiveshell.py:2882] Step 8000 | Loss: 0.0503 | Spent: 10.2 secs | LR: 0.000199
I0415 01:54:08.200510 139745329543040 interactiveshell.py:2882] Step 8050 | Loss: 0.0473 | Spent: 10.3 secs | LR: 0.000199
I0415 01:54:18.487461 139745329543040 interactiveshell.py:2882] Step 8100 | Loss: 0.0850 | Spent: 10.3 secs | LR: 0.000198
I0415 01:54:28.809647 139745329543040 interactiveshell.py:2882] Step 8150 | Loss: 0.0880 | Spent: 10.3 secs | LR: 0.000198
I0415 01:54:39.178393 139745329543040 interactiveshell.py:2882] Step 8200 | Loss: 0.0941 | Spent: 10.4 secs | LR: 0.000197
I0415 01:54:49.602063 139745329543040 interactiveshell.py:2882] Step 8250 | Loss: 0.0445 | Spent: 10.4 secs | LR: 0.000196
I0415 01:55:00.066465 139745329543040 interactiveshell.py:2882] Step 8300 | Loss: 0.1262 | Spent: 10.5 secs | LR: 0.000196
I0415 01:55:10.609553 139745329543040 interactiveshell.py:2882] Step 8350 | Loss: 0.0924 | Spent: 10.5 secs | LR: 0.000195
I0415 01:55:21.262106 139745329543040 interactiveshell.py:2882] Step 8400 | Loss: 0.0931 | Spent: 10.7 secs | LR: 0.000195
I0415 01:55:32.057971 139745329543040 interactiveshell.py:2882] Step 8450 | Loss: 0.0661 | Spent: 10.8 secs | LR: 0.000194
I0415 01:55:43.115536 139745329543040 interactiveshell.py:2882] Step 8500 | Loss: 0.0935 | Spent: 11.1 secs | LR: 0.000194
I0415 01:55:54.501989 139745329543040 interactiveshell.py:2882] Step 8550 | Loss: 0.1346 | Spent: 11.4 secs | LR: 0.000193
I0415 01:56:06.013593 139745329543040 interactiveshell.py:2882] Step 8600 | Loss: 0.1600 | Spent: 11.5 secs | LR: 0.000193
Reading ../data/test.txt
I0415 01:56:39.796809 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.897
I0415 01:56:39.798076 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.897
Reading ../data/train.txt
I0415 01:56:49.767640 139745329543040 interactiveshell.py:2882] Step 8650 | Loss: 0.0163 | Spent: 43.8 secs | LR: 0.000192
I0415 01:56:59.910573 139745329543040 interactiveshell.py:2882] Step 8700 | Loss: 0.0529 | Spent: 10.1 secs | LR: 0.000192
I0415 01:57:10.136873 139745329543040 interactiveshell.py:2882] Step 8750 | Loss: 0.0516 | Spent: 10.2 secs | LR: 0.000192
I0415 01:57:20.416431 139745329543040 interactiveshell.py:2882] Step 8800 | Loss: 0.0441 | Spent: 10.3 secs | LR: 0.000191
I0415 01:57:30.703501 139745329543040 interactiveshell.py:2882] Step 8850 | Loss: 0.0168 | Spent: 10.3 secs | LR: 0.000191
I0415 01:57:41.007178 139745329543040 interactiveshell.py:2882] Step 8900 | Loss: 0.0494 | Spent: 10.3 secs | LR: 0.000190
I0415 01:57:51.343727 139745329543040 interactiveshell.py:2882] Step 8950 | Loss: 0.0421 | Spent: 10.3 secs | LR: 0.000190
I0415 01:58:01.711023 139745329543040 interactiveshell.py:2882] Step 9000 | Loss: 0.0477 | Spent: 10.4 secs | LR: 0.000189
I0415 01:58:12.156428 139745329543040 interactiveshell.py:2882] Step 9050 | Loss: 0.0438 | Spent: 10.4 secs | LR: 0.000189
I0415 01:58:22.640413 139745329543040 interactiveshell.py:2882] Step 9100 | Loss: 0.0792 | Spent: 10.5 secs | LR: 0.000188
I0415 01:58:33.221433 139745329543040 interactiveshell.py:2882] Step 9150 | Loss: 0.1018 | Spent: 10.6 secs | LR: 0.000188
I0415 01:58:43.911850 139745329543040 interactiveshell.py:2882] Step 9200 | Loss: 0.0451 | Spent: 10.7 secs | LR: 0.000187
I0415 01:58:54.787168 139745329543040 interactiveshell.py:2882] Step 9250 | Loss: 0.2473 | Spent: 10.9 secs | LR: 0.000187
I0415 01:59:05.890188 139745329543040 interactiveshell.py:2882] Step 9300 | Loss: 0.1117 | Spent: 11.1 secs | LR: 0.000186
I0415 01:59:17.338641 139745329543040 interactiveshell.py:2882] Step 9350 | Loss: 0.1356 | Spent: 11.4 secs | LR: 0.000186
Reading ../data/test.txt
I0415 01:59:58.580346 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.898
I0415 01:59:58.581697 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.898
Reading ../data/train.txt
I0415 02:00:02.073007 139745329543040 interactiveshell.py:2882] Step 9400 | Loss: 0.0130 | Spent: 44.7 secs | LR: 0.000185
I0415 02:00:12.165306 139745329543040 interactiveshell.py:2882] Step 9450 | Loss: 0.0595 | Spent: 10.1 secs | LR: 0.000185
I0415 02:00:22.336675 139745329543040 interactiveshell.py:2882] Step 9500 | Loss: 0.0811 | Spent: 10.2 secs | LR: 0.000184
I0415 02:00:32.548517 139745329543040 interactiveshell.py:2882] Step 9550 | Loss: 0.0581 | Spent: 10.2 secs | LR: 0.000184
I0415 02:00:42.799042 139745329543040 interactiveshell.py:2882] Step 9600 | Loss: 0.0408 | Spent: 10.2 secs | LR: 0.000183
I0415 02:00:53.146881 139745329543040 interactiveshell.py:2882] Step 9650 | Loss: 0.0325 | Spent: 10.3 secs | LR: 0.000183
I0415 02:01:03.500011 139745329543040 interactiveshell.py:2882] Step 9700 | Loss: 0.0436 | Spent: 10.4 secs | LR: 0.000182
I0415 02:01:13.901577 139745329543040 interactiveshell.py:2882] Step 9750 | Loss: 0.0425 | Spent: 10.4 secs | LR: 0.000182
I0415 02:01:24.322211 139745329543040 interactiveshell.py:2882] Step 9800 | Loss: 0.0430 | Spent: 10.4 secs | LR: 0.000181
I0415 02:01:34.795138 139745329543040 interactiveshell.py:2882] Step 9850 | Loss: 0.0378 | Spent: 10.5 secs | LR: 0.000181
I0415 02:01:45.344248 139745329543040 interactiveshell.py:2882] Step 9900 | Loss: 0.0563 | Spent: 10.5 secs | LR: 0.000181
I0415 02:01:56.006705 139745329543040 interactiveshell.py:2882] Step 9950 | Loss: 0.0437 | Spent: 10.7 secs | LR: 0.000180
I0415 02:02:06.791707 139745329543040 interactiveshell.py:2882] Step 10000 | Loss: 0.0687 | Spent: 10.8 secs | LR: 0.000180
I0415 02:02:17.818943 139745329543040 interactiveshell.py:2882] Step 10050 | Loss: 0.0962 | Spent: 11.0 secs | LR: 0.000179
I0415 02:02:29.110516 139745329543040 interactiveshell.py:2882] Step 10100 | Loss: 0.0918 | Spent: 11.3 secs | LR: 0.000179
I0415 02:02:40.666457 139745329543040 interactiveshell.py:2882] Step 10150 | Loss: 0.1093 | Spent: 11.6 secs | LR: 0.000178
Reading ../data/test.txt
I0415 02:03:17.989763 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.899
I0415 02:03:17.990950 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.899
Reading ../data/train.txt
I0415 02:03:25.105899 139745329543040 interactiveshell.py:2882] Step 10200 | Loss: 0.0106 | Spent: 44.4 secs | LR: 0.000178
I0415 02:03:35.245665 139745329543040 interactiveshell.py:2882] Step 10250 | Loss: 0.1042 | Spent: 10.1 secs | LR: 0.000177
I0415 02:03:45.519788 139745329543040 interactiveshell.py:2882] Step 10300 | Loss: 0.0321 | Spent: 10.3 secs | LR: 0.000177
I0415 02:03:55.756339 139745329543040 interactiveshell.py:2882] Step 10350 | Loss: 0.0226 | Spent: 10.2 secs | LR: 0.000176
I0415 02:04:06.014765 139745329543040 interactiveshell.py:2882] Step 10400 | Loss: 0.0176 | Spent: 10.3 secs | LR: 0.000176
I0415 02:04:16.309518 139745329543040 interactiveshell.py:2882] Step 10450 | Loss: 0.0219 | Spent: 10.3 secs | LR: 0.000176
I0415 02:04:26.634037 139745329543040 interactiveshell.py:2882] Step 10500 | Loss: 0.0346 | Spent: 10.3 secs | LR: 0.000175
I0415 02:04:37.007745 139745329543040 interactiveshell.py:2882] Step 10550 | Loss: 0.0306 | Spent: 10.4 secs | LR: 0.000175
I0415 02:04:47.416496 139745329543040 interactiveshell.py:2882] Step 10600 | Loss: 0.0391 | Spent: 10.4 secs | LR: 0.000174
I0415 02:04:57.878528 139745329543040 interactiveshell.py:2882] Step 10650 | Loss: 0.0421 | Spent: 10.5 secs | LR: 0.000174
I0415 02:05:08.423187 139745329543040 interactiveshell.py:2882] Step 10700 | Loss: 0.0447 | Spent: 10.5 secs | LR: 0.000173
I0415 02:05:19.099644 139745329543040 interactiveshell.py:2882] Step 10750 | Loss: 0.0487 | Spent: 10.7 secs | LR: 0.000173
I0415 02:05:29.962246 139745329543040 interactiveshell.py:2882] Step 10800 | Loss: 0.0828 | Spent: 10.9 secs | LR: 0.000172
I0415 02:05:41.006062 139745329543040 interactiveshell.py:2882] Step 10850 | Loss: 0.0437 | Spent: 11.0 secs | LR: 0.000172
I0415 02:05:52.434099 139745329543040 interactiveshell.py:2882] Step 10900 | Loss: 0.0954 | Spent: 11.4 secs | LR: 0.000172
Reading ../data/test.txt
I0415 02:06:37.896023 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.900
I0415 02:06:37.897310 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.900
Reading ../data/train.txt
I0415 02:06:38.587797 139745329543040 interactiveshell.py:2882] Step 10950 | Loss: 0.0124 | Spent: 46.2 secs | LR: 0.000171
I0415 02:06:48.698247 139745329543040 interactiveshell.py:2882] Step 11000 | Loss: 0.0294 | Spent: 10.1 secs | LR: 0.000171
I0415 02:06:58.855822 139745329543040 interactiveshell.py:2882] Step 11050 | Loss: 0.0244 | Spent: 10.2 secs | LR: 0.000170
I0415 02:07:09.072655 139745329543040 interactiveshell.py:2882] Step 11100 | Loss: 0.0115 | Spent: 10.2 secs | LR: 0.000170
I0415 02:07:19.333903 139745329543040 interactiveshell.py:2882] Step 11150 | Loss: 0.0194 | Spent: 10.3 secs | LR: 0.000169
I0415 02:07:29.626584 139745329543040 interactiveshell.py:2882] Step 11200 | Loss: 0.0214 | Spent: 10.3 secs | LR: 0.000169
I0415 02:07:39.981454 139745329543040 interactiveshell.py:2882] Step 11250 | Loss: 0.0316 | Spent: 10.4 secs | LR: 0.000168
I0415 02:07:50.318995 139745329543040 interactiveshell.py:2882] Step 11300 | Loss: 0.0422 | Spent: 10.3 secs | LR: 0.000168
I0415 02:08:00.714549 139745329543040 interactiveshell.py:2882] Step 11350 | Loss: 0.0427 | Spent: 10.4 secs | LR: 0.000168
I0415 02:08:11.163094 139745329543040 interactiveshell.py:2882] Step 11400 | Loss: 0.0480 | Spent: 10.4 secs | LR: 0.000167
I0415 02:08:21.660577 139745329543040 interactiveshell.py:2882] Step 11450 | Loss: 0.0253 | Spent: 10.5 secs | LR: 0.000167
I0415 02:08:32.249327 139745329543040 interactiveshell.py:2882] Step 11500 | Loss: 0.0439 | Spent: 10.6 secs | LR: 0.000166
I0415 02:08:42.957972 139745329543040 interactiveshell.py:2882] Step 11550 | Loss: 0.0419 | Spent: 10.7 secs | LR: 0.000166
I0415 02:08:53.857802 139745329543040 interactiveshell.py:2882] Step 11600 | Loss: 0.0716 | Spent: 10.9 secs | LR: 0.000165
I0415 02:09:04.964133 139745329543040 interactiveshell.py:2882] Step 11650 | Loss: 0.1247 | Spent: 11.1 secs | LR: 0.000165
I0415 02:09:16.468680 139745329543040 interactiveshell.py:2882] Step 11700 | Loss: 0.0825 | Spent: 11.5 secs | LR: 0.000165
Reading ../data/test.txt
I0415 02:09:56.736287 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.901
I0415 02:09:56.741817 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.901
Reading ../data/train.txt
I0415 02:10:01.036561 139745329543040 interactiveshell.py:2882] Step 11750 | Loss: 0.0083 | Spent: 44.6 secs | LR: 0.000164
I0415 02:10:11.162930 139745329543040 interactiveshell.py:2882] Step 11800 | Loss: 0.0115 | Spent: 10.1 secs | LR: 0.000164
I0415 02:10:21.343115 139745329543040 interactiveshell.py:2882] Step 11850 | Loss: 0.0104 | Spent: 10.2 secs | LR: 0.000163
I0415 02:10:31.591525 139745329543040 interactiveshell.py:2882] Step 11900 | Loss: 0.0248 | Spent: 10.2 secs | LR: 0.000163
I0415 02:10:41.842728 139745329543040 interactiveshell.py:2882] Step 11950 | Loss: 0.0073 | Spent: 10.2 secs | LR: 0.000163
I0415 02:10:52.125167 139745329543040 interactiveshell.py:2882] Step 12000 | Loss: 0.0150 | Spent: 10.3 secs | LR: 0.000162
I0415 02:11:02.436958 139745329543040 interactiveshell.py:2882] Step 12050 | Loss: 0.0240 | Spent: 10.3 secs | LR: 0.000162
I0415 02:11:12.846702 139745329543040 interactiveshell.py:2882] Step 12100 | Loss: 0.0266 | Spent: 10.4 secs | LR: 0.000161
I0415 02:11:23.303385 139745329543040 interactiveshell.py:2882] Step 12150 | Loss: 0.0469 | Spent: 10.5 secs | LR: 0.000161
I0415 02:11:33.768357 139745329543040 interactiveshell.py:2882] Step 12200 | Loss: 0.0273 | Spent: 10.5 secs | LR: 0.000160
I0415 02:11:44.290967 139745329543040 interactiveshell.py:2882] Step 12250 | Loss: 0.0302 | Spent: 10.5 secs | LR: 0.000160
I0415 02:11:54.926812 139745329543040 interactiveshell.py:2882] Step 12300 | Loss: 0.0378 | Spent: 10.6 secs | LR: 0.000160
I0415 02:12:05.701366 139745329543040 interactiveshell.py:2882] Step 12350 | Loss: 0.0255 | Spent: 10.8 secs | LR: 0.000159
I0415 02:12:16.648417 139745329543040 interactiveshell.py:2882] Step 12400 | Loss: 0.0338 | Spent: 10.9 secs | LR: 0.000159
I0415 02:12:27.890226 139745329543040 interactiveshell.py:2882] Step 12450 | Loss: 0.0650 | Spent: 11.2 secs | LR: 0.000158
I0415 02:12:39.366712 139745329543040 interactiveshell.py:2882] Step 12500 | Loss: 0.0885 | Spent: 11.5 secs | LR: 0.000158
Reading ../data/test.txt
I0415 02:13:15.664194 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.902
I0415 02:13:15.665432 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.902
Reading ../data/train.txt
I0415 02:13:23.611391 139745329543040 interactiveshell.py:2882] Step 12550 | Loss: 0.0086 | Spent: 44.2 secs | LR: 0.000158
I0415 02:13:33.764309 139745329543040 interactiveshell.py:2882] Step 12600 | Loss: 0.0091 | Spent: 10.2 secs | LR: 0.000157
I0415 02:13:44.034056 139745329543040 interactiveshell.py:2882] Step 12650 | Loss: 0.0135 | Spent: 10.3 secs | LR: 0.000157
I0415 02:13:54.294890 139745329543040 interactiveshell.py:2882] Step 12700 | Loss: 0.0116 | Spent: 10.3 secs | LR: 0.000156
I0415 02:14:04.566057 139745329543040 interactiveshell.py:2882] Step 12750 | Loss: 0.0236 | Spent: 10.3 secs | LR: 0.000156
I0415 02:14:14.869386 139745329543040 interactiveshell.py:2882] Step 12800 | Loss: 0.0323 | Spent: 10.3 secs | LR: 0.000156
I0415 02:14:25.187808 139745329543040 interactiveshell.py:2882] Step 12850 | Loss: 0.0214 | Spent: 10.3 secs | LR: 0.000155
I0415 02:14:35.562218 139745329543040 interactiveshell.py:2882] Step 12900 | Loss: 0.0151 | Spent: 10.4 secs | LR: 0.000155
I0415 02:14:45.987675 139745329543040 interactiveshell.py:2882] Step 12950 | Loss: 0.0205 | Spent: 10.4 secs | LR: 0.000154
I0415 02:14:56.464248 139745329543040 interactiveshell.py:2882] Step 13000 | Loss: 0.0225 | Spent: 10.5 secs | LR: 0.000154
I0415 02:15:07.016118 139745329543040 interactiveshell.py:2882] Step 13050 | Loss: 0.0434 | Spent: 10.5 secs | LR: 0.000154
I0415 02:15:17.680104 139745329543040 interactiveshell.py:2882] Step 13100 | Loss: 0.0238 | Spent: 10.7 secs | LR: 0.000153
I0415 02:15:28.503766 139745329543040 interactiveshell.py:2882] Step 13150 | Loss: 0.0426 | Spent: 10.8 secs | LR: 0.000153
I0415 02:15:39.537260 139745329543040 interactiveshell.py:2882] Step 13200 | Loss: 0.0577 | Spent: 11.0 secs | LR: 0.000152
I0415 02:15:50.881652 139745329543040 interactiveshell.py:2882] Step 13250 | Loss: 0.0472 | Spent: 11.3 secs | LR: 0.000152
Reading ../data/test.txt
I0415 02:16:35.033444 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.902
I0415 02:16:35.034986 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.902
Reading ../data/train.txt
I0415 02:16:36.512006 139745329543040 interactiveshell.py:2882] Step 13300 | Loss: 0.0553 | Spent: 45.6 secs | LR: 0.000152
I0415 02:16:46.600717 139745329543040 interactiveshell.py:2882] Step 13350 | Loss: 0.0235 | Spent: 10.1 secs | LR: 0.000151
I0415 02:16:56.805930 139745329543040 interactiveshell.py:2882] Step 13400 | Loss: 0.0110 | Spent: 10.2 secs | LR: 0.000151
I0415 02:17:07.024159 139745329543040 interactiveshell.py:2882] Step 13450 | Loss: 0.0409 | Spent: 10.2 secs | LR: 0.000150
I0415 02:17:17.272322 139745329543040 interactiveshell.py:2882] Step 13500 | Loss: 0.0060 | Spent: 10.2 secs | LR: 0.000150
I0415 02:17:27.590338 139745329543040 interactiveshell.py:2882] Step 13550 | Loss: 0.0210 | Spent: 10.3 secs | LR: 0.000150
I0415 02:17:37.881848 139745329543040 interactiveshell.py:2882] Step 13600 | Loss: 0.0216 | Spent: 10.3 secs | LR: 0.000149
I0415 02:17:48.274124 139745329543040 interactiveshell.py:2882] Step 13650 | Loss: 0.0174 | Spent: 10.4 secs | LR: 0.000149
I0415 02:17:58.676075 139745329543040 interactiveshell.py:2882] Step 13700 | Loss: 0.0183 | Spent: 10.4 secs | LR: 0.000149
I0415 02:18:09.128955 139745329543040 interactiveshell.py:2882] Step 13750 | Loss: 0.0122 | Spent: 10.5 secs | LR: 0.000148
I0415 02:18:19.629420 139745329543040 interactiveshell.py:2882] Step 13800 | Loss: 0.0252 | Spent: 10.5 secs | LR: 0.000148
I0415 02:18:30.226262 139745329543040 interactiveshell.py:2882] Step 13850 | Loss: 0.0292 | Spent: 10.6 secs | LR: 0.000147
I0415 02:18:40.935187 139745329543040 interactiveshell.py:2882] Step 13900 | Loss: 0.0288 | Spent: 10.7 secs | LR: 0.000147
I0415 02:18:51.830443 139745329543040 interactiveshell.py:2882] Step 13950 | Loss: 0.0248 | Spent: 10.9 secs | LR: 0.000147
I0415 02:19:02.960963 139745329543040 interactiveshell.py:2882] Step 14000 | Loss: 0.0413 | Spent: 11.1 secs | LR: 0.000146
I0415 02:19:14.350351 139745329543040 interactiveshell.py:2882] Step 14050 | Loss: 0.0385 | Spent: 11.4 secs | LR: 0.000146
Reading ../data/test.txt
I0415 02:19:53.915183 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.902
I0415 02:19:53.916664 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.902
Reading ../data/train.txt
I0415 02:19:59.036128 139745329543040 interactiveshell.py:2882] Step 14100 | Loss: 0.0064 | Spent: 44.7 secs | LR: 0.000146
I0415 02:20:09.157428 139745329543040 interactiveshell.py:2882] Step 14150 | Loss: 0.0097 | Spent: 10.1 secs | LR: 0.000145
I0415 02:20:19.402359 139745329543040 interactiveshell.py:2882] Step 14200 | Loss: 0.0058 | Spent: 10.2 secs | LR: 0.000145
I0415 02:20:29.662831 139745329543040 interactiveshell.py:2882] Step 14250 | Loss: 0.0255 | Spent: 10.3 secs | LR: 0.000144
I0415 02:20:39.927043 139745329543040 interactiveshell.py:2882] Step 14300 | Loss: 0.0140 | Spent: 10.3 secs | LR: 0.000144
I0415 02:20:50.270585 139745329543040 interactiveshell.py:2882] Step 14350 | Loss: 0.0071 | Spent: 10.3 secs | LR: 0.000144
I0415 02:21:00.613287 139745329543040 interactiveshell.py:2882] Step 14400 | Loss: 0.0125 | Spent: 10.3 secs | LR: 0.000143
I0415 02:21:10.960657 139745329543040 interactiveshell.py:2882] Step 14450 | Loss: 0.0161 | Spent: 10.3 secs | LR: 0.000143
I0415 02:21:21.380874 139745329543040 interactiveshell.py:2882] Step 14500 | Loss: 0.0187 | Spent: 10.4 secs | LR: 0.000143
I0415 02:21:31.879247 139745329543040 interactiveshell.py:2882] Step 14550 | Loss: 0.0169 | Spent: 10.5 secs | LR: 0.000142
I0415 02:21:42.477777 139745329543040 interactiveshell.py:2882] Step 14600 | Loss: 0.0140 | Spent: 10.6 secs | LR: 0.000142
I0415 02:21:53.121577 139745329543040 interactiveshell.py:2882] Step 14650 | Loss: 0.0290 | Spent: 10.6 secs | LR: 0.000142
I0415 02:22:03.905563 139745329543040 interactiveshell.py:2882] Step 14700 | Loss: 0.0331 | Spent: 10.8 secs | LR: 0.000141
I0415 02:22:14.886218 139745329543040 interactiveshell.py:2882] Step 14750 | Loss: 0.0298 | Spent: 11.0 secs | LR: 0.000141
I0415 02:22:26.156057 139745329543040 interactiveshell.py:2882] Step 14800 | Loss: 0.0292 | Spent: 11.3 secs | LR: 0.000140
I0415 02:22:37.639748 139745329543040 interactiveshell.py:2882] Step 14850 | Loss: 0.0401 | Spent: 11.5 secs | LR: 0.000140
Reading ../data/test.txt
I0415 02:23:13.565904 139745329543040 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.902
I0415 02:23:13.567181 139745329543040 interactiveshell.py:2882] Best Accuracy: 0.902
I0415 02:23:13.572681 139745329543040 interactiveshell.py:2882] Testing Accuracy not improved over 3 epochs, Early Stop