In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tf-nightly-2.0-preview
Requirement already satisfied: tf-nightly-2.0-preview in /usr/local/lib/python3.6/dist-packages (2.0.0.dev20190324)
Requirement already satisfied: tensorflow-estimator-2.0-preview in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0.dev2019032400)
Requirement already satisfied: tb-nightly<1.15.0a0,>=1.14.0a0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0a20190319)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.15.0)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.7)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.1.4)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.6)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (3.7.0)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.2.2)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.9)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.1.0)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.11.0)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.33.1)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (3.0.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (0.14.1)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tf-nightly-2.0-preview) (2.8.0)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tf-nightly-2.0-preview) (40.8.0)
In [3]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-dev20190324
GPU Enabled: False
In [0]:
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      label, text = line.split('\t')
      text = text.split(' ')
      x = [params['word2idx'].get(w, len(word2idx)) for w in text]
      if len(x) > params['max_len']:
        x = x[:params['max_len']]
      y = int(label)
      yield x, y


def dataset(is_training, params):
  _shapes = ([None], ())
  _types = (tf.int32, tf.int32)
  _pads = (0, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class FastText(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embedding = tf.Variable(np.load('../vocab/word.npy'), dtype=tf.float32)
    self.out_linear = tf.keras.layers.Dense(2)
    
  def call(self, inputs):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.cast(tf.sign(inputs), tf.float32)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    
    masked_mean = lambda x, mask: tf.reduce_sum(x * tf.expand_dims(mask, -1), 1) / tf.reduce_sum(mask, 1, keepdims=True)
    x = masked_mean(x, masks)
    
    x = self.out_linear(x)
    
    return x
In [0]:
params = {
  'vocab_path': '../vocab/word.txt',
  'train_path': '../data/train.txt',
  'test_path': '../data/test.txt',
  'num_samples': 25000,
  'num_labels': 2,
  'batch_size': 32,
  'max_len': 1000,
  'num_patience': 3,
  'lr': 3e-4,
}
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [8]:
word2idx = {}
with open(params['vocab_path']) as f:
  for i, line in enumerate(f):
    line = line.rstrip()
    word2idx[line] = i
params['word2idx'] = word2idx
params['vocab_size'] = len(word2idx) + 1


model = FastText(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.95)
optim = tf.keras.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = 0.

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for texts, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model(texts)
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
      loss = tf.reduce_mean(loss)

    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for texts, labels in dataset(is_training=False, params=params):
    logits = model(texts)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))  
  history_acc.append(acc)
  
  if acc > best_acc:
    best_acc = acc
    # you can save model here
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0325 05:40:48.209935 139718610032512 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:421: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('dense/kernel:0', TensorShape([300, 2])),
 ('dense/bias:0', TensorShape([2])),
 ('Variable:0', TensorShape([20599, 300]))]
Reading ../data/train.txt
I0325 05:40:58.481653 139718610032512 interactiveshell.py:2882] Step 0 | Loss: 0.6701 | Spent: 10.3 secs | LR: 0.000300
I0325 05:41:09.558297 139718610032512 interactiveshell.py:2882] Step 50 | Loss: 0.6668 | Spent: 11.1 secs | LR: 0.000299
I0325 05:41:19.896712 139718610032512 interactiveshell.py:2882] Step 100 | Loss: 0.6724 | Spent: 10.3 secs | LR: 0.000298
I0325 05:41:30.766979 139718610032512 interactiveshell.py:2882] Step 150 | Loss: 0.6490 | Spent: 10.9 secs | LR: 0.000298
I0325 05:41:41.175082 139718610032512 interactiveshell.py:2882] Step 200 | Loss: 0.6674 | Spent: 10.4 secs | LR: 0.000297
I0325 05:41:51.983973 139718610032512 interactiveshell.py:2882] Step 250 | Loss: 0.6279 | Spent: 10.8 secs | LR: 0.000296
I0325 05:42:03.113948 139718610032512 interactiveshell.py:2882] Step 300 | Loss: 0.6155 | Spent: 11.1 secs | LR: 0.000295
I0325 05:42:15.285057 139718610032512 interactiveshell.py:2882] Step 350 | Loss: 0.6192 | Spent: 12.2 secs | LR: 0.000295
I0325 05:42:26.869291 139718610032512 interactiveshell.py:2882] Step 400 | Loss: 0.5934 | Spent: 11.6 secs | LR: 0.000294
I0325 05:42:38.147674 139718610032512 interactiveshell.py:2882] Step 450 | Loss: 0.6566 | Spent: 11.3 secs | LR: 0.000293
I0325 05:42:49.316072 139718610032512 interactiveshell.py:2882] Step 500 | Loss: 0.5111 | Spent: 11.2 secs | LR: 0.000292
I0325 05:43:00.240890 139718610032512 interactiveshell.py:2882] Step 550 | Loss: 0.5592 | Spent: 10.9 secs | LR: 0.000292
I0325 05:43:10.180714 139718610032512 interactiveshell.py:2882] Step 600 | Loss: 0.5400 | Spent: 9.9 secs | LR: 0.000291
I0325 05:43:20.689426 139718610032512 interactiveshell.py:2882] Step 650 | Loss: 0.5161 | Spent: 10.5 secs | LR: 0.000290
I0325 05:43:32.175285 139718610032512 interactiveshell.py:2882] Step 700 | Loss: 0.5759 | Spent: 11.5 secs | LR: 0.000289
I0325 05:43:42.467468 139718610032512 interactiveshell.py:2882] Step 750 | Loss: 0.4771 | Spent: 10.3 secs | LR: 0.000289
Reading ../data/test.txt
I0325 05:44:06.186377 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.820
I0325 05:44:06.187939 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.820
Reading ../data/train.txt
I0325 05:44:19.473664 139718610032512 interactiveshell.py:2882] Step 800 | Loss: 0.5069 | Spent: 37.0 secs | LR: 0.000288
I0325 05:44:30.046933 139718610032512 interactiveshell.py:2882] Step 850 | Loss: 0.4046 | Spent: 10.6 secs | LR: 0.000287
I0325 05:44:40.988885 139718610032512 interactiveshell.py:2882] Step 900 | Loss: 0.4066 | Spent: 10.9 secs | LR: 0.000286
I0325 05:44:50.641188 139718610032512 interactiveshell.py:2882] Step 950 | Loss: 0.5095 | Spent: 9.7 secs | LR: 0.000286
I0325 05:45:00.573939 139718610032512 interactiveshell.py:2882] Step 1000 | Loss: 0.4379 | Spent: 9.9 secs | LR: 0.000285
I0325 05:45:10.783356 139718610032512 interactiveshell.py:2882] Step 1050 | Loss: 0.4684 | Spent: 10.2 secs | LR: 0.000284
I0325 05:45:21.045439 139718610032512 interactiveshell.py:2882] Step 1100 | Loss: 0.4039 | Spent: 10.3 secs | LR: 0.000284
I0325 05:45:32.025597 139718610032512 interactiveshell.py:2882] Step 1150 | Loss: 0.4501 | Spent: 11.0 secs | LR: 0.000283
I0325 05:45:43.046267 139718610032512 interactiveshell.py:2882] Step 1200 | Loss: 0.3702 | Spent: 11.0 secs | LR: 0.000282
I0325 05:45:53.301314 139718610032512 interactiveshell.py:2882] Step 1250 | Loss: 0.3812 | Spent: 10.3 secs | LR: 0.000281
I0325 05:46:04.306764 139718610032512 interactiveshell.py:2882] Step 1300 | Loss: 0.3935 | Spent: 11.0 secs | LR: 0.000281
I0325 05:46:15.238584 139718610032512 interactiveshell.py:2882] Step 1350 | Loss: 0.2966 | Spent: 10.9 secs | LR: 0.000280
I0325 05:46:25.339786 139718610032512 interactiveshell.py:2882] Step 1400 | Loss: 0.3430 | Spent: 10.1 secs | LR: 0.000279
I0325 05:46:35.872238 139718610032512 interactiveshell.py:2882] Step 1450 | Loss: 0.3251 | Spent: 10.5 secs | LR: 0.000278
I0325 05:46:46.648439 139718610032512 interactiveshell.py:2882] Step 1500 | Loss: 0.3745 | Spent: 10.8 secs | LR: 0.000278
I0325 05:46:57.240416 139718610032512 interactiveshell.py:2882] Step 1550 | Loss: 0.3630 | Spent: 10.6 secs | LR: 0.000277
Reading ../data/test.txt
I0325 05:47:16.231189 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.861
I0325 05:47:16.233038 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.861
Reading ../data/train.txt
I0325 05:47:33.083090 139718610032512 interactiveshell.py:2882] Step 1600 | Loss: 0.3440 | Spent: 35.8 secs | LR: 0.000276
I0325 05:47:43.532961 139718610032512 interactiveshell.py:2882] Step 1650 | Loss: 0.2493 | Spent: 10.4 secs | LR: 0.000276
I0325 05:47:54.171642 139718610032512 interactiveshell.py:2882] Step 1700 | Loss: 0.3428 | Spent: 10.6 secs | LR: 0.000275
I0325 05:48:05.042701 139718610032512 interactiveshell.py:2882] Step 1750 | Loss: 0.1931 | Spent: 10.9 secs | LR: 0.000274
I0325 05:48:15.468419 139718610032512 interactiveshell.py:2882] Step 1800 | Loss: 0.2831 | Spent: 10.4 secs | LR: 0.000274
I0325 05:48:25.158149 139718610032512 interactiveshell.py:2882] Step 1850 | Loss: 0.2170 | Spent: 9.7 secs | LR: 0.000273
I0325 05:48:34.944097 139718610032512 interactiveshell.py:2882] Step 1900 | Loss: 0.3759 | Spent: 9.8 secs | LR: 0.000272
I0325 05:48:45.212447 139718610032512 interactiveshell.py:2882] Step 1950 | Loss: 0.2352 | Spent: 10.3 secs | LR: 0.000271
I0325 05:48:55.321689 139718610032512 interactiveshell.py:2882] Step 2000 | Loss: 0.3186 | Spent: 10.1 secs | LR: 0.000271
I0325 05:49:05.578538 139718610032512 interactiveshell.py:2882] Step 2050 | Loss: 0.3086 | Spent: 10.3 secs | LR: 0.000270
I0325 05:49:16.992333 139718610032512 interactiveshell.py:2882] Step 2100 | Loss: 0.2137 | Spent: 11.4 secs | LR: 0.000269
I0325 05:49:27.608608 139718610032512 interactiveshell.py:2882] Step 2150 | Loss: 0.2644 | Spent: 10.6 secs | LR: 0.000269
I0325 05:49:38.542297 139718610032512 interactiveshell.py:2882] Step 2200 | Loss: 0.3077 | Spent: 10.9 secs | LR: 0.000268
I0325 05:49:48.091552 139718610032512 interactiveshell.py:2882] Step 2250 | Loss: 0.3030 | Spent: 9.5 secs | LR: 0.000267
I0325 05:49:57.486980 139718610032512 interactiveshell.py:2882] Step 2300 | Loss: 0.1433 | Spent: 9.4 secs | LR: 0.000267
Reading ../data/test.txt
I0325 05:50:22.440232 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.875
I0325 05:50:22.442029 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.875
Reading ../data/train.txt
I0325 05:50:32.957278 139718610032512 interactiveshell.py:2882] Step 2350 | Loss: 0.2249 | Spent: 35.5 secs | LR: 0.000266
I0325 05:50:42.290294 139718610032512 interactiveshell.py:2882] Step 2400 | Loss: 0.2052 | Spent: 9.3 secs | LR: 0.000265
I0325 05:50:51.942509 139718610032512 interactiveshell.py:2882] Step 2450 | Loss: 0.2150 | Spent: 9.7 secs | LR: 0.000265
I0325 05:51:02.348151 139718610032512 interactiveshell.py:2882] Step 2500 | Loss: 0.3230 | Spent: 10.4 secs | LR: 0.000264
I0325 05:51:12.879138 139718610032512 interactiveshell.py:2882] Step 2550 | Loss: 0.2002 | Spent: 10.5 secs | LR: 0.000263
I0325 05:51:23.074313 139718610032512 interactiveshell.py:2882] Step 2600 | Loss: 0.2430 | Spent: 10.2 secs | LR: 0.000263
I0325 05:51:34.005279 139718610032512 interactiveshell.py:2882] Step 2650 | Loss: 0.2252 | Spent: 10.9 secs | LR: 0.000262
I0325 05:51:44.359135 139718610032512 interactiveshell.py:2882] Step 2700 | Loss: 0.3523 | Spent: 10.4 secs | LR: 0.000261
I0325 05:51:54.737703 139718610032512 interactiveshell.py:2882] Step 2750 | Loss: 0.2514 | Spent: 10.4 secs | LR: 0.000261
I0325 05:52:05.203742 139718610032512 interactiveshell.py:2882] Step 2800 | Loss: 0.2397 | Spent: 10.5 secs | LR: 0.000260
I0325 05:52:16.059342 139718610032512 interactiveshell.py:2882] Step 2850 | Loss: 0.2541 | Spent: 10.9 secs | LR: 0.000259
I0325 05:52:27.594129 139718610032512 interactiveshell.py:2882] Step 2900 | Loss: 0.1929 | Spent: 11.5 secs | LR: 0.000259
I0325 05:52:37.936665 139718610032512 interactiveshell.py:2882] Step 2950 | Loss: 0.1922 | Spent: 10.3 secs | LR: 0.000258
I0325 05:52:47.938663 139718610032512 interactiveshell.py:2882] Step 3000 | Loss: 0.2836 | Spent: 10.0 secs | LR: 0.000257
I0325 05:52:58.075478 139718610032512 interactiveshell.py:2882] Step 3050 | Loss: 0.1982 | Spent: 10.1 secs | LR: 0.000257
I0325 05:53:08.658239 139718610032512 interactiveshell.py:2882] Step 3100 | Loss: 0.2018 | Spent: 10.6 secs | LR: 0.000256
Reading ../data/test.txt
I0325 05:53:29.504842 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.882
I0325 05:53:29.506863 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.882
Reading ../data/train.txt
I0325 05:53:43.557659 139718610032512 interactiveshell.py:2882] Step 3150 | Loss: 0.2781 | Spent: 34.9 secs | LR: 0.000255
I0325 05:53:53.572859 139718610032512 interactiveshell.py:2882] Step 3200 | Loss: 0.2885 | Spent: 10.0 secs | LR: 0.000255
I0325 05:54:03.365720 139718610032512 interactiveshell.py:2882] Step 3250 | Loss: 0.2111 | Spent: 9.8 secs | LR: 0.000254
I0325 05:54:13.660693 139718610032512 interactiveshell.py:2882] Step 3300 | Loss: 0.2524 | Spent: 10.3 secs | LR: 0.000253
I0325 05:54:23.926557 139718610032512 interactiveshell.py:2882] Step 3350 | Loss: 0.2838 | Spent: 10.3 secs | LR: 0.000253
I0325 05:54:33.485955 139718610032512 interactiveshell.py:2882] Step 3400 | Loss: 0.2043 | Spent: 9.6 secs | LR: 0.000252
I0325 05:54:43.135841 139718610032512 interactiveshell.py:2882] Step 3450 | Loss: 0.2353 | Spent: 9.6 secs | LR: 0.000251
I0325 05:54:53.918156 139718610032512 interactiveshell.py:2882] Step 3500 | Loss: 0.2717 | Spent: 10.8 secs | LR: 0.000251
I0325 05:55:04.429039 139718610032512 interactiveshell.py:2882] Step 3550 | Loss: 0.1849 | Spent: 10.5 secs | LR: 0.000250
I0325 05:55:15.173968 139718610032512 interactiveshell.py:2882] Step 3600 | Loss: 0.3239 | Spent: 10.7 secs | LR: 0.000249
I0325 05:55:25.557617 139718610032512 interactiveshell.py:2882] Step 3650 | Loss: 0.2080 | Spent: 10.4 secs | LR: 0.000249
I0325 05:55:36.158222 139718610032512 interactiveshell.py:2882] Step 3700 | Loss: 0.1554 | Spent: 10.6 secs | LR: 0.000248
I0325 05:55:46.998563 139718610032512 interactiveshell.py:2882] Step 3750 | Loss: 0.2187 | Spent: 10.8 secs | LR: 0.000248
I0325 05:55:57.896373 139718610032512 interactiveshell.py:2882] Step 3800 | Loss: 0.4366 | Spent: 10.9 secs | LR: 0.000247
I0325 05:56:08.133068 139718610032512 interactiveshell.py:2882] Step 3850 | Loss: 0.1159 | Spent: 10.2 secs | LR: 0.000246
I0325 05:56:18.630618 139718610032512 interactiveshell.py:2882] Step 3900 | Loss: 0.1903 | Spent: 10.5 secs | LR: 0.000246
Reading ../data/test.txt
I0325 05:56:37.245941 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.888
I0325 05:56:37.247787 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.888
Reading ../data/train.txt
I0325 05:56:56.237777 139718610032512 interactiveshell.py:2882] Step 3950 | Loss: 0.1763 | Spent: 37.6 secs | LR: 0.000245
I0325 05:57:06.386624 139718610032512 interactiveshell.py:2882] Step 4000 | Loss: 0.1579 | Spent: 10.1 secs | LR: 0.000244
I0325 05:57:16.599211 139718610032512 interactiveshell.py:2882] Step 4050 | Loss: 0.1808 | Spent: 10.2 secs | LR: 0.000244
I0325 05:57:26.576598 139718610032512 interactiveshell.py:2882] Step 4100 | Loss: 0.2651 | Spent: 10.0 secs | LR: 0.000243
I0325 05:57:36.785382 139718610032512 interactiveshell.py:2882] Step 4150 | Loss: 0.2642 | Spent: 10.2 secs | LR: 0.000242
I0325 05:57:46.979973 139718610032512 interactiveshell.py:2882] Step 4200 | Loss: 0.3984 | Spent: 10.2 secs | LR: 0.000242
I0325 05:57:57.010406 139718610032512 interactiveshell.py:2882] Step 4250 | Loss: 0.1782 | Spent: 10.0 secs | LR: 0.000241
I0325 05:58:07.155603 139718610032512 interactiveshell.py:2882] Step 4300 | Loss: 0.2634 | Spent: 10.1 secs | LR: 0.000241
I0325 05:58:16.792866 139718610032512 interactiveshell.py:2882] Step 4350 | Loss: 0.2040 | Spent: 9.6 secs | LR: 0.000240
I0325 05:58:26.961250 139718610032512 interactiveshell.py:2882] Step 4400 | Loss: 0.1629 | Spent: 10.2 secs | LR: 0.000239
I0325 05:58:36.786108 139718610032512 interactiveshell.py:2882] Step 4450 | Loss: 0.1370 | Spent: 9.8 secs | LR: 0.000239
I0325 05:58:46.504501 139718610032512 interactiveshell.py:2882] Step 4500 | Loss: 0.2469 | Spent: 9.7 secs | LR: 0.000238
I0325 05:58:56.456693 139718610032512 interactiveshell.py:2882] Step 4550 | Loss: 0.2774 | Spent: 10.0 secs | LR: 0.000238
I0325 05:59:06.742506 139718610032512 interactiveshell.py:2882] Step 4600 | Loss: 0.3549 | Spent: 10.3 secs | LR: 0.000237
I0325 05:59:18.051434 139718610032512 interactiveshell.py:2882] Step 4650 | Loss: 0.1169 | Spent: 11.3 secs | LR: 0.000236
Reading ../data/test.txt
I0325 05:59:42.924393 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.890
I0325 05:59:42.925986 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.890
Reading ../data/train.txt
I0325 05:59:53.627554 139718610032512 interactiveshell.py:2882] Step 4700 | Loss: 0.0990 | Spent: 35.6 secs | LR: 0.000236
I0325 06:00:04.114367 139718610032512 interactiveshell.py:2882] Step 4750 | Loss: 0.1279 | Spent: 10.5 secs | LR: 0.000235
I0325 06:00:14.704493 139718610032512 interactiveshell.py:2882] Step 4800 | Loss: 0.3301 | Spent: 10.6 secs | LR: 0.000235
I0325 06:00:24.698556 139718610032512 interactiveshell.py:2882] Step 4850 | Loss: 0.1693 | Spent: 10.0 secs | LR: 0.000234
I0325 06:00:35.180059 139718610032512 interactiveshell.py:2882] Step 4900 | Loss: 0.1469 | Spent: 10.5 secs | LR: 0.000233
I0325 06:00:45.492220 139718610032512 interactiveshell.py:2882] Step 4950 | Loss: 0.1999 | Spent: 10.3 secs | LR: 0.000233
I0325 06:00:55.316783 139718610032512 interactiveshell.py:2882] Step 5000 | Loss: 0.1757 | Spent: 9.8 secs | LR: 0.000232
I0325 06:01:05.518841 139718610032512 interactiveshell.py:2882] Step 5050 | Loss: 0.1335 | Spent: 10.2 secs | LR: 0.000232
I0325 06:01:15.673511 139718610032512 interactiveshell.py:2882] Step 5100 | Loss: 0.3028 | Spent: 10.2 secs | LR: 0.000231
I0325 06:01:27.129086 139718610032512 interactiveshell.py:2882] Step 5150 | Loss: 0.2494 | Spent: 11.5 secs | LR: 0.000230
I0325 06:01:38.342546 139718610032512 interactiveshell.py:2882] Step 5200 | Loss: 0.0802 | Spent: 11.2 secs | LR: 0.000230
I0325 06:01:48.684990 139718610032512 interactiveshell.py:2882] Step 5250 | Loss: 0.3423 | Spent: 10.3 secs | LR: 0.000229
I0325 06:01:59.098787 139718610032512 interactiveshell.py:2882] Step 5300 | Loss: 0.2449 | Spent: 10.4 secs | LR: 0.000229
I0325 06:02:08.633554 139718610032512 interactiveshell.py:2882] Step 5350 | Loss: 0.1497 | Spent: 9.5 secs | LR: 0.000228
I0325 06:02:18.588530 139718610032512 interactiveshell.py:2882] Step 5400 | Loss: 0.2000 | Spent: 10.0 secs | LR: 0.000227
I0325 06:02:28.385317 139718610032512 interactiveshell.py:2882] Step 5450 | Loss: 0.3344 | Spent: 9.8 secs | LR: 0.000227
Reading ../data/test.txt
I0325 06:02:49.056727 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 06:02:49.058781 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.890
Reading ../data/train.txt
I0325 06:03:04.144329 139718610032512 interactiveshell.py:2882] Step 5500 | Loss: 0.1715 | Spent: 35.8 secs | LR: 0.000226
I0325 06:03:13.656474 139718610032512 interactiveshell.py:2882] Step 5550 | Loss: 0.2048 | Spent: 9.5 secs | LR: 0.000226
I0325 06:03:23.096522 139718610032512 interactiveshell.py:2882] Step 5600 | Loss: 0.1238 | Spent: 9.4 secs | LR: 0.000225
I0325 06:03:33.690383 139718610032512 interactiveshell.py:2882] Step 5650 | Loss: 0.2395 | Spent: 10.6 secs | LR: 0.000225
I0325 06:03:43.295419 139718610032512 interactiveshell.py:2882] Step 5700 | Loss: 0.1472 | Spent: 9.6 secs | LR: 0.000224
I0325 06:03:52.992487 139718610032512 interactiveshell.py:2882] Step 5750 | Loss: 0.1501 | Spent: 9.7 secs | LR: 0.000223
I0325 06:04:03.175918 139718610032512 interactiveshell.py:2882] Step 5800 | Loss: 0.2129 | Spent: 10.2 secs | LR: 0.000223
I0325 06:04:12.599272 139718610032512 interactiveshell.py:2882] Step 5850 | Loss: 0.0812 | Spent: 9.4 secs | LR: 0.000222
I0325 06:04:23.250398 139718610032512 interactiveshell.py:2882] Step 5900 | Loss: 0.1706 | Spent: 10.6 secs | LR: 0.000222
I0325 06:04:34.506574 139718610032512 interactiveshell.py:2882] Step 5950 | Loss: 0.1207 | Spent: 11.3 secs | LR: 0.000221
I0325 06:04:45.388653 139718610032512 interactiveshell.py:2882] Step 6000 | Loss: 0.2667 | Spent: 10.9 secs | LR: 0.000221
I0325 06:04:55.686822 139718610032512 interactiveshell.py:2882] Step 6050 | Loss: 0.1277 | Spent: 10.3 secs | LR: 0.000220
I0325 06:05:06.098560 139718610032512 interactiveshell.py:2882] Step 6100 | Loss: 0.4015 | Spent: 10.4 secs | LR: 0.000219
I0325 06:05:15.778564 139718610032512 interactiveshell.py:2882] Step 6150 | Loss: 0.2697 | Spent: 9.7 secs | LR: 0.000219
I0325 06:05:26.079283 139718610032512 interactiveshell.py:2882] Step 6200 | Loss: 0.0856 | Spent: 10.3 secs | LR: 0.000218
I0325 06:05:36.768370 139718610032512 interactiveshell.py:2882] Step 6250 | Loss: 0.1285 | Spent: 10.7 secs | LR: 0.000218
Reading ../data/test.txt
I0325 06:05:54.352475 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.891
I0325 06:05:54.354124 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 06:06:12.378715 139718610032512 interactiveshell.py:2882] Step 6300 | Loss: 0.1386 | Spent: 35.6 secs | LR: 0.000217
I0325 06:06:22.837375 139718610032512 interactiveshell.py:2882] Step 6350 | Loss: 0.0640 | Spent: 10.5 secs | LR: 0.000217
I0325 06:06:33.067552 139718610032512 interactiveshell.py:2882] Step 6400 | Loss: 0.1661 | Spent: 10.2 secs | LR: 0.000216
I0325 06:06:43.968713 139718610032512 interactiveshell.py:2882] Step 6450 | Loss: 0.1753 | Spent: 10.9 secs | LR: 0.000215
I0325 06:06:55.334229 139718610032512 interactiveshell.py:2882] Step 6500 | Loss: 0.1068 | Spent: 11.4 secs | LR: 0.000215
I0325 06:07:06.967577 139718610032512 interactiveshell.py:2882] Step 6550 | Loss: 0.2264 | Spent: 11.6 secs | LR: 0.000214
I0325 06:07:17.709998 139718610032512 interactiveshell.py:2882] Step 6600 | Loss: 0.1893 | Spent: 10.7 secs | LR: 0.000214
I0325 06:07:28.132337 139718610032512 interactiveshell.py:2882] Step 6650 | Loss: 0.1774 | Spent: 10.4 secs | LR: 0.000213
I0325 06:07:38.328560 139718610032512 interactiveshell.py:2882] Step 6700 | Loss: 0.0571 | Spent: 10.2 secs | LR: 0.000213
I0325 06:07:48.800523 139718610032512 interactiveshell.py:2882] Step 6750 | Loss: 0.0597 | Spent: 10.5 secs | LR: 0.000212
I0325 06:07:59.642202 139718610032512 interactiveshell.py:2882] Step 6800 | Loss: 0.2345 | Spent: 10.8 secs | LR: 0.000212
I0325 06:08:09.609209 139718610032512 interactiveshell.py:2882] Step 6850 | Loss: 0.1213 | Spent: 10.0 secs | LR: 0.000211
I0325 06:08:19.823572 139718610032512 interactiveshell.py:2882] Step 6900 | Loss: 0.0927 | Spent: 10.2 secs | LR: 0.000211
I0325 06:08:30.251879 139718610032512 interactiveshell.py:2882] Step 6950 | Loss: 0.1429 | Spent: 10.4 secs | LR: 0.000210
I0325 06:08:40.482061 139718610032512 interactiveshell.py:2882] Step 7000 | Loss: 0.2718 | Spent: 10.2 secs | LR: 0.000210
Reading ../data/test.txt
I0325 06:09:04.346443 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 06:09:04.348273 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 06:09:16.236006 139718610032512 interactiveshell.py:2882] Step 7050 | Loss: 0.0926 | Spent: 35.8 secs | LR: 0.000209
I0325 06:09:27.322147 139718610032512 interactiveshell.py:2882] Step 7100 | Loss: 0.1254 | Spent: 11.1 secs | LR: 0.000208
I0325 06:09:37.430165 139718610032512 interactiveshell.py:2882] Step 7150 | Loss: 0.1127 | Spent: 10.1 secs | LR: 0.000208
I0325 06:09:48.001961 139718610032512 interactiveshell.py:2882] Step 7200 | Loss: 0.1470 | Spent: 10.6 secs | LR: 0.000207
I0325 06:09:58.969653 139718610032512 interactiveshell.py:2882] Step 7250 | Loss: 0.1730 | Spent: 11.0 secs | LR: 0.000207
I0325 06:10:08.293431 139718610032512 interactiveshell.py:2882] Step 7300 | Loss: 0.1173 | Spent: 9.3 secs | LR: 0.000206
I0325 06:10:17.556357 139718610032512 interactiveshell.py:2882] Step 7350 | Loss: 0.1423 | Spent: 9.3 secs | LR: 0.000206
I0325 06:10:26.985085 139718610032512 interactiveshell.py:2882] Step 7400 | Loss: 0.0671 | Spent: 9.4 secs | LR: 0.000205
I0325 06:10:36.946904 139718610032512 interactiveshell.py:2882] Step 7450 | Loss: 0.1352 | Spent: 10.0 secs | LR: 0.000205
I0325 06:10:47.716018 139718610032512 interactiveshell.py:2882] Step 7500 | Loss: 0.2188 | Spent: 10.8 secs | LR: 0.000204
I0325 06:10:58.488164 139718610032512 interactiveshell.py:2882] Step 7550 | Loss: 0.3065 | Spent: 10.8 secs | LR: 0.000204
I0325 06:11:08.563933 139718610032512 interactiveshell.py:2882] Step 7600 | Loss: 0.2044 | Spent: 10.1 secs | LR: 0.000203
I0325 06:11:18.743620 139718610032512 interactiveshell.py:2882] Step 7650 | Loss: 0.1410 | Spent: 10.2 secs | LR: 0.000203
I0325 06:11:29.704674 139718610032512 interactiveshell.py:2882] Step 7700 | Loss: 0.1491 | Spent: 11.0 secs | LR: 0.000202
I0325 06:11:39.982733 139718610032512 interactiveshell.py:2882] Step 7750 | Loss: 0.1379 | Spent: 10.3 secs | LR: 0.000202
I0325 06:11:50.682159 139718610032512 interactiveshell.py:2882] Step 7800 | Loss: 0.0731 | Spent: 10.7 secs | LR: 0.000201
Reading ../data/test.txt
I0325 06:12:11.378674 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.890
I0325 06:12:11.380410 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 06:12:30.088994 139718610032512 interactiveshell.py:2882] Step 7850 | Loss: 0.0400 | Spent: 39.4 secs | LR: 0.000201
I0325 06:12:40.505179 139718610032512 interactiveshell.py:2882] Step 7900 | Loss: 0.1249 | Spent: 10.4 secs | LR: 0.000200
I0325 06:12:51.090413 139718610032512 interactiveshell.py:2882] Step 7950 | Loss: 0.2133 | Spent: 10.6 secs | LR: 0.000200
I0325 06:13:01.574738 139718610032512 interactiveshell.py:2882] Step 8000 | Loss: 0.1047 | Spent: 10.5 secs | LR: 0.000199
I0325 06:13:12.387208 139718610032512 interactiveshell.py:2882] Step 8050 | Loss: 0.1596 | Spent: 10.8 secs | LR: 0.000199
I0325 06:13:22.351325 139718610032512 interactiveshell.py:2882] Step 8100 | Loss: 0.0902 | Spent: 10.0 secs | LR: 0.000198
I0325 06:13:32.117336 139718610032512 interactiveshell.py:2882] Step 8150 | Loss: 0.0905 | Spent: 9.8 secs | LR: 0.000198
I0325 06:13:42.242712 139718610032512 interactiveshell.py:2882] Step 8200 | Loss: 0.1575 | Spent: 10.1 secs | LR: 0.000197
I0325 06:13:51.872192 139718610032512 interactiveshell.py:2882] Step 8250 | Loss: 0.1277 | Spent: 9.6 secs | LR: 0.000196
I0325 06:14:01.941074 139718610032512 interactiveshell.py:2882] Step 8300 | Loss: 0.1471 | Spent: 10.1 secs | LR: 0.000196
I0325 06:14:12.380247 139718610032512 interactiveshell.py:2882] Step 8350 | Loss: 0.0946 | Spent: 10.4 secs | LR: 0.000195
I0325 06:14:22.465491 139718610032512 interactiveshell.py:2882] Step 8400 | Loss: 0.1215 | Spent: 10.1 secs | LR: 0.000195
I0325 06:14:33.454318 139718610032512 interactiveshell.py:2882] Step 8450 | Loss: 0.0650 | Spent: 11.0 secs | LR: 0.000194
I0325 06:14:44.735546 139718610032512 interactiveshell.py:2882] Step 8500 | Loss: 0.2785 | Spent: 11.3 secs | LR: 0.000194
I0325 06:14:54.937496 139718610032512 interactiveshell.py:2882] Step 8550 | Loss: 0.1946 | Spent: 10.2 secs | LR: 0.000193
I0325 06:15:05.811365 139718610032512 interactiveshell.py:2882] Step 8600 | Loss: 0.1968 | Spent: 10.9 secs | LR: 0.000193
Reading ../data/test.txt
I0325 06:15:23.408179 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 06:15:23.409836 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 06:15:43.979525 139718610032512 interactiveshell.py:2882] Step 8650 | Loss: 0.0735 | Spent: 38.2 secs | LR: 0.000192
I0325 06:15:55.711593 139718610032512 interactiveshell.py:2882] Step 8700 | Loss: 0.1110 | Spent: 11.7 secs | LR: 0.000192
I0325 06:16:06.119322 139718610032512 interactiveshell.py:2882] Step 8750 | Loss: 0.0813 | Spent: 10.4 secs | LR: 0.000192
I0325 06:16:16.769369 139718610032512 interactiveshell.py:2882] Step 8800 | Loss: 0.1598 | Spent: 10.6 secs | LR: 0.000191
I0325 06:16:27.090892 139718610032512 interactiveshell.py:2882] Step 8850 | Loss: 0.0735 | Spent: 10.3 secs | LR: 0.000191
I0325 06:16:37.944897 139718610032512 interactiveshell.py:2882] Step 8900 | Loss: 0.0948 | Spent: 10.9 secs | LR: 0.000190
I0325 06:16:48.069232 139718610032512 interactiveshell.py:2882] Step 8950 | Loss: 0.1420 | Spent: 10.1 secs | LR: 0.000190
I0325 06:16:58.921869 139718610032512 interactiveshell.py:2882] Step 9000 | Loss: 0.1025 | Spent: 10.9 secs | LR: 0.000189
I0325 06:17:09.327739 139718610032512 interactiveshell.py:2882] Step 9050 | Loss: 0.1514 | Spent: 10.4 secs | LR: 0.000189
I0325 06:17:19.503611 139718610032512 interactiveshell.py:2882] Step 9100 | Loss: 0.1284 | Spent: 10.2 secs | LR: 0.000188
I0325 06:17:30.180389 139718610032512 interactiveshell.py:2882] Step 9150 | Loss: 0.1901 | Spent: 10.7 secs | LR: 0.000188
I0325 06:17:40.696547 139718610032512 interactiveshell.py:2882] Step 9200 | Loss: 0.1133 | Spent: 10.5 secs | LR: 0.000187
I0325 06:17:51.269556 139718610032512 interactiveshell.py:2882] Step 9250 | Loss: 0.1220 | Spent: 10.6 secs | LR: 0.000187
I0325 06:18:01.677673 139718610032512 interactiveshell.py:2882] Step 9300 | Loss: 0.2745 | Spent: 10.4 secs | LR: 0.000186
I0325 06:18:12.384622 139718610032512 interactiveshell.py:2882] Step 9350 | Loss: 0.0438 | Spent: 10.7 secs | LR: 0.000186
Reading ../data/test.txt
I0325 06:18:37.457513 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 06:18:37.459676 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 06:18:50.904248 139718610032512 interactiveshell.py:2882] Step 9400 | Loss: 0.0863 | Spent: 38.5 secs | LR: 0.000185
I0325 06:19:01.613096 139718610032512 interactiveshell.py:2882] Step 9450 | Loss: 0.0424 | Spent: 10.7 secs | LR: 0.000185
I0325 06:19:12.034677 139718610032512 interactiveshell.py:2882] Step 9500 | Loss: 0.0738 | Spent: 10.4 secs | LR: 0.000184
I0325 06:19:23.019459 139718610032512 interactiveshell.py:2882] Step 9550 | Loss: 0.0799 | Spent: 11.0 secs | LR: 0.000184
I0325 06:19:32.859451 139718610032512 interactiveshell.py:2882] Step 9600 | Loss: 0.1087 | Spent: 9.8 secs | LR: 0.000183
I0325 06:19:42.338783 139718610032512 interactiveshell.py:2882] Step 9650 | Loss: 0.1658 | Spent: 9.5 secs | LR: 0.000183
I0325 06:19:52.775578 139718610032512 interactiveshell.py:2882] Step 9700 | Loss: 0.1610 | Spent: 10.4 secs | LR: 0.000182
I0325 06:20:02.830017 139718610032512 interactiveshell.py:2882] Step 9750 | Loss: 0.1023 | Spent: 10.1 secs | LR: 0.000182
I0325 06:20:13.001899 139718610032512 interactiveshell.py:2882] Step 9800 | Loss: 0.2030 | Spent: 10.2 secs | LR: 0.000181
I0325 06:20:22.950518 139718610032512 interactiveshell.py:2882] Step 9850 | Loss: 0.0574 | Spent: 9.9 secs | LR: 0.000181
I0325 06:20:33.273942 139718610032512 interactiveshell.py:2882] Step 9900 | Loss: 0.1159 | Spent: 10.3 secs | LR: 0.000181
I0325 06:20:43.242191 139718610032512 interactiveshell.py:2882] Step 9950 | Loss: 0.1310 | Spent: 10.0 secs | LR: 0.000180
I0325 06:20:54.288247 139718610032512 interactiveshell.py:2882] Step 10000 | Loss: 0.1061 | Spent: 11.0 secs | LR: 0.000180
I0325 06:21:05.371961 139718610032512 interactiveshell.py:2882] Step 10050 | Loss: 0.1048 | Spent: 11.1 secs | LR: 0.000179
I0325 06:21:16.106091 139718610032512 interactiveshell.py:2882] Step 10100 | Loss: 0.2133 | Spent: 10.7 secs | LR: 0.000179
I0325 06:21:27.382838 139718610032512 interactiveshell.py:2882] Step 10150 | Loss: 0.1181 | Spent: 11.3 secs | LR: 0.000178
Reading ../data/test.txt
I0325 06:21:47.600651 139718610032512 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 06:21:47.602356 139718610032512 interactiveshell.py:2882] Best Accuracy: 0.891
I0325 06:21:47.608852 139718610032512 interactiveshell.py:2882] Testing Accuracy not improved over 3 epochs, Early Stop