In [0]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [0]:
!pip install tf-nightly-2.0-preview
Requirement already satisfied: tf-nightly-2.0-preview in /usr/local/lib/python3.6/dist-packages (2.0.0.dev20190324)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (3.7.0)
Requirement already satisfied: tb-nightly<1.15.0a0,>=1.14.0a0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0a20190319)
Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.1.4)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.6)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.15.0)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.2.2)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.11.0)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.33.1)
Requirement already satisfied: tensorflow-estimator-2.0-preview in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0.dev2019032400)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.7)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.1.0)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.9)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tf-nightly-2.0-preview) (40.8.0)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (0.14.1)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (3.0.1)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tf-nightly-2.0-preview) (2.8.0)
In [0]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-dev20190324
GPU Enabled: False
In [0]:
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      label, text = line.split('\t')
      text = text.split(' ')
      x = [params['word2idx'].get(w, len(word2idx)) for w in text]
      if len(x) > params['max_len']:
        x = x[:params['max_len']]
      y = int(label)
      yield x, y


def dataset(is_training, params):
  _shapes = ([None], ())
  _types = (tf.int32, tf.int32)
  _pads = (0, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class KernelAttentivePooling(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.kernel = tf.keras.layers.Conv1D(filters=1,
                                         kernel_size=params['kernel_size'],
                                         padding='same',
                                         activation=tf.tanh,
                                         use_bias=False)

  
  def call(self, inputs, training=False):
    x, masks = inputs
    # alignment
    align = tf.squeeze(self.kernel(self.dropout(x, training=training)), -1)
    # masking
    paddings = tf.fill(tf.shape(align), float('-inf'))
    align = tf.where(tf.equal(masks, 0), paddings, align)
    # probability
    align = tf.nn.softmax(align)
    align = tf.expand_dims(align, -1)
    # weighted sum
    return tf.squeeze(tf.matmul(x, align, transpose_a=True), -1)
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    
    self.embedding = tf.Variable(np.load('../vocab/word.npy'),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    
    self.attentive_pooling = KernelAttentivePooling(params)
    
    self.out_linear = tf.keras.layers.Dense(2)
  
  
  def call(self, inputs, training=False):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    
    x = self.attentive_pooling((x, masks), training=training)
    
    x = self.out_linear(x)
    
    return x
In [0]:
params = {
  'vocab_path': '../vocab/word.txt',
  'train_path': '../data/train.txt',
  'test_path': '../data/test.txt',
  'num_samples': 25000,
  'num_labels': 2,
  'batch_size': 32,
  'max_len': 1000,
  'dropout_rate': 0.2,
  'kernel_size': 5,
  'num_patience': 3,
  'lr': 3e-4,
}
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [0]:
word2idx = {}
with open(params['vocab_path']) as f:
  for i, line in enumerate(f):
    line = line.rstrip()
    word2idx[line] = i
params['word2idx'] = word2idx
params['vocab_size'] = len(word2idx) + 1


model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.95)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for texts, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model(texts, training=True)
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
      loss = tf.reduce_mean(loss)
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for texts, labels in dataset(is_training=False, params=params):
    logits = model(texts, training=False)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))
  history_acc.append(acc)
  
  if acc > best_acc:
    best_acc = acc
    # you can save model here
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0325 04:18:18.007171 140330694072192 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:421: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('kernel_attentive_pooling/conv1d/kernel:0', TensorShape([5, 300, 1])),
 ('dense/kernel:0', TensorShape([300, 2])),
 ('dense/bias:0', TensorShape([2])),
 ('pretrained_embedding:0', TensorShape([20599, 300]))]
Reading ../data/train.txt
I0325 04:18:27.617407 140330694072192 interactiveshell.py:2882] Step 0 | Loss: 0.7042 | Spent: 9.6 secs | LR: 0.000300
I0325 04:18:55.245556 140330694072192 interactiveshell.py:2882] Step 50 | Loss: 0.6714 | Spent: 27.6 secs | LR: 0.000299
I0325 04:19:24.115016 140330694072192 interactiveshell.py:2882] Step 100 | Loss: 0.6572 | Spent: 28.9 secs | LR: 0.000298
I0325 04:19:53.799033 140330694072192 interactiveshell.py:2882] Step 150 | Loss: 0.6297 | Spent: 29.7 secs | LR: 0.000298
I0325 04:20:21.545069 140330694072192 interactiveshell.py:2882] Step 200 | Loss: 0.6028 | Spent: 27.7 secs | LR: 0.000297
I0325 04:20:51.325325 140330694072192 interactiveshell.py:2882] Step 250 | Loss: 0.6247 | Spent: 29.8 secs | LR: 0.000296
I0325 04:21:19.237365 140330694072192 interactiveshell.py:2882] Step 300 | Loss: 0.5696 | Spent: 27.9 secs | LR: 0.000295
I0325 04:21:47.601436 140330694072192 interactiveshell.py:2882] Step 350 | Loss: 0.5103 | Spent: 28.4 secs | LR: 0.000295
I0325 04:22:16.564269 140330694072192 interactiveshell.py:2882] Step 400 | Loss: 0.4488 | Spent: 29.0 secs | LR: 0.000294
I0325 04:22:45.286784 140330694072192 interactiveshell.py:2882] Step 450 | Loss: 0.4662 | Spent: 28.7 secs | LR: 0.000293
I0325 04:23:15.144928 140330694072192 interactiveshell.py:2882] Step 500 | Loss: 0.4065 | Spent: 29.9 secs | LR: 0.000292
I0325 04:23:44.519619 140330694072192 interactiveshell.py:2882] Step 550 | Loss: 0.4623 | Spent: 29.4 secs | LR: 0.000292
I0325 04:24:13.250108 140330694072192 interactiveshell.py:2882] Step 600 | Loss: 0.3426 | Spent: 28.7 secs | LR: 0.000291
I0325 04:24:43.136655 140330694072192 interactiveshell.py:2882] Step 650 | Loss: 0.3684 | Spent: 29.9 secs | LR: 0.000290
I0325 04:25:11.481082 140330694072192 interactiveshell.py:2882] Step 700 | Loss: 0.3446 | Spent: 28.3 secs | LR: 0.000289
I0325 04:25:40.708647 140330694072192 interactiveshell.py:2882] Step 750 | Loss: 0.3957 | Spent: 29.2 secs | LR: 0.000289
Reading ../data/test.txt
I0325 04:26:31.839523 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.876
I0325 04:26:31.841579 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.876
Reading ../data/train.txt
I0325 04:26:52.023116 140330694072192 interactiveshell.py:2882] Step 800 | Loss: 0.3262 | Spent: 71.3 secs | LR: 0.000288
I0325 04:27:21.656743 140330694072192 interactiveshell.py:2882] Step 850 | Loss: 0.3295 | Spent: 29.6 secs | LR: 0.000287
I0325 04:27:51.161322 140330694072192 interactiveshell.py:2882] Step 900 | Loss: 0.2990 | Spent: 29.5 secs | LR: 0.000286
I0325 04:28:19.435937 140330694072192 interactiveshell.py:2882] Step 950 | Loss: 0.4632 | Spent: 28.3 secs | LR: 0.000286
I0325 04:28:47.684500 140330694072192 interactiveshell.py:2882] Step 1000 | Loss: 0.3478 | Spent: 28.2 secs | LR: 0.000285
I0325 04:29:16.799108 140330694072192 interactiveshell.py:2882] Step 1050 | Loss: 0.2490 | Spent: 29.1 secs | LR: 0.000284
I0325 04:29:46.280645 140330694072192 interactiveshell.py:2882] Step 1100 | Loss: 0.3068 | Spent: 29.5 secs | LR: 0.000284
I0325 04:30:15.008606 140330694072192 interactiveshell.py:2882] Step 1150 | Loss: 0.2527 | Spent: 28.7 secs | LR: 0.000283
I0325 04:30:44.387171 140330694072192 interactiveshell.py:2882] Step 1200 | Loss: 0.3291 | Spent: 29.4 secs | LR: 0.000282
I0325 04:31:14.287755 140330694072192 interactiveshell.py:2882] Step 1250 | Loss: 0.2747 | Spent: 29.9 secs | LR: 0.000281
I0325 04:31:44.106447 140330694072192 interactiveshell.py:2882] Step 1300 | Loss: 0.1477 | Spent: 29.8 secs | LR: 0.000281
I0325 04:32:13.674831 140330694072192 interactiveshell.py:2882] Step 1350 | Loss: 0.2372 | Spent: 29.6 secs | LR: 0.000280
I0325 04:32:43.321360 140330694072192 interactiveshell.py:2882] Step 1400 | Loss: 0.2587 | Spent: 29.6 secs | LR: 0.000279
I0325 04:33:13.261599 140330694072192 interactiveshell.py:2882] Step 1450 | Loss: 0.2453 | Spent: 29.9 secs | LR: 0.000278
I0325 04:33:42.255184 140330694072192 interactiveshell.py:2882] Step 1500 | Loss: 0.2178 | Spent: 29.0 secs | LR: 0.000278
I0325 04:34:12.410470 140330694072192 interactiveshell.py:2882] Step 1550 | Loss: 0.3409 | Spent: 30.2 secs | LR: 0.000277
Reading ../data/test.txt
I0325 04:34:53.803978 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.896
I0325 04:34:53.805620 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.896
Reading ../data/train.txt
I0325 04:35:24.342016 140330694072192 interactiveshell.py:2882] Step 1600 | Loss: 0.1590 | Spent: 71.9 secs | LR: 0.000276
I0325 04:35:55.244550 140330694072192 interactiveshell.py:2882] Step 1650 | Loss: 0.1463 | Spent: 30.9 secs | LR: 0.000276
I0325 04:36:27.883532 140330694072192 interactiveshell.py:2882] Step 1700 | Loss: 0.2360 | Spent: 32.6 secs | LR: 0.000275
I0325 04:37:02.287747 140330694072192 interactiveshell.py:2882] Step 1750 | Loss: 0.2257 | Spent: 34.4 secs | LR: 0.000274
I0325 04:37:36.157027 140330694072192 interactiveshell.py:2882] Step 1800 | Loss: 0.1617 | Spent: 33.9 secs | LR: 0.000274
I0325 04:38:07.594483 140330694072192 interactiveshell.py:2882] Step 1850 | Loss: 0.3359 | Spent: 31.4 secs | LR: 0.000273
I0325 04:38:36.154201 140330694072192 interactiveshell.py:2882] Step 1900 | Loss: 0.1758 | Spent: 28.6 secs | LR: 0.000272
I0325 04:39:04.592441 140330694072192 interactiveshell.py:2882] Step 1950 | Loss: 0.2434 | Spent: 28.4 secs | LR: 0.000271
I0325 04:39:33.951900 140330694072192 interactiveshell.py:2882] Step 2000 | Loss: 0.1302 | Spent: 29.4 secs | LR: 0.000271
I0325 04:40:02.992082 140330694072192 interactiveshell.py:2882] Step 2050 | Loss: 0.2821 | Spent: 29.0 secs | LR: 0.000270
I0325 04:40:32.695911 140330694072192 interactiveshell.py:2882] Step 2100 | Loss: 0.1704 | Spent: 29.7 secs | LR: 0.000269
I0325 04:41:01.571173 140330694072192 interactiveshell.py:2882] Step 2150 | Loss: 0.3067 | Spent: 28.9 secs | LR: 0.000269
I0325 04:41:31.412907 140330694072192 interactiveshell.py:2882] Step 2200 | Loss: 0.2309 | Spent: 29.8 secs | LR: 0.000268
I0325 04:42:00.100025 140330694072192 interactiveshell.py:2882] Step 2250 | Loss: 0.2013 | Spent: 28.7 secs | LR: 0.000267
I0325 04:42:27.922711 140330694072192 interactiveshell.py:2882] Step 2300 | Loss: 0.1324 | Spent: 27.8 secs | LR: 0.000267
Reading ../data/test.txt
I0325 04:43:28.123033 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.903
I0325 04:43:28.124924 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.903
Reading ../data/train.txt
I0325 04:43:39.884716 140330694072192 interactiveshell.py:2882] Step 2350 | Loss: 0.1939 | Spent: 72.0 secs | LR: 0.000266
I0325 04:44:09.545179 140330694072192 interactiveshell.py:2882] Step 2400 | Loss: 0.3061 | Spent: 29.7 secs | LR: 0.000265
I0325 04:44:41.297590 140330694072192 interactiveshell.py:2882] Step 2450 | Loss: 0.2629 | Spent: 31.8 secs | LR: 0.000265
I0325 04:45:11.319201 140330694072192 interactiveshell.py:2882] Step 2500 | Loss: 0.1109 | Spent: 30.0 secs | LR: 0.000264
I0325 04:45:41.208551 140330694072192 interactiveshell.py:2882] Step 2550 | Loss: 0.2012 | Spent: 29.9 secs | LR: 0.000263
I0325 04:46:12.159189 140330694072192 interactiveshell.py:2882] Step 2600 | Loss: 0.3378 | Spent: 30.9 secs | LR: 0.000263
I0325 04:46:41.796611 140330694072192 interactiveshell.py:2882] Step 2650 | Loss: 0.1594 | Spent: 29.6 secs | LR: 0.000262
I0325 04:47:10.799943 140330694072192 interactiveshell.py:2882] Step 2700 | Loss: 0.2644 | Spent: 29.0 secs | LR: 0.000261
I0325 04:47:40.714452 140330694072192 interactiveshell.py:2882] Step 2750 | Loss: 0.2689 | Spent: 29.9 secs | LR: 0.000261
I0325 04:48:08.741296 140330694072192 interactiveshell.py:2882] Step 2800 | Loss: 0.1168 | Spent: 28.0 secs | LR: 0.000260
I0325 04:48:38.268350 140330694072192 interactiveshell.py:2882] Step 2850 | Loss: 0.0825 | Spent: 29.5 secs | LR: 0.000259
I0325 04:49:08.728992 140330694072192 interactiveshell.py:2882] Step 2900 | Loss: 0.2478 | Spent: 30.5 secs | LR: 0.000259
I0325 04:49:37.994164 140330694072192 interactiveshell.py:2882] Step 2950 | Loss: 0.1043 | Spent: 29.3 secs | LR: 0.000258
I0325 04:50:06.809506 140330694072192 interactiveshell.py:2882] Step 3000 | Loss: 0.3487 | Spent: 28.8 secs | LR: 0.000257
I0325 04:50:35.155446 140330694072192 interactiveshell.py:2882] Step 3050 | Loss: 0.3840 | Spent: 28.3 secs | LR: 0.000257
I0325 04:51:03.959428 140330694072192 interactiveshell.py:2882] Step 3100 | Loss: 0.2698 | Spent: 28.8 secs | LR: 0.000256
Reading ../data/test.txt
I0325 04:51:53.697185 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.906
I0325 04:51:53.698982 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.906
Reading ../data/train.txt
I0325 04:52:16.065910 140330694072192 interactiveshell.py:2882] Step 3150 | Loss: 0.2079 | Spent: 72.1 secs | LR: 0.000255
I0325 04:52:45.855844 140330694072192 interactiveshell.py:2882] Step 3200 | Loss: 0.3299 | Spent: 29.8 secs | LR: 0.000255
I0325 04:53:15.119430 140330694072192 interactiveshell.py:2882] Step 3250 | Loss: 0.1486 | Spent: 29.3 secs | LR: 0.000254
I0325 04:53:45.088431 140330694072192 interactiveshell.py:2882] Step 3300 | Loss: 0.1204 | Spent: 30.0 secs | LR: 0.000253
I0325 04:54:14.526267 140330694072192 interactiveshell.py:2882] Step 3350 | Loss: 0.0899 | Spent: 29.4 secs | LR: 0.000253
I0325 04:54:43.608558 140330694072192 interactiveshell.py:2882] Step 3400 | Loss: 0.1187 | Spent: 29.1 secs | LR: 0.000252
I0325 04:55:14.940479 140330694072192 interactiveshell.py:2882] Step 3450 | Loss: 0.1133 | Spent: 31.3 secs | LR: 0.000251
I0325 04:55:47.136855 140330694072192 interactiveshell.py:2882] Step 3500 | Loss: 0.1360 | Spent: 32.2 secs | LR: 0.000251
I0325 04:56:20.534447 140330694072192 interactiveshell.py:2882] Step 3550 | Loss: 0.1168 | Spent: 33.4 secs | LR: 0.000250
I0325 04:56:54.858688 140330694072192 interactiveshell.py:2882] Step 3600 | Loss: 0.1274 | Spent: 34.3 secs | LR: 0.000249
I0325 04:57:28.655448 140330694072192 interactiveshell.py:2882] Step 3650 | Loss: 0.1557 | Spent: 33.8 secs | LR: 0.000249
I0325 04:58:00.197962 140330694072192 interactiveshell.py:2882] Step 3700 | Loss: 0.0966 | Spent: 31.5 secs | LR: 0.000248
I0325 04:58:28.659495 140330694072192 interactiveshell.py:2882] Step 3750 | Loss: 0.1232 | Spent: 28.5 secs | LR: 0.000248
I0325 04:59:01.139638 140330694072192 interactiveshell.py:2882] Step 3800 | Loss: 0.3250 | Spent: 32.5 secs | LR: 0.000247
I0325 04:59:32.428555 140330694072192 interactiveshell.py:2882] Step 3850 | Loss: 0.0907 | Spent: 31.3 secs | LR: 0.000246
I0325 05:00:07.156657 140330694072192 interactiveshell.py:2882] Step 3900 | Loss: 0.1368 | Spent: 34.7 secs | LR: 0.000246
Reading ../data/test.txt
I0325 05:00:51.090763 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.907
I0325 05:00:51.092947 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.907
Reading ../data/train.txt
I0325 05:01:28.670997 140330694072192 interactiveshell.py:2882] Step 3950 | Loss: 0.1046 | Spent: 81.5 secs | LR: 0.000245
I0325 05:02:03.556692 140330694072192 interactiveshell.py:2882] Step 4000 | Loss: 0.1263 | Spent: 34.9 secs | LR: 0.000244
I0325 05:02:32.945419 140330694072192 interactiveshell.py:2882] Step 4050 | Loss: 0.0839 | Spent: 29.4 secs | LR: 0.000244
I0325 05:03:03.229538 140330694072192 interactiveshell.py:2882] Step 4100 | Loss: 0.1668 | Spent: 30.3 secs | LR: 0.000243
I0325 05:03:31.903601 140330694072192 interactiveshell.py:2882] Step 4150 | Loss: 0.0671 | Spent: 28.7 secs | LR: 0.000242
I0325 05:04:00.518083 140330694072192 interactiveshell.py:2882] Step 4200 | Loss: 0.2339 | Spent: 28.6 secs | LR: 0.000242
I0325 05:04:29.279020 140330694072192 interactiveshell.py:2882] Step 4250 | Loss: 0.0988 | Spent: 28.8 secs | LR: 0.000241
I0325 05:04:58.801181 140330694072192 interactiveshell.py:2882] Step 4300 | Loss: 0.0822 | Spent: 29.5 secs | LR: 0.000241
I0325 05:05:27.468344 140330694072192 interactiveshell.py:2882] Step 4350 | Loss: 0.0849 | Spent: 28.7 secs | LR: 0.000240
I0325 05:05:56.228021 140330694072192 interactiveshell.py:2882] Step 4400 | Loss: 0.0915 | Spent: 28.8 secs | LR: 0.000239
I0325 05:06:24.975781 140330694072192 interactiveshell.py:2882] Step 4450 | Loss: 0.1477 | Spent: 28.7 secs | LR: 0.000239
I0325 05:06:54.983189 140330694072192 interactiveshell.py:2882] Step 4500 | Loss: 0.2174 | Spent: 30.0 secs | LR: 0.000238
I0325 05:07:22.352818 140330694072192 interactiveshell.py:2882] Step 4550 | Loss: 0.0656 | Spent: 27.4 secs | LR: 0.000238
I0325 05:07:55.652014 140330694072192 interactiveshell.py:2882] Step 4600 | Loss: 0.1009 | Spent: 33.3 secs | LR: 0.000237
I0325 05:08:31.159040 140330694072192 interactiveshell.py:2882] Step 4650 | Loss: 0.1334 | Spent: 35.5 secs | LR: 0.000236
Reading ../data/test.txt
I0325 05:09:37.111234 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.907
I0325 05:09:37.113339 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.907
Reading ../data/train.txt
I0325 05:09:51.793969 140330694072192 interactiveshell.py:2882] Step 4700 | Loss: 0.0751 | Spent: 80.6 secs | LR: 0.000236
I0325 05:10:25.016329 140330694072192 interactiveshell.py:2882] Step 4750 | Loss: 0.0546 | Spent: 33.2 secs | LR: 0.000235
I0325 05:10:58.031346 140330694072192 interactiveshell.py:2882] Step 4800 | Loss: 0.0469 | Spent: 33.0 secs | LR: 0.000235
I0325 05:11:33.758636 140330694072192 interactiveshell.py:2882] Step 4850 | Loss: 0.0831 | Spent: 35.7 secs | LR: 0.000234
I0325 05:12:09.260271 140330694072192 interactiveshell.py:2882] Step 4900 | Loss: 0.0723 | Spent: 35.5 secs | LR: 0.000233
I0325 05:12:42.891066 140330694072192 interactiveshell.py:2882] Step 4950 | Loss: 0.0667 | Spent: 33.6 secs | LR: 0.000233
I0325 05:13:15.943295 140330694072192 interactiveshell.py:2882] Step 5000 | Loss: 0.0955 | Spent: 33.1 secs | LR: 0.000232
I0325 05:13:49.567934 140330694072192 interactiveshell.py:2882] Step 5050 | Loss: 0.0731 | Spent: 33.6 secs | LR: 0.000232
I0325 05:14:22.177613 140330694072192 interactiveshell.py:2882] Step 5100 | Loss: 0.1581 | Spent: 32.6 secs | LR: 0.000231
I0325 05:14:54.455900 140330694072192 interactiveshell.py:2882] Step 5150 | Loss: 0.0567 | Spent: 32.3 secs | LR: 0.000230
I0325 05:15:29.314813 140330694072192 interactiveshell.py:2882] Step 5200 | Loss: 0.3259 | Spent: 34.9 secs | LR: 0.000230
I0325 05:16:03.123210 140330694072192 interactiveshell.py:2882] Step 5250 | Loss: 0.0553 | Spent: 33.8 secs | LR: 0.000229
I0325 05:16:37.055830 140330694072192 interactiveshell.py:2882] Step 5300 | Loss: 0.0498 | Spent: 33.9 secs | LR: 0.000229
I0325 05:17:10.565616 140330694072192 interactiveshell.py:2882] Step 5350 | Loss: 0.0881 | Spent: 33.5 secs | LR: 0.000228
I0325 05:17:43.403454 140330694072192 interactiveshell.py:2882] Step 5400 | Loss: 0.1851 | Spent: 32.8 secs | LR: 0.000227
I0325 05:18:18.359013 140330694072192 interactiveshell.py:2882] Step 5450 | Loss: 0.0637 | Spent: 35.0 secs | LR: 0.000227
Reading ../data/test.txt
I0325 05:19:11.393041 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.906
I0325 05:19:11.394832 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.907
Reading ../data/train.txt
I0325 05:19:38.998790 140330694072192 interactiveshell.py:2882] Step 5500 | Loss: 0.1874 | Spent: 80.6 secs | LR: 0.000226
I0325 05:20:11.357097 140330694072192 interactiveshell.py:2882] Step 5550 | Loss: 0.0192 | Spent: 32.4 secs | LR: 0.000226
I0325 05:20:48.253049 140330694072192 interactiveshell.py:2882] Step 5600 | Loss: 0.1799 | Spent: 36.9 secs | LR: 0.000225
I0325 05:21:23.102731 140330694072192 interactiveshell.py:2882] Step 5650 | Loss: 0.0644 | Spent: 34.8 secs | LR: 0.000225
I0325 05:21:57.173828 140330694072192 interactiveshell.py:2882] Step 5700 | Loss: 0.0460 | Spent: 34.1 secs | LR: 0.000224
I0325 05:22:29.483545 140330694072192 interactiveshell.py:2882] Step 5750 | Loss: 0.0315 | Spent: 32.3 secs | LR: 0.000223
I0325 05:23:01.896551 140330694072192 interactiveshell.py:2882] Step 5800 | Loss: 0.0725 | Spent: 32.4 secs | LR: 0.000223
I0325 05:23:34.944211 140330694072192 interactiveshell.py:2882] Step 5850 | Loss: 0.0424 | Spent: 33.0 secs | LR: 0.000222
I0325 05:24:09.773103 140330694072192 interactiveshell.py:2882] Step 5900 | Loss: 0.0786 | Spent: 34.8 secs | LR: 0.000222
I0325 05:24:42.766442 140330694072192 interactiveshell.py:2882] Step 5950 | Loss: 0.0368 | Spent: 33.0 secs | LR: 0.000221
I0325 05:25:16.346496 140330694072192 interactiveshell.py:2882] Step 6000 | Loss: 0.1223 | Spent: 33.6 secs | LR: 0.000221
I0325 05:25:50.740726 140330694072192 interactiveshell.py:2882] Step 6050 | Loss: 0.1196 | Spent: 34.4 secs | LR: 0.000220
I0325 05:26:25.337159 140330694072192 interactiveshell.py:2882] Step 6100 | Loss: 0.1388 | Spent: 34.6 secs | LR: 0.000219
I0325 05:26:58.268901 140330694072192 interactiveshell.py:2882] Step 6150 | Loss: 0.0608 | Spent: 32.9 secs | LR: 0.000219
I0325 05:27:30.828666 140330694072192 interactiveshell.py:2882] Step 6200 | Loss: 0.1745 | Spent: 32.6 secs | LR: 0.000218
I0325 05:28:05.201857 140330694072192 interactiveshell.py:2882] Step 6250 | Loss: 0.0282 | Spent: 34.4 secs | LR: 0.000218
Reading ../data/test.txt
I0325 05:28:47.158318 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.905
I0325 05:28:47.160113 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.907
Reading ../data/train.txt
I0325 05:29:25.190452 140330694072192 interactiveshell.py:2882] Step 6300 | Loss: 0.0254 | Spent: 80.0 secs | LR: 0.000217
I0325 05:29:56.447685 140330694072192 interactiveshell.py:2882] Step 6350 | Loss: 0.0666 | Spent: 31.3 secs | LR: 0.000217
I0325 05:30:29.869020 140330694072192 interactiveshell.py:2882] Step 6400 | Loss: 0.0528 | Spent: 33.4 secs | LR: 0.000216
I0325 05:31:02.201377 140330694072192 interactiveshell.py:2882] Step 6450 | Loss: 0.0655 | Spent: 32.3 secs | LR: 0.000215
I0325 05:31:35.251884 140330694072192 interactiveshell.py:2882] Step 6500 | Loss: 0.0687 | Spent: 33.0 secs | LR: 0.000215
I0325 05:32:07.177057 140330694072192 interactiveshell.py:2882] Step 6550 | Loss: 0.0476 | Spent: 31.9 secs | LR: 0.000214
I0325 05:32:37.499593 140330694072192 interactiveshell.py:2882] Step 6600 | Loss: 0.0488 | Spent: 30.3 secs | LR: 0.000214
I0325 05:33:10.877472 140330694072192 interactiveshell.py:2882] Step 6650 | Loss: 0.0737 | Spent: 33.4 secs | LR: 0.000213
I0325 05:33:40.015162 140330694072192 interactiveshell.py:2882] Step 6700 | Loss: 0.0636 | Spent: 29.1 secs | LR: 0.000213
I0325 05:34:12.635749 140330694072192 interactiveshell.py:2882] Step 6750 | Loss: 0.3841 | Spent: 32.6 secs | LR: 0.000212
I0325 05:34:46.252620 140330694072192 interactiveshell.py:2882] Step 6800 | Loss: 0.1357 | Spent: 33.6 secs | LR: 0.000212
I0325 05:35:19.358985 140330694072192 interactiveshell.py:2882] Step 6850 | Loss: 0.1239 | Spent: 33.1 secs | LR: 0.000211
I0325 05:35:54.112353 140330694072192 interactiveshell.py:2882] Step 6900 | Loss: 0.0245 | Spent: 34.8 secs | LR: 0.000211
I0325 05:36:29.930253 140330694072192 interactiveshell.py:2882] Step 6950 | Loss: 0.1906 | Spent: 35.8 secs | LR: 0.000210
I0325 05:37:03.703213 140330694072192 interactiveshell.py:2882] Step 7000 | Loss: 0.1703 | Spent: 33.8 secs | LR: 0.000210
Reading ../data/test.txt
I0325 05:38:05.868737 140330694072192 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.903
I0325 05:38:05.873281 140330694072192 interactiveshell.py:2882] Best Accuracy: 0.907
I0325 05:38:05.877082 140330694072192 interactiveshell.py:2882] Testing Accuracy not improved over 3 epochs, Early Stop