In [0]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [0]:
!pip install tf-nightly-2.0-preview
Requirement already satisfied: tf-nightly-2.0-preview in /usr/local/lib/python3.6/dist-packages (2.0.0.dev20190324)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.6)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.2.2)
Requirement already satisfied: tensorflow-estimator-2.0-preview in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0.dev2019032400)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.9)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.1.0)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.33.1)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (3.7.0)
Requirement already satisfied: tb-nightly<1.15.0a0,>=1.14.0a0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.14.0a20190319)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.15.0)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.7.1)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.11.0)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (1.0.7)
Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-2.0-preview) (0.1.4)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tf-nightly-2.0-preview) (40.8.0)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (3.0.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.15.0a0,>=1.14.0a0->tf-nightly-2.0-preview) (0.14.1)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tf-nightly-2.0-preview) (2.8.0)
In [0]:
import tensorflow as tf
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-dev20190324
GPU Enabled: False
In [0]:
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      label, text = line.split('\t')
      text = text.split(' ')
      x = [params['word2idx'].get(w, len(word2idx)) for w in text]
      if len(x) > params['max_len']:
        x = x[:params['max_len']]
      y = int(label)
      yield x, y


def dataset(is_training, params):
  _shapes = ([None], ())
  _types = (tf.int32, tf.int32)
  _pads = (0, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [0]:
class KernelAttentivePooling(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.kernel = tf.keras.layers.Dense(units=1,
                                        activation=tf.tanh,
                                        use_bias=False)

  
  def call(self, inputs, training=False):
    x, masks = inputs
    # alignment
    align = tf.squeeze(self.kernel(self.dropout(x, training=training)), -1)
    # masking
    paddings = tf.fill(tf.shape(align), float('-inf'))
    align = tf.where(tf.equal(masks, 0), paddings, align)
    # probability
    align = tf.nn.softmax(align)
    align = tf.expand_dims(align, -1)
    # weighted sum
    return tf.squeeze(tf.matmul(x, align, transpose_a=True), -1)
In [0]:
class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    
    self.embedding = tf.Variable(np.load('../vocab/word.npy'),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
    
    self.attentive_pooling = KernelAttentivePooling(params)
    
    self.out_linear = tf.keras.layers.Dense(2)
  
  
  def call(self, inputs, training=False):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    masks = tf.sign(inputs)
    
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    
    x = self.attentive_pooling((x, masks), training=training)
    
    x = self.out_linear(x)
    
    return x
In [0]:
params = {
  'vocab_path': '../vocab/word.txt',
  'train_path': '../data/train.txt',
  'test_path': '../data/test.txt',
  'num_samples': 25000,
  'num_labels': 2,
  'batch_size': 32,
  'max_len': 1000,
  'dropout_rate': 0.2,
  'kernel_size': 5,
  'num_patience': 3,
  'lr': 3e-4,
}
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True    
In [0]:
word2idx = {}
with open(params['vocab_path']) as f:
  for i, line in enumerate(f):
    line = line.rstrip()
    word2idx[line] = i
params['word2idx'] = word2idx
params['vocab_size'] = len(word2idx) + 1


model = Model(params)
model.build(input_shape=(None, None))
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.95)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for texts, labels in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model(texts, training=True)
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
      loss = tf.reduce_mean(loss)
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for texts, labels in dataset(is_training=False, params=params):
    logits = model(texts, training=False)
    y_pred = tf.argmax(logits, axis=-1)
    m.update_state(y_true=labels, y_pred=y_pred)
    
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))
  history_acc.append(acc)
  
  if acc > best_acc:
    best_acc = acc
    # you can save model here
  logger.info("Best Accuracy: {:.3f}".format(best_acc))
  
  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0325 03:30:16.754851 140022398179200 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:421: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('kernel_attentive_pooling/dense/kernel:0', TensorShape([300, 1])),
 ('dense_1/kernel:0', TensorShape([300, 2])),
 ('dense_1/bias:0', TensorShape([2])),
 ('pretrained_embedding:0', TensorShape([20599, 300]))]
Reading ../data/train.txt
I0325 03:30:26.536055 140022398179200 interactiveshell.py:2882] Step 0 | Loss: 0.7376 | Spent: 9.8 secs | LR: 0.000300
I0325 03:30:45.361990 140022398179200 interactiveshell.py:2882] Step 50 | Loss: 0.6950 | Spent: 18.8 secs | LR: 0.000299
I0325 03:31:04.888937 140022398179200 interactiveshell.py:2882] Step 100 | Loss: 0.6872 | Spent: 19.5 secs | LR: 0.000298
I0325 03:31:24.072847 140022398179200 interactiveshell.py:2882] Step 150 | Loss: 0.6553 | Spent: 19.2 secs | LR: 0.000298
I0325 03:31:42.505526 140022398179200 interactiveshell.py:2882] Step 200 | Loss: 0.6436 | Spent: 18.4 secs | LR: 0.000297
I0325 03:32:00.057708 140022398179200 interactiveshell.py:2882] Step 250 | Loss: 0.6088 | Spent: 17.6 secs | LR: 0.000296
I0325 03:32:19.260957 140022398179200 interactiveshell.py:2882] Step 300 | Loss: 0.5804 | Spent: 19.2 secs | LR: 0.000295
I0325 03:32:38.410742 140022398179200 interactiveshell.py:2882] Step 350 | Loss: 0.5310 | Spent: 19.1 secs | LR: 0.000295
I0325 03:32:56.727094 140022398179200 interactiveshell.py:2882] Step 400 | Loss: 0.5155 | Spent: 18.3 secs | LR: 0.000294
I0325 03:33:16.974620 140022398179200 interactiveshell.py:2882] Step 450 | Loss: 0.4845 | Spent: 20.2 secs | LR: 0.000293
I0325 03:33:36.632110 140022398179200 interactiveshell.py:2882] Step 500 | Loss: 0.4728 | Spent: 19.7 secs | LR: 0.000292
I0325 03:33:55.689022 140022398179200 interactiveshell.py:2882] Step 550 | Loss: 0.3575 | Spent: 19.1 secs | LR: 0.000292
I0325 03:34:16.367763 140022398179200 interactiveshell.py:2882] Step 600 | Loss: 0.4687 | Spent: 20.7 secs | LR: 0.000291
I0325 03:34:34.908947 140022398179200 interactiveshell.py:2882] Step 650 | Loss: 0.4529 | Spent: 18.5 secs | LR: 0.000290
I0325 03:34:55.359016 140022398179200 interactiveshell.py:2882] Step 700 | Loss: 0.3640 | Spent: 20.4 secs | LR: 0.000289
I0325 03:35:13.933534 140022398179200 interactiveshell.py:2882] Step 750 | Loss: 0.4056 | Spent: 18.6 secs | LR: 0.000289
Reading ../data/test.txt
I0325 03:35:40.789431 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.867
I0325 03:35:40.791317 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.867
Reading ../data/train.txt
I0325 03:35:58.257108 140022398179200 interactiveshell.py:2882] Step 800 | Loss: 0.2685 | Spent: 44.3 secs | LR: 0.000288
I0325 03:36:16.753606 140022398179200 interactiveshell.py:2882] Step 850 | Loss: 0.3405 | Spent: 18.5 secs | LR: 0.000287
I0325 03:36:33.767656 140022398179200 interactiveshell.py:2882] Step 900 | Loss: 0.2763 | Spent: 17.0 secs | LR: 0.000286
I0325 03:36:49.980346 140022398179200 interactiveshell.py:2882] Step 950 | Loss: 0.2499 | Spent: 16.2 secs | LR: 0.000286
I0325 03:37:06.194425 140022398179200 interactiveshell.py:2882] Step 1000 | Loss: 0.2668 | Spent: 16.2 secs | LR: 0.000285
I0325 03:37:22.921469 140022398179200 interactiveshell.py:2882] Step 1050 | Loss: 0.3046 | Spent: 16.7 secs | LR: 0.000284
I0325 03:37:39.571293 140022398179200 interactiveshell.py:2882] Step 1100 | Loss: 0.3539 | Spent: 16.6 secs | LR: 0.000284
I0325 03:37:56.334148 140022398179200 interactiveshell.py:2882] Step 1150 | Loss: 0.2754 | Spent: 16.8 secs | LR: 0.000283
I0325 03:38:13.268370 140022398179200 interactiveshell.py:2882] Step 1200 | Loss: 0.2079 | Spent: 16.9 secs | LR: 0.000282
I0325 03:38:30.364040 140022398179200 interactiveshell.py:2882] Step 1250 | Loss: 0.3416 | Spent: 17.1 secs | LR: 0.000281
I0325 03:38:46.338656 140022398179200 interactiveshell.py:2882] Step 1300 | Loss: 0.2800 | Spent: 16.0 secs | LR: 0.000281
I0325 03:39:02.410163 140022398179200 interactiveshell.py:2882] Step 1350 | Loss: 0.1766 | Spent: 16.1 secs | LR: 0.000280
I0325 03:39:18.981061 140022398179200 interactiveshell.py:2882] Step 1400 | Loss: 0.2780 | Spent: 16.6 secs | LR: 0.000279
I0325 03:39:35.756258 140022398179200 interactiveshell.py:2882] Step 1450 | Loss: 0.2689 | Spent: 16.8 secs | LR: 0.000278
I0325 03:39:52.891721 140022398179200 interactiveshell.py:2882] Step 1500 | Loss: 0.2895 | Spent: 17.1 secs | LR: 0.000278
I0325 03:40:09.672029 140022398179200 interactiveshell.py:2882] Step 1550 | Loss: 0.2521 | Spent: 16.8 secs | LR: 0.000277
Reading ../data/test.txt
I0325 03:40:27.521064 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.886
I0325 03:40:27.522983 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.886
Reading ../data/train.txt
I0325 03:40:48.331700 140022398179200 interactiveshell.py:2882] Step 1600 | Loss: 0.2124 | Spent: 38.7 secs | LR: 0.000276
I0325 03:41:05.857304 140022398179200 interactiveshell.py:2882] Step 1650 | Loss: 0.1989 | Spent: 17.5 secs | LR: 0.000276
I0325 03:41:22.754487 140022398179200 interactiveshell.py:2882] Step 1700 | Loss: 0.1353 | Spent: 16.9 secs | LR: 0.000275
I0325 03:41:39.517929 140022398179200 interactiveshell.py:2882] Step 1750 | Loss: 0.2493 | Spent: 16.8 secs | LR: 0.000274
I0325 03:41:56.105601 140022398179200 interactiveshell.py:2882] Step 1800 | Loss: 0.2003 | Spent: 16.6 secs | LR: 0.000274
I0325 03:42:12.927456 140022398179200 interactiveshell.py:2882] Step 1850 | Loss: 0.1578 | Spent: 16.8 secs | LR: 0.000273
I0325 03:42:29.306477 140022398179200 interactiveshell.py:2882] Step 1900 | Loss: 0.4522 | Spent: 16.4 secs | LR: 0.000272
I0325 03:42:46.384248 140022398179200 interactiveshell.py:2882] Step 1950 | Loss: 0.4553 | Spent: 17.1 secs | LR: 0.000271
I0325 03:43:03.028537 140022398179200 interactiveshell.py:2882] Step 2000 | Loss: 0.1684 | Spent: 16.6 secs | LR: 0.000271
I0325 03:43:19.571934 140022398179200 interactiveshell.py:2882] Step 2050 | Loss: 0.2164 | Spent: 16.5 secs | LR: 0.000270
I0325 03:43:36.549591 140022398179200 interactiveshell.py:2882] Step 2100 | Loss: 0.1466 | Spent: 17.0 secs | LR: 0.000269
I0325 03:43:53.754223 140022398179200 interactiveshell.py:2882] Step 2150 | Loss: 0.1853 | Spent: 17.2 secs | LR: 0.000269
I0325 03:44:10.233977 140022398179200 interactiveshell.py:2882] Step 2200 | Loss: 0.2971 | Spent: 16.5 secs | LR: 0.000268
I0325 03:44:27.072910 140022398179200 interactiveshell.py:2882] Step 2250 | Loss: 0.2526 | Spent: 16.8 secs | LR: 0.000267
I0325 03:44:44.420129 140022398179200 interactiveshell.py:2882] Step 2300 | Loss: 0.2984 | Spent: 17.3 secs | LR: 0.000267
Reading ../data/test.txt
I0325 03:45:12.328164 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.891
I0325 03:45:12.329895 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.891
Reading ../data/train.txt
I0325 03:45:22.737331 140022398179200 interactiveshell.py:2882] Step 2350 | Loss: 0.3217 | Spent: 38.3 secs | LR: 0.000266
I0325 03:45:39.475934 140022398179200 interactiveshell.py:2882] Step 2400 | Loss: 0.2727 | Spent: 16.7 secs | LR: 0.000265
I0325 03:45:57.192050 140022398179200 interactiveshell.py:2882] Step 2450 | Loss: 0.3776 | Spent: 17.7 secs | LR: 0.000265
I0325 03:46:14.353575 140022398179200 interactiveshell.py:2882] Step 2500 | Loss: 0.1676 | Spent: 17.2 secs | LR: 0.000264
I0325 03:46:31.285637 140022398179200 interactiveshell.py:2882] Step 2550 | Loss: 0.4214 | Spent: 16.9 secs | LR: 0.000263
I0325 03:46:48.407405 140022398179200 interactiveshell.py:2882] Step 2600 | Loss: 0.2714 | Spent: 17.1 secs | LR: 0.000263
I0325 03:47:05.446139 140022398179200 interactiveshell.py:2882] Step 2650 | Loss: 0.3025 | Spent: 17.0 secs | LR: 0.000262
I0325 03:47:22.433084 140022398179200 interactiveshell.py:2882] Step 2700 | Loss: 0.3387 | Spent: 17.0 secs | LR: 0.000261
I0325 03:47:38.750429 140022398179200 interactiveshell.py:2882] Step 2750 | Loss: 0.2367 | Spent: 16.3 secs | LR: 0.000261
I0325 03:47:55.025598 140022398179200 interactiveshell.py:2882] Step 2800 | Loss: 0.1613 | Spent: 16.3 secs | LR: 0.000260
I0325 03:48:11.117359 140022398179200 interactiveshell.py:2882] Step 2850 | Loss: 0.2189 | Spent: 16.1 secs | LR: 0.000259
I0325 03:48:27.778836 140022398179200 interactiveshell.py:2882] Step 2900 | Loss: 0.1613 | Spent: 16.7 secs | LR: 0.000259
I0325 03:48:43.680014 140022398179200 interactiveshell.py:2882] Step 2950 | Loss: 0.3108 | Spent: 15.9 secs | LR: 0.000258
I0325 03:49:00.113502 140022398179200 interactiveshell.py:2882] Step 3000 | Loss: 0.2687 | Spent: 16.4 secs | LR: 0.000257
I0325 03:49:16.783196 140022398179200 interactiveshell.py:2882] Step 3050 | Loss: 0.3495 | Spent: 16.7 secs | LR: 0.000257
I0325 03:49:33.814852 140022398179200 interactiveshell.py:2882] Step 3100 | Loss: 0.1789 | Spent: 17.0 secs | LR: 0.000256
Reading ../data/test.txt
I0325 03:49:56.902552 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.895
I0325 03:49:56.904222 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0325 03:50:13.113441 140022398179200 interactiveshell.py:2882] Step 3150 | Loss: 0.1607 | Spent: 39.3 secs | LR: 0.000255
I0325 03:50:29.700300 140022398179200 interactiveshell.py:2882] Step 3200 | Loss: 0.1294 | Spent: 16.6 secs | LR: 0.000255
I0325 03:50:46.742290 140022398179200 interactiveshell.py:2882] Step 3250 | Loss: 0.1381 | Spent: 17.0 secs | LR: 0.000254
I0325 03:51:03.979642 140022398179200 interactiveshell.py:2882] Step 3300 | Loss: 0.1988 | Spent: 17.2 secs | LR: 0.000253
I0325 03:51:21.005246 140022398179200 interactiveshell.py:2882] Step 3350 | Loss: 0.2887 | Spent: 17.0 secs | LR: 0.000253
I0325 03:51:37.823299 140022398179200 interactiveshell.py:2882] Step 3400 | Loss: 0.1918 | Spent: 16.8 secs | LR: 0.000252
I0325 03:51:54.400311 140022398179200 interactiveshell.py:2882] Step 3450 | Loss: 0.3522 | Spent: 16.6 secs | LR: 0.000251
I0325 03:52:11.059819 140022398179200 interactiveshell.py:2882] Step 3500 | Loss: 0.2566 | Spent: 16.7 secs | LR: 0.000251
I0325 03:52:27.521015 140022398179200 interactiveshell.py:2882] Step 3550 | Loss: 0.1067 | Spent: 16.5 secs | LR: 0.000250
I0325 03:52:43.951972 140022398179200 interactiveshell.py:2882] Step 3600 | Loss: 0.1482 | Spent: 16.4 secs | LR: 0.000249
I0325 03:52:59.989140 140022398179200 interactiveshell.py:2882] Step 3650 | Loss: 0.1568 | Spent: 16.0 secs | LR: 0.000249
I0325 03:53:16.663929 140022398179200 interactiveshell.py:2882] Step 3700 | Loss: 0.1056 | Spent: 16.7 secs | LR: 0.000248
I0325 03:53:32.833154 140022398179200 interactiveshell.py:2882] Step 3750 | Loss: 0.2877 | Spent: 16.2 secs | LR: 0.000248
I0325 03:53:49.208958 140022398179200 interactiveshell.py:2882] Step 3800 | Loss: 0.1781 | Spent: 16.4 secs | LR: 0.000247
I0325 03:54:06.335289 140022398179200 interactiveshell.py:2882] Step 3850 | Loss: 0.2033 | Spent: 17.1 secs | LR: 0.000246
I0325 03:54:22.975320 140022398179200 interactiveshell.py:2882] Step 3900 | Loss: 0.0615 | Spent: 16.6 secs | LR: 0.000246
Reading ../data/test.txt
I0325 03:54:39.552122 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.894
I0325 03:54:39.553550 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0325 03:55:01.994681 140022398179200 interactiveshell.py:2882] Step 3950 | Loss: 0.2447 | Spent: 39.0 secs | LR: 0.000245
I0325 03:55:18.059489 140022398179200 interactiveshell.py:2882] Step 4000 | Loss: 0.1998 | Spent: 16.1 secs | LR: 0.000244
I0325 03:55:35.434613 140022398179200 interactiveshell.py:2882] Step 4050 | Loss: 0.2550 | Spent: 17.4 secs | LR: 0.000244
I0325 03:55:52.359549 140022398179200 interactiveshell.py:2882] Step 4100 | Loss: 0.1258 | Spent: 16.9 secs | LR: 0.000243
I0325 03:56:09.153433 140022398179200 interactiveshell.py:2882] Step 4150 | Loss: 0.0951 | Spent: 16.8 secs | LR: 0.000242
I0325 03:56:26.300997 140022398179200 interactiveshell.py:2882] Step 4200 | Loss: 0.1632 | Spent: 17.1 secs | LR: 0.000242
I0325 03:56:43.155941 140022398179200 interactiveshell.py:2882] Step 4250 | Loss: 0.2935 | Spent: 16.9 secs | LR: 0.000241
I0325 03:56:59.678212 140022398179200 interactiveshell.py:2882] Step 4300 | Loss: 0.1977 | Spent: 16.5 secs | LR: 0.000241
I0325 03:57:16.063052 140022398179200 interactiveshell.py:2882] Step 4350 | Loss: 0.1999 | Spent: 16.4 secs | LR: 0.000240
I0325 03:57:32.842365 140022398179200 interactiveshell.py:2882] Step 4400 | Loss: 0.0950 | Spent: 16.8 secs | LR: 0.000239
I0325 03:57:49.662498 140022398179200 interactiveshell.py:2882] Step 4450 | Loss: 0.1177 | Spent: 16.8 secs | LR: 0.000239
I0325 03:58:06.605592 140022398179200 interactiveshell.py:2882] Step 4500 | Loss: 0.2118 | Spent: 16.9 secs | LR: 0.000238
I0325 03:58:22.988858 140022398179200 interactiveshell.py:2882] Step 4550 | Loss: 0.1814 | Spent: 16.4 secs | LR: 0.000238
I0325 03:58:39.304465 140022398179200 interactiveshell.py:2882] Step 4600 | Loss: 0.1787 | Spent: 16.3 secs | LR: 0.000237
I0325 03:58:56.893966 140022398179200 interactiveshell.py:2882] Step 4650 | Loss: 0.0822 | Spent: 17.6 secs | LR: 0.000236
Reading ../data/test.txt
I0325 03:59:28.323561 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.894
I0325 03:59:28.326061 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0325 03:59:40.815753 140022398179200 interactiveshell.py:2882] Step 4700 | Loss: 0.1232 | Spent: 43.9 secs | LR: 0.000236
I0325 04:00:00.451601 140022398179200 interactiveshell.py:2882] Step 4750 | Loss: 0.1817 | Spent: 19.6 secs | LR: 0.000235
I0325 04:00:18.553692 140022398179200 interactiveshell.py:2882] Step 4800 | Loss: 0.0834 | Spent: 18.1 secs | LR: 0.000235
I0325 04:00:36.859032 140022398179200 interactiveshell.py:2882] Step 4850 | Loss: 0.0797 | Spent: 18.3 secs | LR: 0.000234
I0325 04:00:55.848888 140022398179200 interactiveshell.py:2882] Step 4900 | Loss: 0.0756 | Spent: 19.0 secs | LR: 0.000233
I0325 04:01:14.818666 140022398179200 interactiveshell.py:2882] Step 4950 | Loss: 0.1375 | Spent: 19.0 secs | LR: 0.000233
I0325 04:01:32.824758 140022398179200 interactiveshell.py:2882] Step 5000 | Loss: 0.1141 | Spent: 18.0 secs | LR: 0.000232
I0325 04:01:51.861589 140022398179200 interactiveshell.py:2882] Step 5050 | Loss: 0.2165 | Spent: 19.0 secs | LR: 0.000232
I0325 04:02:10.283536 140022398179200 interactiveshell.py:2882] Step 5100 | Loss: 0.1066 | Spent: 18.4 secs | LR: 0.000231
I0325 04:02:28.153270 140022398179200 interactiveshell.py:2882] Step 5150 | Loss: 0.1115 | Spent: 17.9 secs | LR: 0.000230
I0325 04:02:45.536251 140022398179200 interactiveshell.py:2882] Step 5200 | Loss: 0.0923 | Spent: 17.4 secs | LR: 0.000230
I0325 04:03:02.374162 140022398179200 interactiveshell.py:2882] Step 5250 | Loss: 0.3379 | Spent: 16.8 secs | LR: 0.000229
I0325 04:03:20.279114 140022398179200 interactiveshell.py:2882] Step 5300 | Loss: 0.1014 | Spent: 17.9 secs | LR: 0.000229
I0325 04:03:37.079680 140022398179200 interactiveshell.py:2882] Step 5350 | Loss: 0.1360 | Spent: 16.8 secs | LR: 0.000228
I0325 04:03:54.500293 140022398179200 interactiveshell.py:2882] Step 5400 | Loss: 0.1606 | Spent: 17.4 secs | LR: 0.000227
I0325 04:04:13.513936 140022398179200 interactiveshell.py:2882] Step 5450 | Loss: 0.1513 | Spent: 19.0 secs | LR: 0.000227
Reading ../data/test.txt
I0325 04:04:36.945525 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.890
I0325 04:04:36.947145 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0325 04:04:57.172717 140022398179200 interactiveshell.py:2882] Step 5500 | Loss: 0.0503 | Spent: 43.7 secs | LR: 0.000226
I0325 04:05:15.191678 140022398179200 interactiveshell.py:2882] Step 5550 | Loss: 0.0996 | Spent: 18.0 secs | LR: 0.000226
I0325 04:05:34.019433 140022398179200 interactiveshell.py:2882] Step 5600 | Loss: 0.0568 | Spent: 18.8 secs | LR: 0.000225
I0325 04:05:52.520728 140022398179200 interactiveshell.py:2882] Step 5650 | Loss: 0.1515 | Spent: 18.5 secs | LR: 0.000225
I0325 04:06:10.327186 140022398179200 interactiveshell.py:2882] Step 5700 | Loss: 0.1045 | Spent: 17.8 secs | LR: 0.000224
I0325 04:06:28.165599 140022398179200 interactiveshell.py:2882] Step 5750 | Loss: 0.0760 | Spent: 17.8 secs | LR: 0.000223
I0325 04:06:46.422015 140022398179200 interactiveshell.py:2882] Step 5800 | Loss: 0.0473 | Spent: 18.3 secs | LR: 0.000223
I0325 04:07:06.700240 140022398179200 interactiveshell.py:2882] Step 5850 | Loss: 0.1107 | Spent: 20.3 secs | LR: 0.000222
I0325 04:07:25.419937 140022398179200 interactiveshell.py:2882] Step 5900 | Loss: 0.0832 | Spent: 18.7 secs | LR: 0.000222
I0325 04:07:45.611305 140022398179200 interactiveshell.py:2882] Step 5950 | Loss: 0.1762 | Spent: 20.2 secs | LR: 0.000221
I0325 04:08:03.787918 140022398179200 interactiveshell.py:2882] Step 6000 | Loss: 0.1047 | Spent: 18.2 secs | LR: 0.000221
I0325 04:08:22.263847 140022398179200 interactiveshell.py:2882] Step 6050 | Loss: 0.1207 | Spent: 18.5 secs | LR: 0.000220
I0325 04:08:41.768623 140022398179200 interactiveshell.py:2882] Step 6100 | Loss: 0.2801 | Spent: 19.5 secs | LR: 0.000219
I0325 04:09:00.173652 140022398179200 interactiveshell.py:2882] Step 6150 | Loss: 0.0537 | Spent: 18.4 secs | LR: 0.000219
I0325 04:09:18.272282 140022398179200 interactiveshell.py:2882] Step 6200 | Loss: 0.0694 | Spent: 18.1 secs | LR: 0.000218
I0325 04:09:37.667584 140022398179200 interactiveshell.py:2882] Step 6250 | Loss: 0.1799 | Spent: 19.4 secs | LR: 0.000218
Reading ../data/test.txt
I0325 04:09:55.233154 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.889
I0325 04:09:55.234862 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
Reading ../data/train.txt
I0325 04:10:22.488271 140022398179200 interactiveshell.py:2882] Step 6300 | Loss: 0.1728 | Spent: 44.8 secs | LR: 0.000217
I0325 04:10:41.382654 140022398179200 interactiveshell.py:2882] Step 6350 | Loss: 0.0873 | Spent: 18.9 secs | LR: 0.000217
I0325 04:11:00.888543 140022398179200 interactiveshell.py:2882] Step 6400 | Loss: 0.0740 | Spent: 19.5 secs | LR: 0.000216
I0325 04:11:18.480540 140022398179200 interactiveshell.py:2882] Step 6450 | Loss: 0.1267 | Spent: 17.6 secs | LR: 0.000215
I0325 04:11:36.405515 140022398179200 interactiveshell.py:2882] Step 6500 | Loss: 0.0763 | Spent: 17.9 secs | LR: 0.000215
I0325 04:11:54.951826 140022398179200 interactiveshell.py:2882] Step 6550 | Loss: 0.0566 | Spent: 18.5 secs | LR: 0.000214
I0325 04:12:13.411118 140022398179200 interactiveshell.py:2882] Step 6600 | Loss: 0.1167 | Spent: 18.5 secs | LR: 0.000214
I0325 04:12:32.519585 140022398179200 interactiveshell.py:2882] Step 6650 | Loss: 0.0745 | Spent: 19.1 secs | LR: 0.000213
I0325 04:12:51.332306 140022398179200 interactiveshell.py:2882] Step 6700 | Loss: 0.0643 | Spent: 18.8 secs | LR: 0.000213
I0325 04:13:09.895300 140022398179200 interactiveshell.py:2882] Step 6750 | Loss: 0.1256 | Spent: 18.6 secs | LR: 0.000212
I0325 04:13:27.290911 140022398179200 interactiveshell.py:2882] Step 6800 | Loss: 0.0534 | Spent: 17.4 secs | LR: 0.000212
I0325 04:13:44.743788 140022398179200 interactiveshell.py:2882] Step 6850 | Loss: 0.1446 | Spent: 17.5 secs | LR: 0.000211
I0325 04:14:02.566674 140022398179200 interactiveshell.py:2882] Step 6900 | Loss: 0.1749 | Spent: 17.8 secs | LR: 0.000211
I0325 04:14:21.315324 140022398179200 interactiveshell.py:2882] Step 6950 | Loss: 0.1070 | Spent: 18.7 secs | LR: 0.000210
I0325 04:14:40.360705 140022398179200 interactiveshell.py:2882] Step 7000 | Loss: 0.1014 | Spent: 19.0 secs | LR: 0.000210
Reading ../data/test.txt
I0325 04:15:09.607374 140022398179200 interactiveshell.py:2882] Evaluation: Testing Accuracy: 0.887
I0325 04:15:09.609139 140022398179200 interactiveshell.py:2882] Best Accuracy: 0.895
I0325 04:15:09.617720 140022398179200 interactiveshell.py:2882] Testing Accuracy not improved over 3 epochs, Early Stop