1. tensorflow-nlp
  2. finch
  3. tensorflow2
  4. text_matching
  5. chinese
  6. main
In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_matching/chinese/main')

Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install transformers

Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)
Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)
Requirement already satisfied: dataclasses; python_version < "3.7" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)
Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)
Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)
Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.9)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)
Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)
Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.15.1)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)
In [3]:
from transformers import BertTokenizer, TFBertModel

import os
import csv
import time
import logging
import pprint
import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 2.2.0
WARNING:tensorflow:From <ipython-input-3-c30b15e954fc>:13: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
params = {
  'pretrain_path': 'bert-base-chinese',
  'train_path': '../data/train.csv',
  'test_path': '../data/test.csv',
  'batch_size': 32,
  'max_len': 64,
  'buffer_size': 100000,
  'init_lr': 1e-5,
  'max_lr': 5e-5,
  'n_epochs': 12,
  'num_patience': 5,
}

tokenizer = BertTokenizer.from_pretrained(params['pretrain_path'],
                                          lowercase = True,
                                          add_special_tokens = True)
In [5]:
# stream data from text files
def data_generator(f_path, params):
  with open(f_path, encoding='utf-8') as f:
    print('Reading', f_path)
    for i, line in enumerate(csv.reader(f, delimiter=',')):
      if i == 0:
        continue
      text1, text2, label = line
      if len(text1) + len(text2) + 2 > params['max_len']:
        _max_len = (params['max_len'] - 2) // 2
        text1 = text1[:_max_len]
        text2 = text2[:_max_len]
      text1 = list(text1)
      text2 = list(text2)
      text = ['[CLS]'] + text1 + ['[SEP]'] + text2
      seg = [0] + [0] * len(text1) + [0] + [1] * len(text2)
      text = tokenizer.convert_tokens_to_ids(text)
      yield (text, seg), int(label)


def dataset(is_training, params):
  _shapes = (([None], [None]), ())
  _types = ((tf.int32, tf.int32), tf.int32)
  _pads = ((0, 0), -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    #ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [6]:
# input stream ids check
(text, seg), _ = next(data_generator(params['train_path'], params))
print(text)
print(seg)

Reading ../data/train.csv
[101, 4500, 2544, 928, 6963, 127, 2399, 8024, 2544, 928, 3766, 3300, 2544, 5108, 6587, 1216, 5543, 102, 125, 511, 100, 100, 1384, 4772, 3341, 2544, 5108, 6587]
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
In [7]:
class BertFinetune(tf.keras.Model):
  def __init__(self, params):
    super(BertFinetune, self).__init__()
    self.bert = TFBertModel.from_pretrained(params['pretrain_path'],
                                            trainable = True)
    self.drop_1 = tf.keras.layers.Dropout(.1)
    self.fc = tf.keras.layers.Dense(300, tf.nn.elu, name='down_stream/fc')
    self.drop_2 = tf.keras.layers.Dropout(.1)
    self.out = tf.keras.layers.Dense(2, name='down_stream/out')

  def call(self, bert_inputs, training):
    bert_inputs = [tf.cast(inp, tf.int32) for inp in bert_inputs]
    x = self.bert(bert_inputs, training=training)[1]
    x = self.drop_1(x, training=training)
    x = self.fc(x)
    x = self.drop_2(x, training=training)
    x = self.out(x)
    return x
In [8]:
model = BertFinetune(params)
model.build([[None, None], [None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

step_size = 2 * params['buffer_size'] // params['batch_size']
decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = step_size,)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc = .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

for _ in range(params['n_epochs']):
  # TRAINING
  for ((text, seg), labels) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model([text, tf.sign(text), seg], training=True)
      loss = tf.compat.v1.losses.softmax_cross_entropy(
        tf.one_hot(labels, 2, dtype=tf.float32),
        logits = logits,
        label_smoothing = .1,)
      
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, 5.)
    optim.apply_gradients(zip(grads, model.trainable_variables))
    
    if global_step % 100 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for ((text, seg), labels) in dataset(is_training=False, params=params):
    logits = model([text, tf.sign(text), seg], training=False)
    m.update_state(y_true=labels, y_pred=tf.argmax(logits, -1))

  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))

  if acc > best_acc:
    best_acc = acc
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best Accuracy: {:.3f}".format(best_acc))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break

Some weights of the model checkpoint at bert-base-chinese were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']
- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFBertModel were initialized from the model checkpoint at bert-base-chinese.
If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.

[('tf_bert_model/bert/embeddings/word_embeddings/weight:0',
  TensorShape([21128, 768])),
 ('tf_bert_model/bert/embeddings/position_embeddings/embeddings:0',
  TensorShape([512, 768])),
 ('tf_bert_model/bert/embeddings/token_type_embeddings/embeddings:0',
  TensorShape([2, 768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/gamma:0', TensorShape([768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/beta:0', TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/pooler/dense/kernel:0', TensorShape([768, 768])),
 ('tf_bert_model/bert/pooler/dense/bias:0', TensorShape([768])),
 ('down_stream/fc/kernel:0', TensorShape([768, 300])),
 ('down_stream/fc/bias:0', TensorShape([300])),
 ('down_stream/out/kernel:0', TensorShape([300, 2])),
 ('down_stream/out/bias:0', TensorShape([2]))]
Reading ../data/train.csv
INFO:tensorflow:Step 0 | Loss: 1.3536 | Spent: 0.9 secs | LR: 0.000010
INFO:tensorflow:Step 100 | Loss: 0.7859 | Spent: 46.6 secs | LR: 0.000011
INFO:tensorflow:Step 200 | Loss: 0.4251 | Spent: 45.3 secs | LR: 0.000011
INFO:tensorflow:Step 300 | Loss: 0.5814 | Spent: 45.9 secs | LR: 0.000012
INFO:tensorflow:Step 400 | Loss: 0.4883 | Spent: 45.8 secs | LR: 0.000013
INFO:tensorflow:Step 500 | Loss: 0.5622 | Spent: 45.9 secs | LR: 0.000013
INFO:tensorflow:Step 600 | Loss: 0.4912 | Spent: 46.0 secs | LR: 0.000014
INFO:tensorflow:Step 700 | Loss: 0.5496 | Spent: 45.7 secs | LR: 0.000014
INFO:tensorflow:Step 800 | Loss: 0.5410 | Spent: 46.2 secs | LR: 0.000015
INFO:tensorflow:Step 900 | Loss: 0.6012 | Spent: 46.3 secs | LR: 0.000016
INFO:tensorflow:Step 1000 | Loss: 0.3443 | Spent: 46.3 secs | LR: 0.000016
INFO:tensorflow:Step 1100 | Loss: 0.3336 | Spent: 46.1 secs | LR: 0.000017
INFO:tensorflow:Step 1200 | Loss: 0.5493 | Spent: 46.3 secs | LR: 0.000018
INFO:tensorflow:Step 1300 | Loss: 0.4409 | Spent: 46.5 secs | LR: 0.000018
INFO:tensorflow:Step 1400 | Loss: 0.4616 | Spent: 46.6 secs | LR: 0.000019
INFO:tensorflow:Step 1500 | Loss: 0.4711 | Spent: 46.7 secs | LR: 0.000020
INFO:tensorflow:Step 1600 | Loss: 0.3575 | Spent: 46.2 secs | LR: 0.000020
INFO:tensorflow:Step 1700 | Loss: 0.4678 | Spent: 46.3 secs | LR: 0.000021
INFO:tensorflow:Step 1800 | Loss: 0.4473 | Spent: 46.3 secs | LR: 0.000022
INFO:tensorflow:Step 1900 | Loss: 0.5045 | Spent: 46.2 secs | LR: 0.000022
INFO:tensorflow:Step 2000 | Loss: 0.4452 | Spent: 45.6 secs | LR: 0.000023
INFO:tensorflow:Step 2100 | Loss: 0.6363 | Spent: 46.1 secs | LR: 0.000023
INFO:tensorflow:Step 2200 | Loss: 0.3788 | Spent: 45.9 secs | LR: 0.000024
INFO:tensorflow:Step 2300 | Loss: 0.3740 | Spent: 46.2 secs | LR: 0.000025
INFO:tensorflow:Step 2400 | Loss: 0.4650 | Spent: 46.1 secs | LR: 0.000025
INFO:tensorflow:Step 2500 | Loss: 0.3012 | Spent: 46.6 secs | LR: 0.000026
INFO:tensorflow:Step 2600 | Loss: 0.4363 | Spent: 46.0 secs | LR: 0.000027
INFO:tensorflow:Step 2700 | Loss: 0.3706 | Spent: 46.6 secs | LR: 0.000027
INFO:tensorflow:Step 2800 | Loss: 0.3756 | Spent: 45.4 secs | LR: 0.000028
INFO:tensorflow:Step 2900 | Loss: 0.4230 | Spent: 46.1 secs | LR: 0.000029
INFO:tensorflow:Step 3000 | Loss: 0.4514 | Spent: 46.2 secs | LR: 0.000029
INFO:tensorflow:Step 3100 | Loss: 0.5087 | Spent: 45.8 secs | LR: 0.000030
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.830
INFO:tensorflow:Best Accuracy: 0.830
Reading ../data/train.csv
INFO:tensorflow:Step 3200 | Loss: 0.5950 | Spent: 88.7 secs | LR: 0.000030
INFO:tensorflow:Step 3300 | Loss: 0.3199 | Spent: 46.3 secs | LR: 0.000031
INFO:tensorflow:Step 3400 | Loss: 0.4534 | Spent: 45.7 secs | LR: 0.000032
INFO:tensorflow:Step 3500 | Loss: 0.3422 | Spent: 46.2 secs | LR: 0.000032
INFO:tensorflow:Step 3600 | Loss: 0.4092 | Spent: 46.0 secs | LR: 0.000033
INFO:tensorflow:Step 3700 | Loss: 0.3025 | Spent: 46.1 secs | LR: 0.000034
INFO:tensorflow:Step 3800 | Loss: 0.4157 | Spent: 46.3 secs | LR: 0.000034
INFO:tensorflow:Step 3900 | Loss: 0.3909 | Spent: 46.2 secs | LR: 0.000035
INFO:tensorflow:Step 4000 | Loss: 0.3785 | Spent: 46.6 secs | LR: 0.000036
INFO:tensorflow:Step 4100 | Loss: 0.3519 | Spent: 46.4 secs | LR: 0.000036
INFO:tensorflow:Step 4200 | Loss: 0.3047 | Spent: 46.0 secs | LR: 0.000037
INFO:tensorflow:Step 4300 | Loss: 0.4430 | Spent: 46.4 secs | LR: 0.000038
INFO:tensorflow:Step 4400 | Loss: 0.2814 | Spent: 46.2 secs | LR: 0.000038
INFO:tensorflow:Step 4500 | Loss: 0.3683 | Spent: 46.8 secs | LR: 0.000039
INFO:tensorflow:Step 4600 | Loss: 0.3120 | Spent: 46.3 secs | LR: 0.000039
INFO:tensorflow:Step 4700 | Loss: 0.4458 | Spent: 46.5 secs | LR: 0.000040
INFO:tensorflow:Step 4800 | Loss: 0.3059 | Spent: 46.1 secs | LR: 0.000041
INFO:tensorflow:Step 4900 | Loss: 0.5498 | Spent: 47.1 secs | LR: 0.000041
INFO:tensorflow:Step 5000 | Loss: 0.3168 | Spent: 46.3 secs | LR: 0.000042
INFO:tensorflow:Step 5100 | Loss: 0.2496 | Spent: 45.6 secs | LR: 0.000043
INFO:tensorflow:Step 5200 | Loss: 0.3131 | Spent: 46.9 secs | LR: 0.000043
INFO:tensorflow:Step 5300 | Loss: 0.3916 | Spent: 46.1 secs | LR: 0.000044
INFO:tensorflow:Step 5400 | Loss: 0.4027 | Spent: 47.2 secs | LR: 0.000045
INFO:tensorflow:Step 5500 | Loss: 0.4606 | Spent: 46.6 secs | LR: 0.000045
INFO:tensorflow:Step 5600 | Loss: 0.3695 | Spent: 46.7 secs | LR: 0.000046
INFO:tensorflow:Step 5700 | Loss: 0.4105 | Spent: 46.7 secs | LR: 0.000046
INFO:tensorflow:Step 5800 | Loss: 0.3493 | Spent: 46.7 secs | LR: 0.000047
INFO:tensorflow:Step 5900 | Loss: 0.3435 | Spent: 46.4 secs | LR: 0.000048
INFO:tensorflow:Step 6000 | Loss: 0.3078 | Spent: 46.3 secs | LR: 0.000048
INFO:tensorflow:Step 6100 | Loss: 0.3165 | Spent: 46.3 secs | LR: 0.000049
INFO:tensorflow:Step 6200 | Loss: 0.4040 | Spent: 45.8 secs | LR: 0.000050
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.823
INFO:tensorflow:Best Accuracy: 0.830
Reading ../data/train.csv
INFO:tensorflow:Step 6300 | Loss: 0.3679 | Spent: 88.7 secs | LR: 0.000050
INFO:tensorflow:Step 6400 | Loss: 0.3199 | Spent: 47.2 secs | LR: 0.000049
INFO:tensorflow:Step 6500 | Loss: 0.2636 | Spent: 45.8 secs | LR: 0.000048
INFO:tensorflow:Step 6600 | Loss: 0.4014 | Spent: 46.2 secs | LR: 0.000048
INFO:tensorflow:Step 6700 | Loss: 0.2891 | Spent: 46.3 secs | LR: 0.000047
INFO:tensorflow:Step 6800 | Loss: 0.4835 | Spent: 45.8 secs | LR: 0.000046
INFO:tensorflow:Step 6900 | Loss: 0.4304 | Spent: 46.6 secs | LR: 0.000046
INFO:tensorflow:Step 7000 | Loss: 0.6127 | Spent: 46.1 secs | LR: 0.000045
INFO:tensorflow:Step 7100 | Loss: 0.2787 | Spent: 46.8 secs | LR: 0.000045
INFO:tensorflow:Step 7200 | Loss: 0.4103 | Spent: 46.2 secs | LR: 0.000044
INFO:tensorflow:Step 7300 | Loss: 0.3733 | Spent: 46.6 secs | LR: 0.000043
INFO:tensorflow:Step 7400 | Loss: 0.2461 | Spent: 46.7 secs | LR: 0.000043
INFO:tensorflow:Step 7500 | Loss: 0.5120 | Spent: 46.0 secs | LR: 0.000042
INFO:tensorflow:Step 7600 | Loss: 0.2686 | Spent: 46.8 secs | LR: 0.000041
INFO:tensorflow:Step 7700 | Loss: 0.4329 | Spent: 46.6 secs | LR: 0.000041
INFO:tensorflow:Step 7800 | Loss: 0.3161 | Spent: 46.2 secs | LR: 0.000040
INFO:tensorflow:Step 7900 | Loss: 0.4046 | Spent: 46.0 secs | LR: 0.000039
INFO:tensorflow:Step 8000 | Loss: 0.3155 | Spent: 46.3 secs | LR: 0.000039
INFO:tensorflow:Step 8100 | Loss: 0.2719 | Spent: 46.3 secs | LR: 0.000038
INFO:tensorflow:Step 8200 | Loss: 0.3072 | Spent: 46.0 secs | LR: 0.000038
INFO:tensorflow:Step 8300 | Loss: 0.3419 | Spent: 45.7 secs | LR: 0.000037
INFO:tensorflow:Step 8400 | Loss: 0.2591 | Spent: 46.2 secs | LR: 0.000036
INFO:tensorflow:Step 8500 | Loss: 0.4458 | Spent: 46.1 secs | LR: 0.000036
INFO:tensorflow:Step 8600 | Loss: 0.2940 | Spent: 46.0 secs | LR: 0.000035
INFO:tensorflow:Step 8700 | Loss: 0.4005 | Spent: 46.9 secs | LR: 0.000034
INFO:tensorflow:Step 8800 | Loss: 0.3060 | Spent: 46.1 secs | LR: 0.000034
INFO:tensorflow:Step 8900 | Loss: 0.2705 | Spent: 46.3 secs | LR: 0.000033
INFO:tensorflow:Step 9000 | Loss: 0.2774 | Spent: 46.3 secs | LR: 0.000032
INFO:tensorflow:Step 9100 | Loss: 0.2720 | Spent: 46.0 secs | LR: 0.000032
INFO:tensorflow:Step 9200 | Loss: 0.2324 | Spent: 46.4 secs | LR: 0.000031
INFO:tensorflow:Step 9300 | Loss: 0.3112 | Spent: 45.6 secs | LR: 0.000030
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.838
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 9400 | Loss: 0.3436 | Spent: 88.7 secs | LR: 0.000030
INFO:tensorflow:Step 9500 | Loss: 0.2131 | Spent: 46.5 secs | LR: 0.000029
INFO:tensorflow:Step 9600 | Loss: 0.3966 | Spent: 45.8 secs | LR: 0.000029
INFO:tensorflow:Step 9700 | Loss: 0.2481 | Spent: 45.8 secs | LR: 0.000028
INFO:tensorflow:Step 9800 | Loss: 0.2391 | Spent: 46.2 secs | LR: 0.000027
INFO:tensorflow:Step 9900 | Loss: 0.2131 | Spent: 45.8 secs | LR: 0.000027
INFO:tensorflow:Step 10000 | Loss: 0.2533 | Spent: 46.3 secs | LR: 0.000026
INFO:tensorflow:Step 10100 | Loss: 0.2357 | Spent: 46.2 secs | LR: 0.000025
INFO:tensorflow:Step 10200 | Loss: 0.2460 | Spent: 46.1 secs | LR: 0.000025
INFO:tensorflow:Step 10300 | Loss: 0.2183 | Spent: 46.8 secs | LR: 0.000024
INFO:tensorflow:Step 10400 | Loss: 0.3011 | Spent: 46.6 secs | LR: 0.000023
INFO:tensorflow:Step 10500 | Loss: 0.3408 | Spent: 46.7 secs | LR: 0.000023
INFO:tensorflow:Step 10600 | Loss: 0.3178 | Spent: 46.7 secs | LR: 0.000022
INFO:tensorflow:Step 10700 | Loss: 0.3286 | Spent: 46.4 secs | LR: 0.000022
INFO:tensorflow:Step 10800 | Loss: 0.2147 | Spent: 46.6 secs | LR: 0.000021
INFO:tensorflow:Step 10900 | Loss: 0.3564 | Spent: 46.7 secs | LR: 0.000020
INFO:tensorflow:Step 11000 | Loss: 0.3536 | Spent: 46.5 secs | LR: 0.000020
INFO:tensorflow:Step 11100 | Loss: 0.2032 | Spent: 45.8 secs | LR: 0.000019
INFO:tensorflow:Step 11200 | Loss: 0.2738 | Spent: 46.1 secs | LR: 0.000018
INFO:tensorflow:Step 11300 | Loss: 0.2014 | Spent: 45.3 secs | LR: 0.000018
INFO:tensorflow:Step 11400 | Loss: 0.2704 | Spent: 45.4 secs | LR: 0.000017
INFO:tensorflow:Step 11500 | Loss: 0.2060 | Spent: 46.1 secs | LR: 0.000016
INFO:tensorflow:Step 11600 | Loss: 0.2792 | Spent: 46.1 secs | LR: 0.000016
INFO:tensorflow:Step 11700 | Loss: 0.2379 | Spent: 46.6 secs | LR: 0.000015
INFO:tensorflow:Step 11800 | Loss: 0.2837 | Spent: 46.2 secs | LR: 0.000014
INFO:tensorflow:Step 11900 | Loss: 0.3181 | Spent: 46.5 secs | LR: 0.000014
INFO:tensorflow:Step 12000 | Loss: 0.2084 | Spent: 46.3 secs | LR: 0.000013
INFO:tensorflow:Step 12100 | Loss: 0.2370 | Spent: 46.6 secs | LR: 0.000013
INFO:tensorflow:Step 12200 | Loss: 0.2878 | Spent: 45.7 secs | LR: 0.000012
INFO:tensorflow:Step 12300 | Loss: 0.2024 | Spent: 47.3 secs | LR: 0.000011
INFO:tensorflow:Step 12400 | Loss: 0.2215 | Spent: 46.8 secs | LR: 0.000011
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.838
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 12500 | Loss: 0.2027 | Spent: 90.0 secs | LR: 0.000010
INFO:tensorflow:Step 12600 | Loss: 0.2311 | Spent: 47.3 secs | LR: 0.000010
INFO:tensorflow:Step 12700 | Loss: 0.2037 | Spent: 46.9 secs | LR: 0.000011
INFO:tensorflow:Step 12800 | Loss: 0.2688 | Spent: 46.2 secs | LR: 0.000011
INFO:tensorflow:Step 12900 | Loss: 0.2043 | Spent: 47.1 secs | LR: 0.000011
INFO:tensorflow:Step 13000 | Loss: 0.3522 | Spent: 46.8 secs | LR: 0.000012
INFO:tensorflow:Step 13100 | Loss: 0.2410 | Spent: 46.9 secs | LR: 0.000012
INFO:tensorflow:Step 13200 | Loss: 0.2688 | Spent: 46.3 secs | LR: 0.000012
INFO:tensorflow:Step 13300 | Loss: 0.2533 | Spent: 46.4 secs | LR: 0.000013
INFO:tensorflow:Step 13400 | Loss: 0.2105 | Spent: 46.6 secs | LR: 0.000013
INFO:tensorflow:Step 13500 | Loss: 0.2175 | Spent: 46.2 secs | LR: 0.000013
INFO:tensorflow:Step 13600 | Loss: 0.2188 | Spent: 46.9 secs | LR: 0.000014
INFO:tensorflow:Step 13700 | Loss: 0.3078 | Spent: 46.5 secs | LR: 0.000014
INFO:tensorflow:Step 13800 | Loss: 0.2066 | Spent: 46.4 secs | LR: 0.000014
INFO:tensorflow:Step 13900 | Loss: 0.2054 | Spent: 46.6 secs | LR: 0.000014
INFO:tensorflow:Step 14000 | Loss: 0.2015 | Spent: 46.2 secs | LR: 0.000015
INFO:tensorflow:Step 14100 | Loss: 0.2753 | Spent: 46.5 secs | LR: 0.000015
INFO:tensorflow:Step 14200 | Loss: 0.2034 | Spent: 45.6 secs | LR: 0.000015
INFO:tensorflow:Step 14300 | Loss: 0.2017 | Spent: 46.7 secs | LR: 0.000016
INFO:tensorflow:Step 14400 | Loss: 0.2026 | Spent: 45.7 secs | LR: 0.000016
INFO:tensorflow:Step 14500 | Loss: 0.2157 | Spent: 45.5 secs | LR: 0.000016
INFO:tensorflow:Step 14600 | Loss: 0.2369 | Spent: 45.9 secs | LR: 0.000017
INFO:tensorflow:Step 14700 | Loss: 0.2159 | Spent: 45.9 secs | LR: 0.000017
INFO:tensorflow:Step 14800 | Loss: 0.2165 | Spent: 46.4 secs | LR: 0.000017
INFO:tensorflow:Step 14900 | Loss: 0.2042 | Spent: 46.3 secs | LR: 0.000018
INFO:tensorflow:Step 15000 | Loss: 0.2185 | Spent: 46.9 secs | LR: 0.000018
INFO:tensorflow:Step 15100 | Loss: 0.2001 | Spent: 46.1 secs | LR: 0.000018
INFO:tensorflow:Step 15200 | Loss: 0.2046 | Spent: 46.8 secs | LR: 0.000019
INFO:tensorflow:Step 15300 | Loss: 0.2016 | Spent: 45.7 secs | LR: 0.000019
INFO:tensorflow:Step 15400 | Loss: 0.2154 | Spent: 46.4 secs | LR: 0.000019
INFO:tensorflow:Step 15500 | Loss: 0.3159 | Spent: 46.6 secs | LR: 0.000020
INFO:tensorflow:Step 15600 | Loss: 0.2162 | Spent: 46.3 secs | LR: 0.000020
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.836
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 15700 | Loss: 0.2213 | Spent: 88.4 secs | LR: 0.000020
INFO:tensorflow:Step 15800 | Loss: 0.2036 | Spent: 46.3 secs | LR: 0.000021
INFO:tensorflow:Step 15900 | Loss: 0.2833 | Spent: 45.6 secs | LR: 0.000021
INFO:tensorflow:Step 16000 | Loss: 0.2005 | Spent: 46.5 secs | LR: 0.000021
INFO:tensorflow:Step 16100 | Loss: 0.2330 | Spent: 46.4 secs | LR: 0.000022
INFO:tensorflow:Step 16200 | Loss: 0.2011 | Spent: 46.0 secs | LR: 0.000022
INFO:tensorflow:Step 16300 | Loss: 0.2317 | Spent: 46.3 secs | LR: 0.000022
INFO:tensorflow:Step 16400 | Loss: 0.2205 | Spent: 46.0 secs | LR: 0.000022
INFO:tensorflow:Step 16500 | Loss: 0.2031 | Spent: 46.8 secs | LR: 0.000023
INFO:tensorflow:Step 16600 | Loss: 0.2760 | Spent: 46.9 secs | LR: 0.000023
INFO:tensorflow:Step 16700 | Loss: 0.2275 | Spent: 47.2 secs | LR: 0.000023
INFO:tensorflow:Step 16800 | Loss: 0.2096 | Spent: 46.9 secs | LR: 0.000024
INFO:tensorflow:Step 16900 | Loss: 0.2049 | Spent: 46.7 secs | LR: 0.000024
INFO:tensorflow:Step 17000 | Loss: 0.3602 | Spent: 47.3 secs | LR: 0.000024
INFO:tensorflow:Step 17100 | Loss: 0.2597 | Spent: 46.6 secs | LR: 0.000025
INFO:tensorflow:Step 17200 | Loss: 0.2436 | Spent: 46.9 secs | LR: 0.000025
INFO:tensorflow:Step 17300 | Loss: 0.2122 | Spent: 46.0 secs | LR: 0.000025
INFO:tensorflow:Step 17400 | Loss: 0.2847 | Spent: 47.1 secs | LR: 0.000026
INFO:tensorflow:Step 17500 | Loss: 0.2460 | Spent: 46.3 secs | LR: 0.000026
INFO:tensorflow:Step 17600 | Loss: 0.2081 | Spent: 45.0 secs | LR: 0.000026
INFO:tensorflow:Step 17700 | Loss: 0.2250 | Spent: 46.5 secs | LR: 0.000027
INFO:tensorflow:Step 17800 | Loss: 0.2863 | Spent: 45.5 secs | LR: 0.000027
INFO:tensorflow:Step 17900 | Loss: 0.2132 | Spent: 46.2 secs | LR: 0.000027
INFO:tensorflow:Step 18000 | Loss: 0.2038 | Spent: 46.1 secs | LR: 0.000028
INFO:tensorflow:Step 18100 | Loss: 0.2063 | Spent: 46.5 secs | LR: 0.000028
INFO:tensorflow:Step 18200 | Loss: 0.3367 | Spent: 46.5 secs | LR: 0.000028
INFO:tensorflow:Step 18300 | Loss: 0.2198 | Spent: 46.3 secs | LR: 0.000029
INFO:tensorflow:Step 18400 | Loss: 0.2637 | Spent: 46.4 secs | LR: 0.000029
INFO:tensorflow:Step 18500 | Loss: 0.2416 | Spent: 45.9 secs | LR: 0.000029
INFO:tensorflow:Step 18600 | Loss: 0.2060 | Spent: 46.6 secs | LR: 0.000030
INFO:tensorflow:Step 18700 | Loss: 0.2525 | Spent: 46.1 secs | LR: 0.000030
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.831
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 18800 | Loss: 0.2107 | Spent: 89.2 secs | LR: 0.000030
INFO:tensorflow:Step 18900 | Loss: 0.2385 | Spent: 46.0 secs | LR: 0.000030
INFO:tensorflow:Step 19000 | Loss: 0.2215 | Spent: 45.5 secs | LR: 0.000029
INFO:tensorflow:Step 19100 | Loss: 0.2147 | Spent: 45.9 secs | LR: 0.000029
INFO:tensorflow:Step 19200 | Loss: 0.2240 | Spent: 45.9 secs | LR: 0.000029
INFO:tensorflow:Step 19300 | Loss: 0.2043 | Spent: 46.1 secs | LR: 0.000028
INFO:tensorflow:Step 19400 | Loss: 0.3482 | Spent: 46.2 secs | LR: 0.000028
INFO:tensorflow:Step 19500 | Loss: 0.2333 | Spent: 46.1 secs | LR: 0.000028
INFO:tensorflow:Step 19600 | Loss: 0.2094 | Spent: 46.3 secs | LR: 0.000027
INFO:tensorflow:Step 19700 | Loss: 0.2158 | Spent: 46.5 secs | LR: 0.000027
INFO:tensorflow:Step 19800 | Loss: 0.2123 | Spent: 46.4 secs | LR: 0.000027
INFO:tensorflow:Step 19900 | Loss: 0.2054 | Spent: 46.5 secs | LR: 0.000026
INFO:tensorflow:Step 20000 | Loss: 0.3438 | Spent: 46.4 secs | LR: 0.000026
INFO:tensorflow:Step 20100 | Loss: 0.2247 | Spent: 47.4 secs | LR: 0.000026
INFO:tensorflow:Step 20200 | Loss: 0.3556 | Spent: 46.8 secs | LR: 0.000025
INFO:tensorflow:Step 20300 | Loss: 0.2192 | Spent: 46.7 secs | LR: 0.000025
INFO:tensorflow:Step 20400 | Loss: 0.2671 | Spent: 46.6 secs | LR: 0.000025
INFO:tensorflow:Step 20500 | Loss: 0.2152 | Spent: 46.7 secs | LR: 0.000024
INFO:tensorflow:Step 20600 | Loss: 0.2035 | Spent: 46.3 secs | LR: 0.000024
INFO:tensorflow:Step 20700 | Loss: 0.2056 | Spent: 45.9 secs | LR: 0.000024
INFO:tensorflow:Step 20800 | Loss: 0.2500 | Spent: 46.1 secs | LR: 0.000023
INFO:tensorflow:Step 20900 | Loss: 0.2072 | Spent: 46.0 secs | LR: 0.000023
INFO:tensorflow:Step 21000 | Loss: 0.2193 | Spent: 46.5 secs | LR: 0.000023
INFO:tensorflow:Step 21100 | Loss: 0.2104 | Spent: 46.0 secs | LR: 0.000022
INFO:tensorflow:Step 21200 | Loss: 0.2120 | Spent: 46.7 secs | LR: 0.000022
INFO:tensorflow:Step 21300 | Loss: 0.2010 | Spent: 46.4 secs | LR: 0.000022
INFO:tensorflow:Step 21400 | Loss: 0.2744 | Spent: 46.6 secs | LR: 0.000022
INFO:tensorflow:Step 21500 | Loss: 0.2019 | Spent: 46.3 secs | LR: 0.000021
INFO:tensorflow:Step 21600 | Loss: 0.2015 | Spent: 45.9 secs | LR: 0.000021
INFO:tensorflow:Step 21700 | Loss: 0.2089 | Spent: 46.6 secs | LR: 0.000021
INFO:tensorflow:Step 21800 | Loss: 0.2006 | Spent: 45.8 secs | LR: 0.000020
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.837
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 21900 | Loss: 0.2006 | Spent: 89.0 secs | LR: 0.000020
INFO:tensorflow:Step 22000 | Loss: 0.2045 | Spent: 46.9 secs | LR: 0.000020
INFO:tensorflow:Step 22100 | Loss: 0.3336 | Spent: 45.9 secs | LR: 0.000019
INFO:tensorflow:Step 22200 | Loss: 0.2032 | Spent: 45.6 secs | LR: 0.000019
INFO:tensorflow:Step 22300 | Loss: 0.2048 | Spent: 46.3 secs | LR: 0.000019
INFO:tensorflow:Step 22400 | Loss: 0.2093 | Spent: 46.1 secs | LR: 0.000018
INFO:tensorflow:Step 22500 | Loss: 0.2045 | Spent: 46.2 secs | LR: 0.000018
INFO:tensorflow:Step 22600 | Loss: 0.2936 | Spent: 45.7 secs | LR: 0.000018
INFO:tensorflow:Step 22700 | Loss: 0.2004 | Spent: 46.6 secs | LR: 0.000017
INFO:tensorflow:Step 22800 | Loss: 0.2422 | Spent: 46.4 secs | LR: 0.000017
INFO:tensorflow:Step 22900 | Loss: 0.2006 | Spent: 46.5 secs | LR: 0.000017
INFO:tensorflow:Step 23000 | Loss: 0.2055 | Spent: 46.5 secs | LR: 0.000016
INFO:tensorflow:Step 23100 | Loss: 0.2003 | Spent: 46.4 secs | LR: 0.000016
INFO:tensorflow:Step 23200 | Loss: 0.2023 | Spent: 45.9 secs | LR: 0.000016
INFO:tensorflow:Step 23300 | Loss: 0.2022 | Spent: 46.4 secs | LR: 0.000015
INFO:tensorflow:Step 23400 | Loss: 0.2948 | Spent: 46.9 secs | LR: 0.000015
INFO:tensorflow:Step 23500 | Loss: 0.2768 | Spent: 46.2 secs | LR: 0.000015
INFO:tensorflow:Step 23600 | Loss: 0.1994 | Spent: 46.0 secs | LR: 0.000014
INFO:tensorflow:Step 23700 | Loss: 0.2010 | Spent: 46.1 secs | LR: 0.000014
INFO:tensorflow:Step 23800 | Loss: 0.2059 | Spent: 45.5 secs | LR: 0.000014
INFO:tensorflow:Step 23900 | Loss: 0.2659 | Spent: 45.4 secs | LR: 0.000014
INFO:tensorflow:Step 24000 | Loss: 0.2555 | Spent: 45.9 secs | LR: 0.000013
INFO:tensorflow:Step 24100 | Loss: 0.2785 | Spent: 46.3 secs | LR: 0.000013
INFO:tensorflow:Step 24200 | Loss: 0.1995 | Spent: 46.0 secs | LR: 0.000013
INFO:tensorflow:Step 24300 | Loss: 0.2023 | Spent: 46.4 secs | LR: 0.000012
INFO:tensorflow:Step 24400 | Loss: 0.2008 | Spent: 46.4 secs | LR: 0.000012
INFO:tensorflow:Step 24500 | Loss: 0.1994 | Spent: 46.3 secs | LR: 0.000012
INFO:tensorflow:Step 24600 | Loss: 0.2012 | Spent: 46.5 secs | LR: 0.000011
INFO:tensorflow:Step 24700 | Loss: 0.2015 | Spent: 45.7 secs | LR: 0.000011
INFO:tensorflow:Step 24800 | Loss: 0.1996 | Spent: 46.6 secs | LR: 0.000011
INFO:tensorflow:Step 24900 | Loss: 0.2009 | Spent: 45.5 secs | LR: 0.000010
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.837
INFO:tensorflow:Best Accuracy: 0.838
Reading ../data/train.csv
INFO:tensorflow:Step 25000 | Loss: 0.2000 | Spent: 88.2 secs | LR: 0.000010
INFO:tensorflow:Step 25100 | Loss: 0.2882 | Spent: 46.3 secs | LR: 0.000010
INFO:tensorflow:Step 25200 | Loss: 0.2006 | Spent: 46.1 secs | LR: 0.000010
INFO:tensorflow:Step 25300 | Loss: 0.1999 | Spent: 45.7 secs | LR: 0.000010
INFO:tensorflow:Step 25400 | Loss: 0.2046 | Spent: 45.5 secs | LR: 0.000011
INFO:tensorflow:Step 25500 | Loss: 0.2045 | Spent: 45.5 secs | LR: 0.000011
INFO:tensorflow:Step 25600 | Loss: 0.2008 | Spent: 46.5 secs | LR: 0.000011
INFO:tensorflow:Step 25700 | Loss: 0.2886 | Spent: 45.7 secs | LR: 0.000011
INFO:tensorflow:Step 25800 | Loss: 0.1998 | Spent: 46.1 secs | LR: 0.000011
INFO:tensorflow:Step 25900 | Loss: 0.2365 | Spent: 46.1 secs | LR: 0.000011
INFO:tensorflow:Step 26000 | Loss: 0.2012 | Spent: 46.1 secs | LR: 0.000012
INFO:tensorflow:Step 26100 | Loss: 0.2007 | Spent: 46.1 secs | LR: 0.000012
INFO:tensorflow:Step 26200 | Loss: 0.2004 | Spent: 46.0 secs | LR: 0.000012
INFO:tensorflow:Step 26300 | Loss: 0.2001 | Spent: 45.4 secs | LR: 0.000012
INFO:tensorflow:Step 26400 | Loss: 0.2013 | Spent: 45.8 secs | LR: 0.000012
INFO:tensorflow:Step 26500 | Loss: 0.2594 | Spent: 46.3 secs | LR: 0.000012
INFO:tensorflow:Step 26600 | Loss: 0.1997 | Spent: 45.6 secs | LR: 0.000013
INFO:tensorflow:Step 26700 | Loss: 0.1999 | Spent: 45.6 secs | LR: 0.000013
INFO:tensorflow:Step 26800 | Loss: 0.2002 | Spent: 45.7 secs | LR: 0.000013
INFO:tensorflow:Step 26900 | Loss: 0.2001 | Spent: 45.1 secs | LR: 0.000013
INFO:tensorflow:Step 27000 | Loss: 0.2024 | Spent: 44.7 secs | LR: 0.000013
INFO:tensorflow:Step 27100 | Loss: 0.1996 | Spent: 45.7 secs | LR: 0.000013
INFO:tensorflow:Step 27200 | Loss: 0.1997 | Spent: 45.2 secs | LR: 0.000014
INFO:tensorflow:Step 27300 | Loss: 0.2052 | Spent: 45.3 secs | LR: 0.000014
INFO:tensorflow:Step 27400 | Loss: 0.2000 | Spent: 45.6 secs | LR: 0.000014
INFO:tensorflow:Step 27500 | Loss: 0.2042 | Spent: 46.1 secs | LR: 0.000014
INFO:tensorflow:Step 27600 | Loss: 0.2004 | Spent: 45.6 secs | LR: 0.000014
INFO:tensorflow:Step 27700 | Loss: 0.1997 | Spent: 46.0 secs | LR: 0.000014
INFO:tensorflow:Step 27800 | Loss: 0.1999 | Spent: 45.0 secs | LR: 0.000014
INFO:tensorflow:Step 27900 | Loss: 0.2023 | Spent: 45.8 secs | LR: 0.000015
INFO:tensorflow:Step 28000 | Loss: 0.3268 | Spent: 46.6 secs | LR: 0.000015
INFO:tensorflow:Step 28100 | Loss: 0.2071 | Spent: 45.8 secs | LR: 0.000015
Reading ../data/test.csv
INFO:tensorflow:Evaluation: Testing Accuracy: 0.836
INFO:tensorflow:Best Accuracy: 0.838
5 times not improve the best result, therefore stop training