1. tensorflow-nlp
  2. finch
  3. tensorflow2
  4. text_classification
  5. imdb
  6. main
In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_classification/imdb/main')

Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons
!pip install transformers

Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)
Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)
Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)
Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)
Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)
Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)
Requirement already satisfied: dataclasses; python_version < "3.7" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)
Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)
Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)
In [3]:
from transformers import BertTokenizer, TFBertModel

import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 2.2.0
WARNING:tensorflow:From <ipython-input-3-0175c78ad17c>:11: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
params = {
  'train_paths': [
    '../data/train_bt_part1.txt',
    '../data/train_bt_part2.txt',
    '../data/train_bt_part3.txt',
    '../data/train_bt_part4.txt',
    '../data/train_bt_part5.txt',
    '../data/train_bt_part6.txt',
  ],
  'test_paths': [
    '../data/test.txt',
  ],
  'pretrain_path': 'bert-base-uncased',
  'num_samples': 25000 * 2,
  'batch_size': 8,
  'max_len': 300,
  'num_patience': 5,
  'init_lr': 1e-5,
  'max_lr': 3e-5,
}
In [5]:
tokenizer = BertTokenizer.from_pretrained(params['pretrain_path'],
                                          lowercase = True,
                                          add_special_tokens = True)
In [6]:
def data_generator(f_paths, params):
  for f_path in f_paths:
    with open(f_path) as f:
      print('Reading', f_path)
      for line in f:
        line = line.rstrip()
        label, text = line.split('\t')
        text = ['[CLS]'] + tokenizer.tokenize(text) + ['[SEP]']
        if len(text) > params['max_len']:
          _max_len = params['max_len'] // 2
          text = text[:_max_len] + text[-_max_len:]
        seg = [0] * len(text)
        text = tokenizer.convert_tokens_to_ids(text)
        y = int(label)
        yield text, seg, y


def dataset(is_training, params):
  _shapes = ([None], [None], ())
  _types = (tf.int32, tf.int32, tf.int32)
  _pads = (0, 0, -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_paths'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [7]:
# input stream ids check
text, seg, _ = next(data_generator(params['train_paths'], params))
print(text)
print(seg)

Reading ../data/train_bt_part1.txt
[101, 1045, 2876, 1005, 1056, 9278, 2023, 2028, 2130, 2006, 7922, 12635, 2305, 102]
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
In [8]:
class BertFinetune(tf.keras.Model):
  def __init__(self, params):
    super(BertFinetune, self).__init__()
    self.bert = TFBertModel.from_pretrained(params['pretrain_path'],
                                            trainable = True)
    self.drop_1 = tf.keras.layers.Dropout(.1)
    self.fc = tf.keras.layers.Dense(300, tf.nn.swish, name='down_stream/fc')
    self.drop_2 = tf.keras.layers.Dropout(.1)
    self.out = tf.keras.layers.Dense(2, name='down_stream/out')

  def call(self, bert_inputs, training):
    bert_inputs = [tf.cast(inp, tf.int32) for inp in bert_inputs]
    x = self.bert(bert_inputs, training=training)[1]
    x = self.drop_1(x, training=training)
    x = self.fc(x)
    x = self.drop_2(x, training=training)
    x = self.out(x)
    return x
In [ ]:
model = BertFinetune(params)
model.build([[None, None], [None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

step_size = 2 * params['num_samples'] // params['batch_size']
decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = step_size,)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc = .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for (text, seg, labels) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model([text, tf.sign(text), seg], training=True)
      loss = tf.compat.v1.losses.softmax_cross_entropy(
        tf.one_hot(labels, 2, dtype=tf.float32),
        logits = logits,
        label_smoothing = .2,)
      
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, 5.)
    optim.apply_gradients(zip(grads, model.trainable_variables))
    
    if global_step % 100 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()

  for (text, seg, labels) in dataset(is_training=False, params=params):
    logits = model([text, tf.sign(text), seg], training=False)
    m.update_state(y_true=labels, y_pred=tf.argmax(logits, -1))

  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))

  if acc > best_acc:
    best_acc = acc
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best Accuracy: {:.3f}".format(best_acc))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break

Some weights of the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']
- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFBertModel were initialized from the model checkpoint at bert-base-uncased.
If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.

[('tf_bert_model/bert/embeddings/word_embeddings/weight:0',
  TensorShape([30522, 768])),
 ('tf_bert_model/bert/embeddings/position_embeddings/embeddings:0',
  TensorShape([512, 768])),
 ('tf_bert_model/bert/embeddings/token_type_embeddings/embeddings:0',
  TensorShape([2, 768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/gamma:0', TensorShape([768])),
 ('tf_bert_model/bert/embeddings/LayerNorm/beta:0', TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._0/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._0/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._1/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._1/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._2/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._2/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._3/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._3/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._4/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._4/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._5/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._5/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._6/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._6/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._7/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._7/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._8/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._8/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._9/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._9/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._10/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._10/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/query/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/key/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/self/value/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/kernel:0',
  TensorShape([768, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/kernel:0',
  TensorShape([768, 3072])),
 ('tf_bert_model/bert/encoder/layer_._11/intermediate/dense/bias:0',
  TensorShape([3072])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/kernel:0',
  TensorShape([3072, 768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/dense/bias:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/gamma:0',
  TensorShape([768])),
 ('tf_bert_model/bert/encoder/layer_._11/output/LayerNorm/beta:0',
  TensorShape([768])),
 ('tf_bert_model/bert/pooler/dense/kernel:0', TensorShape([768, 768])),
 ('tf_bert_model/bert/pooler/dense/bias:0', TensorShape([768])),
 ('down_stream/fc/kernel:0', TensorShape([768, 300])),
 ('down_stream/fc/bias:0', TensorShape([300])),
 ('down_stream/out/kernel:0', TensorShape([300, 2])),
 ('down_stream/out/bias:0', TensorShape([2]))]
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt
Reading ../data/train_bt_part3.txt
Reading ../data/train_bt_part4.txt
Reading ../data/train_bt_part5.txt
Reading ../data/train_bt_part6.txt
INFO:tensorflow:Step 0 | Loss: 0.6308 | Spent: 153.8 secs | LR: 0.000010
INFO:tensorflow:Step 100 | Loss: 0.4728 | Spent: 61.4 secs | LR: 0.000010
INFO:tensorflow:Step 200 | Loss: 0.4458 | Spent: 62.3 secs | LR: 0.000010
INFO:tensorflow:Step 300 | Loss: 0.3979 | Spent: 62.7 secs | LR: 0.000010
INFO:tensorflow:Step 400 | Loss: 0.3815 | Spent: 63.5 secs | LR: 0.000011
INFO:tensorflow:Step 500 | Loss: 0.3344 | Spent: 63.7 secs | LR: 0.000011
INFO:tensorflow:Step 600 | Loss: 0.6167 | Spent: 63.8 secs | LR: 0.000011
INFO:tensorflow:Step 700 | Loss: 0.5215 | Spent: 64.2 secs | LR: 0.000011
INFO:tensorflow:Step 800 | Loss: 0.3452 | Spent: 64.1 secs | LR: 0.000011
INFO:tensorflow:Step 900 | Loss: 0.3315 | Spent: 64.6 secs | LR: 0.000011
INFO:tensorflow:Step 1000 | Loss: 0.5024 | Spent: 63.9 secs | LR: 0.000012
INFO:tensorflow:Step 1100 | Loss: 0.4888 | Spent: 64.2 secs | LR: 0.000012
INFO:tensorflow:Step 1200 | Loss: 0.3642 | Spent: 64.6 secs | LR: 0.000012
INFO:tensorflow:Step 1300 | Loss: 0.4719 | Spent: 63.9 secs | LR: 0.000012
INFO:tensorflow:Step 1400 | Loss: 0.3779 | Spent: 64.4 secs | LR: 0.000012
INFO:tensorflow:Step 1500 | Loss: 0.3994 | Spent: 64.2 secs | LR: 0.000012
INFO:tensorflow:Step 1600 | Loss: 0.5677 | Spent: 64.2 secs | LR: 0.000013
INFO:tensorflow:Step 1700 | Loss: 0.3384 | Spent: 64.1 secs | LR: 0.000013
INFO:tensorflow:Step 1800 | Loss: 0.3328 | Spent: 64.2 secs | LR: 0.000013
INFO:tensorflow:Step 1900 | Loss: 0.4832 | Spent: 64.4 secs | LR: 0.000013
INFO:tensorflow:Step 2000 | Loss: 0.3639 | Spent: 64.4 secs | LR: 0.000013
INFO:tensorflow:Step 2100 | Loss: 0.3408 | Spent: 63.9 secs | LR: 0.000013
INFO:tensorflow:Step 2200 | Loss: 0.3400 | Spent: 63.8 secs | LR: 0.000014
INFO:tensorflow:Step 2300 | Loss: 0.6340 | Spent: 64.2 secs | LR: 0.000014
INFO:tensorflow:Step 2400 | Loss: 0.6736 | Spent: 63.5 secs | LR: 0.000014
INFO:tensorflow:Step 2500 | Loss: 0.5442 | Spent: 64.0 secs | LR: 0.000014
INFO:tensorflow:Step 2600 | Loss: 0.3924 | Spent: 63.9 secs | LR: 0.000014
INFO:tensorflow:Step 2700 | Loss: 0.4568 | Spent: 64.2 secs | LR: 0.000014
INFO:tensorflow:Step 2800 | Loss: 0.5080 | Spent: 64.2 secs | LR: 0.000014
INFO:tensorflow:Step 2900 | Loss: 0.4234 | Spent: 64.3 secs | LR: 0.000015
INFO:tensorflow:Step 3000 | Loss: 0.3841 | Spent: 64.2 secs | LR: 0.000015
INFO:tensorflow:Step 3100 | Loss: 0.3300 | Spent: 63.8 secs | LR: 0.000015
INFO:tensorflow:Step 3200 | Loss: 0.3423 | Spent: 64.0 secs | LR: 0.000015
INFO:tensorflow:Step 3300 | Loss: 0.3842 | Spent: 64.9 secs | LR: 0.000015
INFO:tensorflow:Step 3400 | Loss: 0.3280 | Spent: 63.4 secs | LR: 0.000015
INFO:tensorflow:Step 3500 | Loss: 0.5383 | Spent: 63.5 secs | LR: 0.000016
INFO:tensorflow:Step 3600 | Loss: 0.4577 | Spent: 63.9 secs | LR: 0.000016
INFO:tensorflow:Step 3700 | Loss: 0.3489 | Spent: 64.2 secs | LR: 0.000016
INFO:tensorflow:Step 3800 | Loss: 0.3442 | Spent: 63.3 secs | LR: 0.000016
INFO:tensorflow:Step 3900 | Loss: 0.4467 | Spent: 64.7 secs | LR: 0.000016
INFO:tensorflow:Step 4000 | Loss: 0.4077 | Spent: 64.6 secs | LR: 0.000016
INFO:tensorflow:Step 4100 | Loss: 0.3527 | Spent: 64.6 secs | LR: 0.000017
INFO:tensorflow:Step 4200 | Loss: 0.4292 | Spent: 63.5 secs | LR: 0.000017
INFO:tensorflow:Step 4300 | Loss: 0.5467 | Spent: 64.3 secs | LR: 0.000017
INFO:tensorflow:Step 4400 | Loss: 0.3588 | Spent: 64.4 secs | LR: 0.000017
INFO:tensorflow:Step 4500 | Loss: 0.4578 | Spent: 64.1 secs | LR: 0.000017
INFO:tensorflow:Step 4600 | Loss: 0.5283 | Spent: 64.4 secs | LR: 0.000017
INFO:tensorflow:Step 4700 | Loss: 0.5562 | Spent: 64.2 secs | LR: 0.000018
INFO:tensorflow:Step 4800 | Loss: 0.3287 | Spent: 64.0 secs | LR: 0.000018
INFO:tensorflow:Step 4900 | Loss: 0.3298 | Spent: 63.6 secs | LR: 0.000018
INFO:tensorflow:Step 5000 | Loss: 0.4181 | Spent: 63.7 secs | LR: 0.000018
INFO:tensorflow:Step 5100 | Loss: 0.4582 | Spent: 63.7 secs | LR: 0.000018
INFO:tensorflow:Step 5200 | Loss: 0.5596 | Spent: 64.1 secs | LR: 0.000018
INFO:tensorflow:Step 5300 | Loss: 0.3356 | Spent: 64.7 secs | LR: 0.000018
INFO:tensorflow:Step 5400 | Loss: 0.3717 | Spent: 64.1 secs | LR: 0.000019
INFO:tensorflow:Step 5500 | Loss: 0.3399 | Spent: 64.4 secs | LR: 0.000019
INFO:tensorflow:Step 5600 | Loss: 0.3458 | Spent: 64.4 secs | LR: 0.000019
INFO:tensorflow:Step 5700 | Loss: 0.3287 | Spent: 64.4 secs | LR: 0.000019
INFO:tensorflow:Step 5800 | Loss: 0.3350 | Spent: 64.0 secs | LR: 0.000019
INFO:tensorflow:Step 5900 | Loss: 0.4524 | Spent: 64.2 secs | LR: 0.000019
INFO:tensorflow:Step 6000 | Loss: 0.3419 | Spent: 63.3 secs | LR: 0.000020
INFO:tensorflow:Step 6100 | Loss: 0.3461 | Spent: 64.4 secs | LR: 0.000020
INFO:tensorflow:Step 6200 | Loss: 0.3327 | Spent: 64.5 secs | LR: 0.000020
Reading ../data/test.txt
INFO:tensorflow:Evaluation: Testing Accuracy: 0.940
INFO:tensorflow:Best Accuracy: 0.940
Reading ../data/train_bt_part1.txt
Reading ../data/train_bt_part2.txt