1. tensorflow-nlp
  2. finch
  3. tensorflow2
  4. text_matching
  5. ant
  6. main
In [1]:
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/text_matching/ant/main')

Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
%tensorflow_version 2.x
!pip install tensorflow-addons

Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.8.3)
Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons) (2.7.1)
In [3]:
from sklearn.metrics import classification_report

import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pprint
import logging
import time
import math
import json

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 2.3.0
WARNING:tensorflow:From <ipython-input-3-be5b89cffc7a>:13: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.config.list_physical_devices('GPU')` instead.
GPU Enabled: True
In [4]:
def get_vocab(f_path):
  k2v = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      k2v[line] = i
  return k2v
In [5]:
# stream data from text files
def data_generator(f_path, char2idx):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = json.loads(line.rstrip())
      text1, text2, label = line['sentence1'], line['sentence2'], line['label']
      text1 = [char2idx.get(c, len(char2idx)) for c in list(text1)]
      text2 = [char2idx.get(c, len(char2idx)) for c in list(text2)]
      yield ((text1, text2), int(label))


def dataset(is_training, params):
  _shapes = (([None], [None]), ())
  _types = ((tf.int32, tf.int32), tf.int32)
  _pads = ((0, 0), -1)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params['char2idx']),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params['char2idx']),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds
In [6]:
class FFNBlock(tf.keras.Model):
  def __init__(self, params, name):
    super().__init__(name = name)
    self.dropout1 = tf.keras.layers.Dropout(params['dropout_rate'])
    self.fc1 = tf.keras.layers.Dense(params['hidden_units'], params['activation'])
    self.dropout2 = tf.keras.layers.Dropout(params['dropout_rate'])
    self.fc2 = tf.keras.layers.Dense(params['hidden_units'], params['activation'])
  
  def call(self, inputs, training=False):
    x = inputs
    x = self.dropout1(x, training=training)
    x = self.fc1(x)
    x = self.dropout2(x, training=training)
    x = self.fc2(x)
    return x
In [7]:
class Pyramid(tf.keras.Model):
  def __init__(self, params: dict):
    super().__init__()
    self.embedding = tf.Variable(np.load(params['embedding_path']), name='pretrained_embedding', dtype=tf.float32)
    
    self.inp_dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    
    self.encoder = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(
      params['hidden_units'], return_sequences=True, zero_output_for_mask=True), name='encoder')
    
    self.conv_1 = tf.keras.layers.Conv2D(filters=32, kernel_size=3, activation=params['activation'], padding='same')
    
    self.conv_2 = tf.keras.layers.Conv2D(filters=64, kernel_size=3, activation=params['activation'], padding='same')
    
    self.conv_3 = tf.keras.layers.Conv2D(filters=128, kernel_size=3, activation=params['activation'], padding='same')
    
    self.W_0 = tf.keras.layers.Dense(2*params['hidden_units'], use_bias=False)
    
    self.W_1_1 = tf.keras.layers.Dense(params['hidden_units'], use_bias=False)
    
    self.W_1_2 = tf.keras.layers.Dense(params['hidden_units'], use_bias=False)
    
    self.v_1 = tf.keras.layers.Dense(1, use_bias=False)
    
    self.W_2 = tf.keras.layers.Dense(params['hidden_units'], use_bias=False)
    
    self.v_2 = tf.keras.layers.Dense(1, use_bias=False)
    
    self.W_3 = tf.keras.layers.Dense(params['hidden_units'], use_bias=False)
    
    self.v_3 = tf.keras.layers.Dense(1, use_bias=False)
    
    self.flatten = tf.keras.layers.Flatten()
    
    self.out_hidden = FFNBlock(params, name='out_hidden')
    
    self.out_linear = tf.keras.layers.Dense(1, name='out_linear')

  
  def get_stride(self, x, fixed_len):
    batch_sz = tf.shape(x)[0]
    len = x.shape[1]
    stride = len // fixed_len
    if len // stride != fixed_len:
      remin = (stride + 1) * fixed_len - len
      zeros = tf.zeros([batch_sz, remin], tf.int32)
      x = tf.concat([x, zeros], 1)
      len = x.shape[1]
      stride = len // fixed_len
    return x, stride
  
  
  def call(self, inputs, training=False):
    x1, x2 = inputs
    
    if x1.dtype != tf.int32:
      x1 = tf.cast(x1, tf.int32)
    if x2.dtype != tf.int32:
      x2 = tf.cast(x2, tf.int32)
    
    x1, stride1 = self.get_stride(x1, params['fixed_len1'])
    x2, stride2 = self.get_stride(x2, params['fixed_len2'])
    
    mask1 = tf.sign(x1)
    mask2 = tf.sign(x2)
    
    x1 = tf.nn.embedding_lookup(self.embedding, x1)
    x2 = tf.nn.embedding_lookup(self.embedding, x2)
    
    x1 = self.inp_dropout(x1, training=training)
    x2 = self.inp_dropout(x2, training=training)
    
    x1 = self.encoder(x1, mask=tf.cast(mask1, tf.bool))
    x2 = self.encoder(x2, mask=tf.cast(mask2, tf.bool))
    x = []
    
    # attention 1 (bilinear)
    a = tf.matmul(x1, self.W_0(x2), transpose_b=True)
    x.append(tf.expand_dims(a, -1))
    
    # attention 2 (add)
    a1 = tf.expand_dims(self.W_1_1(x1), 2)
    a2 = tf.expand_dims(self.W_1_2(x2), 1)
    x.append(self.v_1(tf.tanh(a1 + a2)))
    
    # attention 3 (minus)
    a1 = tf.expand_dims(x1, 2)
    a2 = tf.expand_dims(x2, 1)
    x.append(self.v_2(tf.tanh(self.W_2(tf.abs(a1 - a2)))))
    
    # attention 4 (dot)
    a1 = tf.expand_dims(x1, 2)
    a2 = tf.expand_dims(x2, 1)
    x.append(self.v_3(tf.tanh(self.W_3(a1 * a2))))
    
    x = tf.concat(x, -1)
    x = self.conv_1(x)
    x = tf.nn.max_pool(x, [1, stride1, stride2, 1], [1, stride1, stride2, 1], 'VALID')
    x = self.conv_2(x)
    x = tf.nn.max_pool(x, [1, 2, 2, 1], [1, 2, 2, 1], 'VALID')
    x = self.conv_3(x)
    x = tf.nn.max_pool(x, [1, 2, 2, 1], [1, 2, 2, 1], 'VALID')
    
    x = self.flatten(x)
    x = self.out_hidden(x, training=training)
    x = self.out_linear(x)
    x = tf.squeeze(x, 1)
    
    return x
In [8]:
params = {
  'train_path': '../data/train.json',
  'test_path': '../data/dev.json',
  'vocab_path': '../vocab/char.txt',
  'embedding_path': '../vocab/char.npy',
  'batch_size': 32,
  'buffer_size': 34334,
  'dropout_rate': .2,
  'hidden_units': 300,
  'fixed_len1': 12,
  'fixed_len2': 12,
  'activation': tf.nn.swish,
  'init_lr': 1e-4,
  'max_lr': 8e-4,
  'label_smooth': .0,
  'clip_norm': .1,
  'num_patience': 10,
}
In [9]:
def label_smoothing(label, smooth):
  if smooth > 0.:
    return label * (1 - smooth) + 0.5 * smooth
  else:
    return label

params['char2idx'] = get_vocab(params['vocab_path'])
params['vocab_size'] = len(params['char2idx']) + 1

# input stream ids check
(text1, text2), _ = next(data_generator(params['train_path'], params['char2idx']))
print(text1)
print(text2)

Reading ../data/train.json
[26, 25, 5, 1, 416, 20, 4, 7, 21, 22, 99, 128, 301, 87, 51, 246, 15]
[5, 1, 23, 301, 87, 38, 19, 4, 246, 15]
In [10]:
model = Pyramid(params)
model.build([[None, 13], [None, 13]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tfa.optimizers.Triangular2CyclicalLearningRate(
  initial_learning_rate = params['init_lr'],
  maximal_learning_rate = params['max_lr'],
  step_size = 8 * params['buffer_size'] // params['batch_size'],)
optim = tf.optimizers.Adam(params['init_lr'])
global_step = 0

best_acc = .0
count = 0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)

while True:
  # TRAINING
  for ((text1, text2), labels) in dataset(is_training=True, params=params):
    with tf.GradientTape() as tape:
      logits = model((text1, text2), training=True)
      labels = tf.cast(labels, tf.float32)
      num_neg = tf.reduce_sum(tf.cast(tf.equal(labels, 0.), tf.float32)).numpy()
      num_pos = tf.reduce_sum(labels).numpy()
      if num_pos == 0.:
        pos_weight = 1.
      else:
        pos_weight = num_neg / num_pos
      loss = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(
        labels = label_smoothing(labels, params['label_smooth']),
        logits = logits,
        pos_weight = pos_weight))
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, model.trainable_variables))
    
    if global_step % 100 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
        global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  m = tf.keras.metrics.Accuracy()
  intent_true = []
  intent_pred = []

  for ((text1, text2), labels) in dataset(is_training=False, params=params):
    logits = tf.sigmoid(model((text1, text2), training=False))
    y_pred = tf.cast(tf.math.greater_equal(logits, .5), tf.int32)
    m.update_state(y_true=labels, y_pred=y_pred)
    intent_true += labels.numpy().flatten().tolist()
    intent_pred += y_pred.numpy().flatten().tolist()

  acc = m.result().numpy()
  logger.info("Evaluation: Testing Accuracy: {:.3f}".format(acc))

  logger.info('\n'+classification_report(y_true = intent_true,
                                         y_pred = intent_pred,
                                         labels = [0, 1],
                                         target_names = ['Not Matched', 'Matched'],
                                         digits = 3))

  if acc > best_acc:
    best_acc = acc
    # you can save model here
    count = 0
  else:
    count += 1
  logger.info("Best Accuracy: {:.3f}".format(best_acc))

  if count == params['num_patience']:
    print(params['num_patience'], "times not improve the best result, therefore stop training")
    break

[('encoder/forward_lstm/lstm_cell_1/kernel:0', TensorShape([300, 1200])),
 ('encoder/forward_lstm/lstm_cell_1/recurrent_kernel:0',
  TensorShape([300, 1200])),
 ('encoder/forward_lstm/lstm_cell_1/bias:0', TensorShape([1200])),
 ('encoder/backward_lstm/lstm_cell_2/kernel:0', TensorShape([300, 1200])),
 ('encoder/backward_lstm/lstm_cell_2/recurrent_kernel:0',
  TensorShape([300, 1200])),
 ('encoder/backward_lstm/lstm_cell_2/bias:0', TensorShape([1200])),
 ('conv2d/kernel:0', TensorShape([3, 3, 4, 32])),
 ('conv2d/bias:0', TensorShape([32])),
 ('conv2d_1/kernel:0', TensorShape([3, 3, 32, 64])),
 ('conv2d_1/bias:0', TensorShape([64])),
 ('conv2d_2/kernel:0', TensorShape([3, 3, 64, 128])),
 ('conv2d_2/bias:0', TensorShape([128])),
 ('dense/kernel:0', TensorShape([600, 600])),
 ('dense_1/kernel:0', TensorShape([600, 300])),
 ('dense_2/kernel:0', TensorShape([600, 300])),
 ('dense_3/kernel:0', TensorShape([300, 1])),
 ('dense_4/kernel:0', TensorShape([600, 300])),
 ('dense_5/kernel:0', TensorShape([300, 1])),
 ('dense_6/kernel:0', TensorShape([600, 300])),
 ('dense_7/kernel:0', TensorShape([300, 1])),
 ('out_hidden/dense_8/kernel:0', TensorShape([1152, 300])),
 ('out_hidden/dense_8/bias:0', TensorShape([300])),
 ('out_hidden/dense_9/kernel:0', TensorShape([300, 300])),
 ('out_hidden/dense_9/bias:0', TensorShape([300])),
 ('out_linear/kernel:0', TensorShape([300, 1])),
 ('out_linear/bias:0', TensorShape([1])),
 ('pretrained_embedding:0', TensorShape([1706, 300]))]
Reading ../data/train.json
INFO:tensorflow:Step 0 | Loss: 1.0832 | Spent: 7.8 secs | LR: 0.000100
INFO:tensorflow:Step 100 | Loss: 1.0336 | Spent: 8.3 secs | LR: 0.000108
INFO:tensorflow:Step 200 | Loss: 0.9958 | Spent: 8.6 secs | LR: 0.000116
INFO:tensorflow:Step 300 | Loss: 0.9924 | Spent: 8.8 secs | LR: 0.000124
INFO:tensorflow:Step 400 | Loss: 0.8326 | Spent: 8.7 secs | LR: 0.000133
INFO:tensorflow:Step 500 | Loss: 1.1895 | Spent: 8.4 secs | LR: 0.000141
INFO:tensorflow:Step 600 | Loss: 0.9340 | Spent: 8.0 secs | LR: 0.000149
INFO:tensorflow:Step 700 | Loss: 1.0353 | Spent: 8.8 secs | LR: 0.000157
INFO:tensorflow:Step 800 | Loss: 0.8973 | Spent: 8.1 secs | LR: 0.000165
INFO:tensorflow:Step 900 | Loss: 0.9578 | Spent: 9.3 secs | LR: 0.000173
INFO:tensorflow:Step 1000 | Loss: 1.0775 | Spent: 8.5 secs | LR: 0.000182
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.556
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.728     0.570     0.639      2978
     Matched      0.354     0.525     0.423      1338

    accuracy                          0.556      4316
   macro avg      0.541     0.548     0.531      4316
weighted avg      0.612     0.556     0.572      4316

INFO:tensorflow:Best Accuracy: 0.556
Reading ../data/train.json
INFO:tensorflow:Step 1100 | Loss: 0.9656 | Spent: 19.9 secs | LR: 0.000190
INFO:tensorflow:Step 1200 | Loss: 0.7497 | Spent: 8.6 secs | LR: 0.000198
INFO:tensorflow:Step 1300 | Loss: 0.9295 | Spent: 8.4 secs | LR: 0.000206
INFO:tensorflow:Step 1400 | Loss: 0.9532 | Spent: 8.0 secs | LR: 0.000214
INFO:tensorflow:Step 1500 | Loss: 1.0400 | Spent: 8.2 secs | LR: 0.000222
INFO:tensorflow:Step 1600 | Loss: 0.8319 | Spent: 8.4 secs | LR: 0.000230
INFO:tensorflow:Step 1700 | Loss: 0.9274 | Spent: 7.9 secs | LR: 0.000239
INFO:tensorflow:Step 1800 | Loss: 0.9140 | Spent: 8.3 secs | LR: 0.000247
INFO:tensorflow:Step 1900 | Loss: 0.8534 | Spent: 8.4 secs | LR: 0.000255
INFO:tensorflow:Step 2000 | Loss: 0.8092 | Spent: 8.7 secs | LR: 0.000263
INFO:tensorflow:Step 2100 | Loss: 0.6822 | Spent: 8.6 secs | LR: 0.000271
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.570
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.742     0.577     0.649      2978
     Matched      0.370     0.553     0.444      1338

    accuracy                          0.570      4316
   macro avg      0.556     0.565     0.546      4316
weighted avg      0.627     0.570     0.585      4316

INFO:tensorflow:Best Accuracy: 0.570
Reading ../data/train.json
INFO:tensorflow:Step 2200 | Loss: 0.9326 | Spent: 20.1 secs | LR: 0.000279
INFO:tensorflow:Step 2300 | Loss: 0.7212 | Spent: 8.4 secs | LR: 0.000288
INFO:tensorflow:Step 2400 | Loss: 1.0556 | Spent: 8.0 secs | LR: 0.000296
INFO:tensorflow:Step 2500 | Loss: 0.9588 | Spent: 8.8 secs | LR: 0.000304
INFO:tensorflow:Step 2600 | Loss: 0.8533 | Spent: 8.4 secs | LR: 0.000312
INFO:tensorflow:Step 2700 | Loss: 0.8289 | Spent: 8.3 secs | LR: 0.000320
INFO:tensorflow:Step 2800 | Loss: 1.0139 | Spent: 8.6 secs | LR: 0.000328
INFO:tensorflow:Step 2900 | Loss: 0.9944 | Spent: 8.4 secs | LR: 0.000337
INFO:tensorflow:Step 3000 | Loss: 0.8954 | Spent: 8.3 secs | LR: 0.000345
INFO:tensorflow:Step 3100 | Loss: 0.9397 | Spent: 8.7 secs | LR: 0.000353
INFO:tensorflow:Step 3200 | Loss: 0.8626 | Spent: 8.1 secs | LR: 0.000361
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.609
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.758     0.637     0.692      2978
     Matched      0.403     0.546     0.464      1338

    accuracy                          0.609      4316
   macro avg      0.581     0.592     0.578      4316
weighted avg      0.648     0.609     0.621      4316

INFO:tensorflow:Best Accuracy: 0.609
Reading ../data/train.json
INFO:tensorflow:Step 3300 | Loss: 0.8462 | Spent: 19.8 secs | LR: 0.000369
INFO:tensorflow:Step 3400 | Loss: 0.9340 | Spent: 8.4 secs | LR: 0.000377
INFO:tensorflow:Step 3500 | Loss: 0.9135 | Spent: 8.4 secs | LR: 0.000385
INFO:tensorflow:Step 3600 | Loss: 1.0111 | Spent: 8.8 secs | LR: 0.000394
INFO:tensorflow:Step 3700 | Loss: 1.1023 | Spent: 8.4 secs | LR: 0.000402
INFO:tensorflow:Step 3800 | Loss: 1.0976 | Spent: 8.4 secs | LR: 0.000410
INFO:tensorflow:Step 3900 | Loss: 0.9757 | Spent: 9.0 secs | LR: 0.000418
INFO:tensorflow:Step 4000 | Loss: 0.7782 | Spent: 7.9 secs | LR: 0.000426
INFO:tensorflow:Step 4100 | Loss: 0.9397 | Spent: 8.2 secs | LR: 0.000434
INFO:tensorflow:Step 4200 | Loss: 0.8436 | Spent: 8.3 secs | LR: 0.000443
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.577
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.816     0.499     0.619      2978
     Matched      0.402     0.750     0.523      1338

    accuracy                          0.577      4316
   macro avg      0.609     0.624     0.571      4316
weighted avg      0.688     0.577     0.590      4316

INFO:tensorflow:Best Accuracy: 0.609
Reading ../data/train.json
INFO:tensorflow:Step 4300 | Loss: 0.7556 | Spent: 19.1 secs | LR: 0.000451
INFO:tensorflow:Step 4400 | Loss: 1.0379 | Spent: 8.5 secs | LR: 0.000459
INFO:tensorflow:Step 4500 | Loss: 0.8032 | Spent: 8.7 secs | LR: 0.000467
INFO:tensorflow:Step 4600 | Loss: 0.9184 | Spent: 8.2 secs | LR: 0.000475
INFO:tensorflow:Step 4700 | Loss: 0.6998 | Spent: 8.9 secs | LR: 0.000483
INFO:tensorflow:Step 4800 | Loss: 1.0065 | Spent: 8.5 secs | LR: 0.000491
INFO:tensorflow:Step 4900 | Loss: 0.8669 | Spent: 8.3 secs | LR: 0.000500
INFO:tensorflow:Step 5000 | Loss: 0.9266 | Spent: 8.0 secs | LR: 0.000508
INFO:tensorflow:Step 5100 | Loss: 0.7927 | Spent: 8.2 secs | LR: 0.000516
INFO:tensorflow:Step 5200 | Loss: 0.9267 | Spent: 8.5 secs | LR: 0.000524
INFO:tensorflow:Step 5300 | Loss: 0.7126 | Spent: 7.8 secs | LR: 0.000532
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.628
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.791     0.627     0.699      2978
     Matched      0.432     0.632     0.513      1338

    accuracy                          0.628      4316
   macro avg      0.612     0.629     0.606      4316
weighted avg      0.680     0.628     0.642      4316

INFO:tensorflow:Best Accuracy: 0.628
Reading ../data/train.json
INFO:tensorflow:Step 5400 | Loss: 0.6531 | Spent: 19.6 secs | LR: 0.000540
INFO:tensorflow:Step 5500 | Loss: 0.7982 | Spent: 8.9 secs | LR: 0.000549
INFO:tensorflow:Step 5600 | Loss: 0.9255 | Spent: 8.5 secs | LR: 0.000557
INFO:tensorflow:Step 5700 | Loss: 0.9214 | Spent: 8.2 secs | LR: 0.000565
INFO:tensorflow:Step 5800 | Loss: 0.9063 | Spent: 8.4 secs | LR: 0.000573
INFO:tensorflow:Step 5900 | Loss: 0.7927 | Spent: 8.3 secs | LR: 0.000581
INFO:tensorflow:Step 6000 | Loss: 0.9731 | Spent: 8.6 secs | LR: 0.000589
INFO:tensorflow:Step 6100 | Loss: 0.8530 | Spent: 8.4 secs | LR: 0.000597
INFO:tensorflow:Step 6200 | Loss: 0.8098 | Spent: 8.0 secs | LR: 0.000606
INFO:tensorflow:Step 6300 | Loss: 1.0912 | Spent: 8.6 secs | LR: 0.000614
INFO:tensorflow:Step 6400 | Loss: 0.8783 | Spent: 8.0 secs | LR: 0.000622
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.610
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.823     0.554     0.662      2978
     Matched      0.425     0.735     0.539      1338

    accuracy                          0.610      4316
   macro avg      0.624     0.645     0.601      4316
weighted avg      0.700     0.610     0.624      4316

INFO:tensorflow:Best Accuracy: 0.628
Reading ../data/train.json
INFO:tensorflow:Step 6500 | Loss: 1.0149 | Spent: 19.7 secs | LR: 0.000630
INFO:tensorflow:Step 6600 | Loss: 1.1652 | Spent: 8.4 secs | LR: 0.000638
INFO:tensorflow:Step 6700 | Loss: 0.6417 | Spent: 8.2 secs | LR: 0.000646
INFO:tensorflow:Step 6800 | Loss: 0.8481 | Spent: 8.1 secs | LR: 0.000655
INFO:tensorflow:Step 6900 | Loss: 0.9726 | Spent: 8.4 secs | LR: 0.000663
INFO:tensorflow:Step 7000 | Loss: 0.9175 | Spent: 8.4 secs | LR: 0.000671
INFO:tensorflow:Step 7100 | Loss: 0.6940 | Spent: 8.2 secs | LR: 0.000679
INFO:tensorflow:Step 7200 | Loss: 0.5977 | Spent: 8.4 secs | LR: 0.000687
INFO:tensorflow:Step 7300 | Loss: 0.8650 | Spent: 7.9 secs | LR: 0.000695
INFO:tensorflow:Step 7400 | Loss: 0.7460 | Spent: 8.5 secs | LR: 0.000704
INFO:tensorflow:Step 7500 | Loss: 0.8806 | Spent: 8.7 secs | LR: 0.000712
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.631
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.808     0.611     0.696      2978
     Matched      0.439     0.677     0.532      1338

    accuracy                          0.631      4316
   macro avg      0.623     0.644     0.614      4316
weighted avg      0.694     0.631     0.645      4316

INFO:tensorflow:Best Accuracy: 0.631
Reading ../data/train.json
INFO:tensorflow:Step 7600 | Loss: 0.9146 | Spent: 19.2 secs | LR: 0.000720
INFO:tensorflow:Step 7700 | Loss: 0.5889 | Spent: 8.2 secs | LR: 0.000728
INFO:tensorflow:Step 7800 | Loss: 0.7357 | Spent: 8.3 secs | LR: 0.000736
INFO:tensorflow:Step 7900 | Loss: 0.7747 | Spent: 8.6 secs | LR: 0.000744
INFO:tensorflow:Step 8000 | Loss: 0.8474 | Spent: 8.3 secs | LR: 0.000752
INFO:tensorflow:Step 8100 | Loss: 0.8302 | Spent: 8.3 secs | LR: 0.000761
INFO:tensorflow:Step 8200 | Loss: 0.5304 | Spent: 8.4 secs | LR: 0.000769
INFO:tensorflow:Step 8300 | Loss: 0.6192 | Spent: 8.5 secs | LR: 0.000777
INFO:tensorflow:Step 8400 | Loss: 0.7444 | Spent: 8.6 secs | LR: 0.000785
INFO:tensorflow:Step 8500 | Loss: 0.8534 | Spent: 8.5 secs | LR: 0.000793
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.594
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.842     0.506     0.632      2978
     Matched      0.418     0.788     0.546      1338

    accuracy                          0.594      4316
   macro avg      0.630     0.647     0.589      4316
weighted avg      0.710     0.594     0.605      4316

INFO:tensorflow:Best Accuracy: 0.631
Reading ../data/train.json
INFO:tensorflow:Step 8600 | Loss: 0.7403 | Spent: 19.2 secs | LR: 0.000799
INFO:tensorflow:Step 8700 | Loss: 0.7903 | Spent: 8.2 secs | LR: 0.000790
INFO:tensorflow:Step 8800 | Loss: 0.6497 | Spent: 8.4 secs | LR: 0.000782
INFO:tensorflow:Step 8900 | Loss: 0.5577 | Spent: 8.8 secs | LR: 0.000774
INFO:tensorflow:Step 9000 | Loss: 0.6650 | Spent: 8.0 secs | LR: 0.000766
INFO:tensorflow:Step 9100 | Loss: 0.5475 | Spent: 8.0 secs | LR: 0.000758
INFO:tensorflow:Step 9200 | Loss: 0.9318 | Spent: 8.2 secs | LR: 0.000750
INFO:tensorflow:Step 9300 | Loss: 1.0261 | Spent: 8.1 secs | LR: 0.000742
INFO:tensorflow:Step 9400 | Loss: 0.7854 | Spent: 8.0 secs | LR: 0.000733
INFO:tensorflow:Step 9500 | Loss: 0.5408 | Spent: 8.5 secs | LR: 0.000725
INFO:tensorflow:Step 9600 | Loss: 0.6757 | Spent: 8.4 secs | LR: 0.000717
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.614
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.847     0.537     0.657      2978
     Matched      0.432     0.785     0.557      1338

    accuracy                          0.614      4316
   macro avg      0.640     0.661     0.607      4316
weighted avg      0.719     0.614     0.626      4316

INFO:tensorflow:Best Accuracy: 0.631
Reading ../data/train.json
INFO:tensorflow:Step 9700 | Loss: 0.5867 | Spent: 19.9 secs | LR: 0.000709
INFO:tensorflow:Step 9800 | Loss: 0.6197 | Spent: 8.9 secs | LR: 0.000701
INFO:tensorflow:Step 9900 | Loss: 0.6224 | Spent: 8.8 secs | LR: 0.000693
INFO:tensorflow:Step 10000 | Loss: 0.6327 | Spent: 8.4 secs | LR: 0.000684
INFO:tensorflow:Step 10100 | Loss: 0.5609 | Spent: 7.9 secs | LR: 0.000676
INFO:tensorflow:Step 10200 | Loss: 1.1130 | Spent: 8.1 secs | LR: 0.000668
INFO:tensorflow:Step 10300 | Loss: 0.5793 | Spent: 8.0 secs | LR: 0.000660
INFO:tensorflow:Step 10400 | Loss: 0.6612 | Spent: 8.4 secs | LR: 0.000652
INFO:tensorflow:Step 10500 | Loss: 0.7423 | Spent: 8.2 secs | LR: 0.000644
INFO:tensorflow:Step 10600 | Loss: 0.8118 | Spent: 8.4 secs | LR: 0.000636
INFO:tensorflow:Step 10700 | Loss: 0.7278 | Spent: 8.4 secs | LR: 0.000627
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.667
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.808     0.679     0.738      2978
     Matched      0.472     0.641     0.544      1338

    accuracy                          0.667      4316
   macro avg      0.640     0.660     0.641      4316
weighted avg      0.704     0.667     0.678      4316

INFO:tensorflow:Best Accuracy: 0.667
Reading ../data/train.json
INFO:tensorflow:Step 10800 | Loss: 0.3077 | Spent: 19.3 secs | LR: 0.000619
INFO:tensorflow:Step 10900 | Loss: 0.3153 | Spent: 8.7 secs | LR: 0.000611
INFO:tensorflow:Step 11000 | Loss: 0.8995 | Spent: 8.2 secs | LR: 0.000603
INFO:tensorflow:Step 11100 | Loss: 0.6889 | Spent: 8.4 secs | LR: 0.000595
INFO:tensorflow:Step 11200 | Loss: 0.5085 | Spent: 8.2 secs | LR: 0.000587
INFO:tensorflow:Step 11300 | Loss: 0.7073 | Spent: 8.7 secs | LR: 0.000578
INFO:tensorflow:Step 11400 | Loss: 0.6233 | Spent: 8.4 secs | LR: 0.000570
INFO:tensorflow:Step 11500 | Loss: 0.6317 | Spent: 7.8 secs | LR: 0.000562
INFO:tensorflow:Step 11600 | Loss: 0.4564 | Spent: 8.5 secs | LR: 0.000554
INFO:tensorflow:Step 11700 | Loss: 0.4397 | Spent: 8.3 secs | LR: 0.000546
INFO:tensorflow:Step 11800 | Loss: 0.4676 | Spent: 8.3 secs | LR: 0.000538
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.630
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.820     0.595     0.690      2978
     Matched      0.440     0.709     0.543      1338

    accuracy                          0.630      4316
   macro avg      0.630     0.652     0.616      4316
weighted avg      0.702     0.630     0.644      4316

INFO:tensorflow:Best Accuracy: 0.667
Reading ../data/train.json
INFO:tensorflow:Step 11900 | Loss: 0.6894 | Spent: 19.9 secs | LR: 0.000529
INFO:tensorflow:Step 12000 | Loss: 0.3186 | Spent: 8.2 secs | LR: 0.000521
INFO:tensorflow:Step 12100 | Loss: 0.4521 | Spent: 8.5 secs | LR: 0.000513
INFO:tensorflow:Step 12200 | Loss: 0.4647 | Spent: 8.3 secs | LR: 0.000505
INFO:tensorflow:Step 12300 | Loss: 0.3607 | Spent: 8.2 secs | LR: 0.000497
INFO:tensorflow:Step 12400 | Loss: 0.4778 | Spent: 8.3 secs | LR: 0.000489
INFO:tensorflow:Step 12500 | Loss: 0.7255 | Spent: 8.6 secs | LR: 0.000481
INFO:tensorflow:Step 12600 | Loss: 0.4868 | Spent: 8.3 secs | LR: 0.000472
INFO:tensorflow:Step 12700 | Loss: 0.4127 | Spent: 8.1 secs | LR: 0.000464
INFO:tensorflow:Step 12800 | Loss: 0.5549 | Spent: 8.0 secs | LR: 0.000456
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.668
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.793     0.702     0.745      2978
     Matched      0.471     0.592     0.525      1338

    accuracy                          0.668      4316
   macro avg      0.632     0.647     0.635      4316
weighted avg      0.693     0.668     0.676      4316

INFO:tensorflow:Best Accuracy: 0.668
Reading ../data/train.json
INFO:tensorflow:Step 12900 | Loss: 0.2463 | Spent: 20.5 secs | LR: 0.000448
INFO:tensorflow:Step 13000 | Loss: 0.3273 | Spent: 8.4 secs | LR: 0.000440
INFO:tensorflow:Step 13100 | Loss: 0.2550 | Spent: 8.7 secs | LR: 0.000432
INFO:tensorflow:Step 13200 | Loss: 0.6400 | Spent: 8.6 secs | LR: 0.000423
INFO:tensorflow:Step 13300 | Loss: 0.6464 | Spent: 8.3 secs | LR: 0.000415
INFO:tensorflow:Step 13400 | Loss: 0.8268 | Spent: 8.1 secs | LR: 0.000407
INFO:tensorflow:Step 13500 | Loss: 0.3807 | Spent: 8.4 secs | LR: 0.000399
INFO:tensorflow:Step 13600 | Loss: 0.3553 | Spent: 8.1 secs | LR: 0.000391
INFO:tensorflow:Step 13700 | Loss: 0.3250 | Spent: 8.5 secs | LR: 0.000383
INFO:tensorflow:Step 13800 | Loss: 0.3465 | Spent: 8.1 secs | LR: 0.000375
INFO:tensorflow:Step 13900 | Loss: 0.4293 | Spent: 8.4 secs | LR: 0.000366
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.633
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.818     0.601     0.693      2978
     Matched      0.442     0.703     0.542      1338

    accuracy                          0.633      4316
   macro avg      0.630     0.652     0.618      4316
weighted avg      0.701     0.633     0.646      4316

INFO:tensorflow:Best Accuracy: 0.668
Reading ../data/train.json
INFO:tensorflow:Step 14000 | Loss: 0.2235 | Spent: 19.7 secs | LR: 0.000358
INFO:tensorflow:Step 14100 | Loss: 0.8407 | Spent: 8.3 secs | LR: 0.000350
INFO:tensorflow:Step 14200 | Loss: 0.2380 | Spent: 7.9 secs | LR: 0.000342
INFO:tensorflow:Step 14300 | Loss: 0.4128 | Spent: 8.5 secs | LR: 0.000334
INFO:tensorflow:Step 14400 | Loss: 0.1931 | Spent: 8.5 secs | LR: 0.000326
INFO:tensorflow:Step 14500 | Loss: 0.2112 | Spent: 8.4 secs | LR: 0.000317
INFO:tensorflow:Step 14600 | Loss: 0.3489 | Spent: 8.2 secs | LR: 0.000309
INFO:tensorflow:Step 14700 | Loss: 0.4211 | Spent: 7.9 secs | LR: 0.000301
INFO:tensorflow:Step 14800 | Loss: 0.2862 | Spent: 8.3 secs | LR: 0.000293
INFO:tensorflow:Step 14900 | Loss: 0.1206 | Spent: 8.5 secs | LR: 0.000285
INFO:tensorflow:Step 15000 | Loss: 0.2267 | Spent: 8.6 secs | LR: 0.000277
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.673
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.792     0.713     0.750      2978
     Matched      0.477     0.584     0.525      1338

    accuracy                          0.673      4316
   macro avg      0.635     0.648     0.638      4316
weighted avg      0.694     0.673     0.680      4316

INFO:tensorflow:Best Accuracy: 0.673
Reading ../data/train.json
INFO:tensorflow:Step 15100 | Loss: 0.3518 | Spent: 19.2 secs | LR: 0.000268
INFO:tensorflow:Step 15200 | Loss: 0.1307 | Spent: 8.3 secs | LR: 0.000260
INFO:tensorflow:Step 15300 | Loss: 0.2047 | Spent: 8.4 secs | LR: 0.000252
INFO:tensorflow:Step 15400 | Loss: 0.6183 | Spent: 8.4 secs | LR: 0.000244
INFO:tensorflow:Step 15500 | Loss: 0.2762 | Spent: 8.3 secs | LR: 0.000236
INFO:tensorflow:Step 15600 | Loss: 0.1464 | Spent: 8.2 secs | LR: 0.000228
INFO:tensorflow:Step 15700 | Loss: 0.3200 | Spent: 8.5 secs | LR: 0.000220
INFO:tensorflow:Step 15800 | Loss: 0.3495 | Spent: 8.6 secs | LR: 0.000211
INFO:tensorflow:Step 15900 | Loss: 0.1438 | Spent: 8.2 secs | LR: 0.000203
INFO:tensorflow:Step 16000 | Loss: 0.1338 | Spent: 8.5 secs | LR: 0.000195
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.657
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.795     0.677     0.732      2978
     Matched      0.460     0.612     0.525      1338

    accuracy                          0.657      4316
   macro avg      0.628     0.645     0.628      4316
weighted avg      0.691     0.657     0.668      4316

INFO:tensorflow:Best Accuracy: 0.673
Reading ../data/train.json
INFO:tensorflow:Step 16100 | Loss: 0.1030 | Spent: 19.8 secs | LR: 0.000187
INFO:tensorflow:Step 16200 | Loss: 0.3267 | Spent: 8.3 secs | LR: 0.000179
INFO:tensorflow:Step 16300 | Loss: 0.0959 | Spent: 8.4 secs | LR: 0.000171
INFO:tensorflow:Step 16400 | Loss: 0.1758 | Spent: 8.2 secs | LR: 0.000162
INFO:tensorflow:Step 16500 | Loss: 0.1716 | Spent: 8.0 secs | LR: 0.000154
INFO:tensorflow:Step 16600 | Loss: 0.1066 | Spent: 8.2 secs | LR: 0.000146
INFO:tensorflow:Step 16700 | Loss: 0.2235 | Spent: 8.5 secs | LR: 0.000138
INFO:tensorflow:Step 16800 | Loss: 0.5618 | Spent: 8.5 secs | LR: 0.000130
INFO:tensorflow:Step 16900 | Loss: 0.2350 | Spent: 8.2 secs | LR: 0.000122
INFO:tensorflow:Step 17000 | Loss: 0.3031 | Spent: 8.5 secs | LR: 0.000114
INFO:tensorflow:Step 17100 | Loss: 0.1328 | Spent: 8.5 secs | LR: 0.000105
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.681
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.775     0.759     0.767      2978
     Matched      0.486     0.509     0.497      1338

    accuracy                          0.681      4316
   macro avg      0.631     0.634     0.632      4316
weighted avg      0.685     0.681     0.683      4316

INFO:tensorflow:Best Accuracy: 0.681
Reading ../data/train.json
INFO:tensorflow:Step 17200 | Loss: 0.1316 | Spent: 19.5 secs | LR: 0.000101
INFO:tensorflow:Step 17300 | Loss: 0.1195 | Spent: 8.2 secs | LR: 0.000105
INFO:tensorflow:Step 17400 | Loss: 0.1261 | Spent: 8.9 secs | LR: 0.000110
INFO:tensorflow:Step 17500 | Loss: 0.2384 | Spent: 8.3 secs | LR: 0.000114
INFO:tensorflow:Step 17600 | Loss: 0.1986 | Spent: 8.5 secs | LR: 0.000118
INFO:tensorflow:Step 17700 | Loss: 0.1873 | Spent: 8.3 secs | LR: 0.000122
INFO:tensorflow:Step 17800 | Loss: 0.2485 | Spent: 8.1 secs | LR: 0.000126
INFO:tensorflow:Step 17900 | Loss: 0.1514 | Spent: 8.5 secs | LR: 0.000130
INFO:tensorflow:Step 18000 | Loss: 0.3570 | Spent: 8.6 secs | LR: 0.000134
INFO:tensorflow:Step 18100 | Loss: 0.0593 | Spent: 9.0 secs | LR: 0.000138
INFO:tensorflow:Step 18200 | Loss: 0.0569 | Spent: 8.1 secs | LR: 0.000142
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.677
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.767     0.765     0.766      2978
     Matched      0.480     0.484     0.482      1338

    accuracy                          0.677      4316
   macro avg      0.624     0.624     0.624      4316
weighted avg      0.678     0.677     0.678      4316

INFO:tensorflow:Best Accuracy: 0.681
Reading ../data/train.json
INFO:tensorflow:Step 18300 | Loss: 0.2400 | Spent: 19.5 secs | LR: 0.000146
INFO:tensorflow:Step 18400 | Loss: 0.0928 | Spent: 8.5 secs | LR: 0.000150
INFO:tensorflow:Step 18500 | Loss: 0.2331 | Spent: 8.5 secs | LR: 0.000154
INFO:tensorflow:Step 18600 | Loss: 0.3606 | Spent: 8.0 secs | LR: 0.000158
INFO:tensorflow:Step 18700 | Loss: 0.4751 | Spent: 8.1 secs | LR: 0.000163
INFO:tensorflow:Step 18800 | Loss: 0.2135 | Spent: 8.2 secs | LR: 0.000167
INFO:tensorflow:Step 18900 | Loss: 0.0315 | Spent: 8.2 secs | LR: 0.000171
INFO:tensorflow:Step 19000 | Loss: 0.4237 | Spent: 8.4 secs | LR: 0.000175
INFO:tensorflow:Step 19100 | Loss: 0.2947 | Spent: 8.3 secs | LR: 0.000179
INFO:tensorflow:Step 19200 | Loss: 0.2529 | Spent: 8.6 secs | LR: 0.000183
INFO:tensorflow:Step 19300 | Loss: 0.5445 | Spent: 8.6 secs | LR: 0.000187
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.690
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.755     0.817     0.785      2978
     Matched      0.501     0.410     0.451      1338

    accuracy                          0.690      4316
   macro avg      0.628     0.613     0.618      4316
weighted avg      0.676     0.690     0.681      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 19400 | Loss: 0.2983 | Spent: 19.9 secs | LR: 0.000191
INFO:tensorflow:Step 19500 | Loss: 0.1337 | Spent: 8.6 secs | LR: 0.000195
INFO:tensorflow:Step 19600 | Loss: 0.2713 | Spent: 8.6 secs | LR: 0.000199
INFO:tensorflow:Step 19700 | Loss: 0.0977 | Spent: 8.2 secs | LR: 0.000203
INFO:tensorflow:Step 19800 | Loss: 0.3152 | Spent: 8.4 secs | LR: 0.000207
INFO:tensorflow:Step 19900 | Loss: 0.1565 | Spent: 8.0 secs | LR: 0.000211
INFO:tensorflow:Step 20000 | Loss: 0.3338 | Spent: 8.5 secs | LR: 0.000216
INFO:tensorflow:Step 20100 | Loss: 0.2138 | Spent: 8.1 secs | LR: 0.000220
INFO:tensorflow:Step 20200 | Loss: 0.2388 | Spent: 8.4 secs | LR: 0.000224
INFO:tensorflow:Step 20300 | Loss: 0.3281 | Spent: 8.4 secs | LR: 0.000228
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.688
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.762     0.797     0.779      2978
     Matched      0.497     0.446     0.470      1338

    accuracy                          0.688      4316
   macro avg      0.629     0.622     0.625      4316
weighted avg      0.680     0.688     0.683      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 20400 | Loss: 0.2164 | Spent: 19.6 secs | LR: 0.000232
INFO:tensorflow:Step 20500 | Loss: 0.1577 | Spent: 8.6 secs | LR: 0.000236
INFO:tensorflow:Step 20600 | Loss: 0.0613 | Spent: 8.3 secs | LR: 0.000240
INFO:tensorflow:Step 20700 | Loss: 0.0419 | Spent: 8.4 secs | LR: 0.000244
INFO:tensorflow:Step 20800 | Loss: 0.2477 | Spent: 8.6 secs | LR: 0.000248
INFO:tensorflow:Step 20900 | Loss: 0.1313 | Spent: 8.4 secs | LR: 0.000252
INFO:tensorflow:Step 21000 | Loss: 0.0459 | Spent: 8.6 secs | LR: 0.000256
INFO:tensorflow:Step 21100 | Loss: 0.0950 | Spent: 8.2 secs | LR: 0.000260
INFO:tensorflow:Step 21200 | Loss: 0.1463 | Spent: 8.1 secs | LR: 0.000264
INFO:tensorflow:Step 21300 | Loss: 0.1301 | Spent: 8.3 secs | LR: 0.000269
INFO:tensorflow:Step 21400 | Loss: 0.0688 | Spent: 7.8 secs | LR: 0.000273
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.667
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.774     0.731     0.752      2978
     Matched      0.467     0.524     0.494      1338

    accuracy                          0.667      4316
   macro avg      0.620     0.627     0.623      4316
weighted avg      0.678     0.667     0.672      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 21500 | Loss: 0.1253 | Spent: 19.3 secs | LR: 0.000277
INFO:tensorflow:Step 21600 | Loss: 0.1218 | Spent: 8.6 secs | LR: 0.000281
INFO:tensorflow:Step 21700 | Loss: 0.0900 | Spent: 8.4 secs | LR: 0.000285
INFO:tensorflow:Step 21800 | Loss: 0.5571 | Spent: 7.9 secs | LR: 0.000289
INFO:tensorflow:Step 21900 | Loss: 0.1398 | Spent: 8.2 secs | LR: 0.000293
INFO:tensorflow:Step 22000 | Loss: 0.2282 | Spent: 8.6 secs | LR: 0.000297
INFO:tensorflow:Step 22100 | Loss: 0.1692 | Spent: 8.3 secs | LR: 0.000301
INFO:tensorflow:Step 22200 | Loss: 0.1020 | Spent: 8.4 secs | LR: 0.000305
INFO:tensorflow:Step 22300 | Loss: 0.1144 | Spent: 8.6 secs | LR: 0.000309
INFO:tensorflow:Step 22400 | Loss: 0.1778 | Spent: 8.2 secs | LR: 0.000313
INFO:tensorflow:Step 22500 | Loss: 0.4825 | Spent: 8.4 secs | LR: 0.000318
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.671
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.776     0.736     0.756      2978
     Matched      0.473     0.528     0.499      1338

    accuracy                          0.671      4316
   macro avg      0.625     0.632     0.627      4316
weighted avg      0.682     0.671     0.676      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 22600 | Loss: 0.1135 | Spent: 19.8 secs | LR: 0.000322
INFO:tensorflow:Step 22700 | Loss: 0.3096 | Spent: 8.8 secs | LR: 0.000326
INFO:tensorflow:Step 22800 | Loss: 0.0194 | Spent: 9.2 secs | LR: 0.000330
INFO:tensorflow:Step 22900 | Loss: 0.1969 | Spent: 8.2 secs | LR: 0.000334
INFO:tensorflow:Step 23000 | Loss: 0.2019 | Spent: 8.5 secs | LR: 0.000338
INFO:tensorflow:Step 23100 | Loss: 0.0672 | Spent: 8.5 secs | LR: 0.000342
INFO:tensorflow:Step 23200 | Loss: 0.1682 | Spent: 7.9 secs | LR: 0.000346
INFO:tensorflow:Step 23300 | Loss: 0.2816 | Spent: 8.1 secs | LR: 0.000350
INFO:tensorflow:Step 23400 | Loss: 0.3975 | Spent: 8.5 secs | LR: 0.000354
INFO:tensorflow:Step 23500 | Loss: 0.3914 | Spent: 8.5 secs | LR: 0.000358
INFO:tensorflow:Step 23600 | Loss: 0.0434 | Spent: 8.0 secs | LR: 0.000362
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.683
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.759     0.791     0.775      2978
     Matched      0.488     0.442     0.464      1338

    accuracy                          0.683      4316
   macro avg      0.623     0.617     0.619      4316
weighted avg      0.675     0.683     0.678      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 23700 | Loss: 0.1372 | Spent: 19.6 secs | LR: 0.000366
INFO:tensorflow:Step 23800 | Loss: 0.2357 | Spent: 8.8 secs | LR: 0.000371
INFO:tensorflow:Step 23900 | Loss: 0.2066 | Spent: 8.1 secs | LR: 0.000375
INFO:tensorflow:Step 24000 | Loss: 0.3144 | Spent: 8.4 secs | LR: 0.000379
INFO:tensorflow:Step 24100 | Loss: 0.4136 | Spent: 8.7 secs | LR: 0.000383
INFO:tensorflow:Step 24200 | Loss: 0.1270 | Spent: 7.9 secs | LR: 0.000387
INFO:tensorflow:Step 24300 | Loss: 0.0345 | Spent: 8.8 secs | LR: 0.000391
INFO:tensorflow:Step 24400 | Loss: 0.2324 | Spent: 8.3 secs | LR: 0.000395
INFO:tensorflow:Step 24500 | Loss: 0.3013 | Spent: 8.3 secs | LR: 0.000399
INFO:tensorflow:Step 24600 | Loss: 0.0927 | Spent: 8.0 secs | LR: 0.000403
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.663
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.777     0.717     0.746      2978
     Matched      0.463     0.543     0.500      1338

    accuracy                          0.663      4316
   macro avg      0.620     0.630     0.623      4316
weighted avg      0.680     0.663     0.670      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 24700 | Loss: 0.0122 | Spent: 19.6 secs | LR: 0.000407
INFO:tensorflow:Step 24800 | Loss: 0.1327 | Spent: 8.0 secs | LR: 0.000411
INFO:tensorflow:Step 24900 | Loss: 0.1920 | Spent: 8.2 secs | LR: 0.000415
INFO:tensorflow:Step 25000 | Loss: 0.0139 | Spent: 8.3 secs | LR: 0.000419
INFO:tensorflow:Step 25100 | Loss: 0.2421 | Spent: 8.7 secs | LR: 0.000424
INFO:tensorflow:Step 25200 | Loss: 0.1468 | Spent: 8.4 secs | LR: 0.000428
INFO:tensorflow:Step 25300 | Loss: 0.3953 | Spent: 8.2 secs | LR: 0.000432
INFO:tensorflow:Step 25400 | Loss: 1.3979 | Spent: 8.3 secs | LR: 0.000436
INFO:tensorflow:Step 25500 | Loss: 0.0307 | Spent: 7.9 secs | LR: 0.000440
INFO:tensorflow:Step 25600 | Loss: 0.1433 | Spent: 8.8 secs | LR: 0.000444
INFO:tensorflow:Step 25700 | Loss: 0.0241 | Spent: 8.6 secs | LR: 0.000448
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.659
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.794     0.683     0.734      2978
     Matched      0.462     0.606     0.524      1338

    accuracy                          0.659      4316
   macro avg      0.628     0.645     0.629      4316
weighted avg      0.691     0.659     0.669      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 25800 | Loss: 0.2410 | Spent: 19.1 secs | LR: 0.000448
INFO:tensorflow:Step 25900 | Loss: 0.0350 | Spent: 8.6 secs | LR: 0.000444
INFO:tensorflow:Step 26000 | Loss: 0.2825 | Spent: 8.4 secs | LR: 0.000440
INFO:tensorflow:Step 26100 | Loss: 0.1653 | Spent: 8.6 secs | LR: 0.000436
INFO:tensorflow:Step 26200 | Loss: 0.3148 | Spent: 8.2 secs | LR: 0.000432
INFO:tensorflow:Step 26300 | Loss: 0.2293 | Spent: 8.4 secs | LR: 0.000428
INFO:tensorflow:Step 26400 | Loss: 0.3351 | Spent: 8.8 secs | LR: 0.000423
INFO:tensorflow:Step 26500 | Loss: 0.0715 | Spent: 8.6 secs | LR: 0.000419
INFO:tensorflow:Step 26600 | Loss: 0.0774 | Spent: 8.2 secs | LR: 0.000415
INFO:tensorflow:Step 26700 | Loss: 0.0741 | Spent: 8.2 secs | LR: 0.000411
INFO:tensorflow:Step 26800 | Loss: 0.0972 | Spent: 8.2 secs | LR: 0.000407
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.684
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.751     0.811     0.780      2978
     Matched      0.489     0.402     0.441      1338

    accuracy                          0.684      4316
   macro avg      0.620     0.607     0.611      4316
weighted avg      0.670     0.684     0.675      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 26900 | Loss: 0.0482 | Spent: 19.7 secs | LR: 0.000403
INFO:tensorflow:Step 27000 | Loss: 0.0100 | Spent: 8.3 secs | LR: 0.000399
INFO:tensorflow:Step 27100 | Loss: 0.0802 | Spent: 8.2 secs | LR: 0.000395
INFO:tensorflow:Step 27200 | Loss: 0.5003 | Spent: 8.1 secs | LR: 0.000391
INFO:tensorflow:Step 27300 | Loss: 0.0679 | Spent: 8.6 secs | LR: 0.000387
INFO:tensorflow:Step 27400 | Loss: 0.5175 | Spent: 8.4 secs | LR: 0.000383
INFO:tensorflow:Step 27500 | Loss: 0.1248 | Spent: 8.1 secs | LR: 0.000379
INFO:tensorflow:Step 27600 | Loss: 0.0969 | Spent: 8.6 secs | LR: 0.000375
INFO:tensorflow:Step 27700 | Loss: 0.2787 | Spent: 8.4 secs | LR: 0.000370
INFO:tensorflow:Step 27800 | Loss: 0.0696 | Spent: 8.4 secs | LR: 0.000366
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.684
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.769     0.775     0.772      2978
     Matched      0.490     0.481     0.486      1338

    accuracy                          0.684      4316
   macro avg      0.629     0.628     0.629      4316
weighted avg      0.682     0.684     0.683      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 27900 | Loss: 0.2051 | Spent: 19.5 secs | LR: 0.000362
INFO:tensorflow:Step 28000 | Loss: 0.1587 | Spent: 8.6 secs | LR: 0.000358
INFO:tensorflow:Step 28100 | Loss: 0.1260 | Spent: 8.5 secs | LR: 0.000354
INFO:tensorflow:Step 28200 | Loss: 0.0010 | Spent: 8.2 secs | LR: 0.000350
INFO:tensorflow:Step 28300 | Loss: 0.0082 | Spent: 8.4 secs | LR: 0.000346
INFO:tensorflow:Step 28400 | Loss: 0.0050 | Spent: 8.3 secs | LR: 0.000342
INFO:tensorflow:Step 28500 | Loss: 0.0865 | Spent: 8.2 secs | LR: 0.000338
INFO:tensorflow:Step 28600 | Loss: 0.5205 | Spent: 8.5 secs | LR: 0.000334
INFO:tensorflow:Step 28700 | Loss: 0.0752 | Spent: 8.2 secs | LR: 0.000330
INFO:tensorflow:Step 28800 | Loss: 0.2397 | Spent: 8.0 secs | LR: 0.000326
INFO:tensorflow:Step 28900 | Loss: 0.3835 | Spent: 8.4 secs | LR: 0.000322
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.673
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.772     0.747     0.759      2978
     Matched      0.475     0.509     0.491      1338

    accuracy                          0.673      4316
   macro avg      0.623     0.628     0.625      4316
weighted avg      0.680     0.673     0.676      4316

INFO:tensorflow:Best Accuracy: 0.690
Reading ../data/train.json
INFO:tensorflow:Step 29000 | Loss: 0.1917 | Spent: 19.4 secs | LR: 0.000317
INFO:tensorflow:Step 29100 | Loss: 0.2095 | Spent: 8.6 secs | LR: 0.000313
INFO:tensorflow:Step 29200 | Loss: 0.0308 | Spent: 8.3 secs | LR: 0.000309
INFO:tensorflow:Step 29300 | Loss: 0.0063 | Spent: 8.4 secs | LR: 0.000305
INFO:tensorflow:Step 29400 | Loss: 0.0110 | Spent: 8.5 secs | LR: 0.000301
INFO:tensorflow:Step 29500 | Loss: 0.1523 | Spent: 8.4 secs | LR: 0.000297
INFO:tensorflow:Step 29600 | Loss: 0.0163 | Spent: 8.2 secs | LR: 0.000293
INFO:tensorflow:Step 29700 | Loss: 0.0010 | Spent: 8.5 secs | LR: 0.000289
INFO:tensorflow:Step 29800 | Loss: 0.1605 | Spent: 8.5 secs | LR: 0.000285
INFO:tensorflow:Step 29900 | Loss: 0.0018 | Spent: 8.3 secs | LR: 0.000281
INFO:tensorflow:Step 30000 | Loss: 0.0019 | Spent: 8.3 secs | LR: 0.000277
Reading ../data/dev.json
INFO:tensorflow:Evaluation: Testing Accuracy: 0.683
INFO:tensorflow:
              precision    recall  f1-score   support

 Not Matched      0.763     0.783     0.773      2978
     Matched      0.488     0.460     0.473      1338

    accuracy                          0.683      4316
   macro avg      0.626     0.622     0.623      4316
weighted avg      0.678     0.683     0.680      4316

INFO:tensorflow:Best Accuracy: 0.690
10 times not improve the best result, therefore stop training