from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/text_matching/chinese/main')

%tensorflow_version 1.x
!pip install bert4keras

TensorFlow 1.x selected.
Collecting bert4keras
  Downloading https://files.pythonhosted.org/packages/36/37/dfb321f3b50f97512b506fe00ba782bfdaded2cbe97241245a0f4afa54b0/bert4keras-0.7.7.tar.gz
Requirement already satisfied: keras in /usr/local/lib/python3.6/dist-packages (from bert4keras) (2.3.1)
Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (1.18.4)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (2.10.0)
Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (3.13)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (1.0.8)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (1.1.0)
Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (1.4.1)
Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras->bert4keras) (1.12.0)
Building wheels for collected packages: bert4keras
  Building wheel for bert4keras (setup.py) ... done
  Created wheel for bert4keras: filename=bert4keras-0.7.7-cp36-none-any.whl size=36804 sha256=b90c43df7f212f29d7049f8435ee19ffd93fbace107502f2939835980c98b6b4
  Stored in directory: /root/.cache/pip/wheels/73/5f/2b/ee7932b73172503fdb429421c0b433f47a737c0d1c2a85fdd2
Successfully built bert4keras
Installing collected packages: bert4keras
Successfully installed bert4keras-0.7.7

import numpy as np
import csv

from bert4keras.backend import keras, set_gelu, K
from bert4keras.tokenizers import Tokenizer
from bert4keras.models import build_transformer_model
from bert4keras.optimizers import Adam
from bert4keras.snippets import sequence_padding, DataGenerator
from bert4keras.snippets import open
from keras.layers import Dropout, Dense

Using TensorFlow backend.

def load_data(filename):
    D = []
    with open(filename, encoding='utf-8') as f:
      for i, line in enumerate(csv.reader(f, delimiter=',')):
        if i == 0:
          continue
        text1, text2, label = line
        D.append((text1, text2, int(label)))
    return D


class data_generator(DataGenerator):
    def __iter__(self, random=False):
        batch_token_ids, batch_segment_ids, batch_labels = [], [], []
        for is_end, (text1, text2, label) in self.sample(random):
            token_ids, segment_ids = tokenizer.encode(
                text1, text2, max_length=maxlen
            )
            batch_token_ids.append(token_ids)
            batch_segment_ids.append(segment_ids)
            batch_labels.append([label])
            if len(batch_token_ids) == self.batch_size or is_end:
                batch_token_ids = sequence_padding(batch_token_ids)
                batch_segment_ids = sequence_padding(batch_segment_ids)
                batch_labels = sequence_padding(batch_labels)
                yield [batch_token_ids, batch_segment_ids], batch_labels
                batch_token_ids, batch_segment_ids, batch_labels = [], [], []


def evaluate(data):
    total, right = 0., 0.
    for x_true, y_true in data:
        y_pred = model.predict(x_true).argmax(axis=1)
        y_true = y_true[:, 0]
        total += len(y_true)
        right += (y_true == y_pred).sum()
    return right / total


class Evaluator(keras.callbacks.Callback):
    def __init__(self):
        self.best_val_acc = 0.

    def on_epoch_end(self, epoch, logs=None):
        val_acc = evaluate(test_generator)
        if val_acc > self.best_val_acc:
            self.best_val_acc = val_acc
            # you can save your model here
        print(
            'val_acc: {:.4f}, best_val_acc: {:.4f}'.format
            (val_acc, self.best_val_acc)
        )

set_gelu('tanh')
maxlen = 64
batch_size = 32
config_path = '../model/chinese_wwm_L-12_H-768_A-12/bert_config.json'
checkpoint_path = '../model/chinese_wwm_L-12_H-768_A-12/bert_model.ckpt'
dict_path = '../model/chinese_wwm_L-12_H-768_A-12/vocab.txt'

train_data = load_data('../data/train.csv')
test_data = load_data('../data/test.csv')

tokenizer = Tokenizer(dict_path, do_lower_case=True)

train_generator = data_generator(train_data, batch_size)
test_generator = data_generator(test_data, batch_size)

bert = build_transformer_model(
    config_path=config_path,
    checkpoint_path=checkpoint_path,
    with_pool=True,
    return_keras_model=False,
)

output = Dropout(rate=.1)(bert.model.output)
output = Dense(
    units=2, activation='softmax', kernel_initializer=bert.initializer
)(output)

model = keras.models.Model(bert.model.input, output)
model.summary()

model.compile(
    loss='sparse_categorical_crossentropy',
    optimizer=Adam(2e-5),
    metrics=['accuracy'],
)

evaluator = Evaluator()
model.fit_generator(
    train_generator.forfit(),
    steps_per_epoch=len(train_generator),
    epochs=20,
    callbacks=[evaluator]
)

WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
Model: "model_2"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
Input-Token (InputLayer)        (None, None)         0                                            
__________________________________________________________________________________________________
Input-Segment (InputLayer)      (None, None)         0                                            
__________________________________________________________________________________________________
Embedding-Token (Embedding)     (None, None, 768)    16226304    Input-Token[0][0]                
__________________________________________________________________________________________________
Embedding-Segment (Embedding)   (None, None, 768)    1536        Input-Segment[0][0]              
__________________________________________________________________________________________________
Embedding-Token-Segment (Add)   (None, None, 768)    0           Embedding-Token[0][0]            
                                                                 Embedding-Segment[0][0]          
__________________________________________________________________________________________________
Embedding-Position (PositionEmb (None, None, 768)    393216      Embedding-Token-Segment[0][0]    
__________________________________________________________________________________________________
Embedding-Norm (LayerNormalizat (None, None, 768)    1536        Embedding-Position[0][0]         
__________________________________________________________________________________________________
Embedding-Dropout (Dropout)     (None, None, 768)    0           Embedding-Norm[0][0]             
__________________________________________________________________________________________________
Transformer-0-MultiHeadSelfAtte (None, None, 768)    2362368     Embedding-Dropout[0][0]          
                                                                 Embedding-Dropout[0][0]          
                                                                 Embedding-Dropout[0][0]          
__________________________________________________________________________________________________
Transformer-0-MultiHeadSelfAtte (None, None, 768)    0           Transformer-0-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-0-MultiHeadSelfAtte (None, None, 768)    0           Embedding-Dropout[0][0]          
                                                                 Transformer-0-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-0-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-0-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-0-FeedForward (Feed (None, None, 768)    4722432     Transformer-0-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-0-FeedForward-Dropo (None, None, 768)    0           Transformer-0-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-0-FeedForward-Add ( (None, None, 768)    0           Transformer-0-MultiHeadSelfAttent
                                                                 Transformer-0-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-0-FeedForward-Norm  (None, None, 768)    1536        Transformer-0-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-1-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-0-FeedForward-Norm[0]
                                                                 Transformer-0-FeedForward-Norm[0]
                                                                 Transformer-0-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-1-MultiHeadSelfAtte (None, None, 768)    0           Transformer-1-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-1-MultiHeadSelfAtte (None, None, 768)    0           Transformer-0-FeedForward-Norm[0]
                                                                 Transformer-1-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-1-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-1-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-1-FeedForward (Feed (None, None, 768)    4722432     Transformer-1-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-1-FeedForward-Dropo (None, None, 768)    0           Transformer-1-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-1-FeedForward-Add ( (None, None, 768)    0           Transformer-1-MultiHeadSelfAttent
                                                                 Transformer-1-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-1-FeedForward-Norm  (None, None, 768)    1536        Transformer-1-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-2-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-1-FeedForward-Norm[0]
                                                                 Transformer-1-FeedForward-Norm[0]
                                                                 Transformer-1-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-2-MultiHeadSelfAtte (None, None, 768)    0           Transformer-2-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-2-MultiHeadSelfAtte (None, None, 768)    0           Transformer-1-FeedForward-Norm[0]
                                                                 Transformer-2-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-2-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-2-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-2-FeedForward (Feed (None, None, 768)    4722432     Transformer-2-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-2-FeedForward-Dropo (None, None, 768)    0           Transformer-2-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-2-FeedForward-Add ( (None, None, 768)    0           Transformer-2-MultiHeadSelfAttent
                                                                 Transformer-2-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-2-FeedForward-Norm  (None, None, 768)    1536        Transformer-2-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-3-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-2-FeedForward-Norm[0]
                                                                 Transformer-2-FeedForward-Norm[0]
                                                                 Transformer-2-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-3-MultiHeadSelfAtte (None, None, 768)    0           Transformer-3-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-3-MultiHeadSelfAtte (None, None, 768)    0           Transformer-2-FeedForward-Norm[0]
                                                                 Transformer-3-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-3-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-3-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-3-FeedForward (Feed (None, None, 768)    4722432     Transformer-3-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-3-FeedForward-Dropo (None, None, 768)    0           Transformer-3-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-3-FeedForward-Add ( (None, None, 768)    0           Transformer-3-MultiHeadSelfAttent
                                                                 Transformer-3-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-3-FeedForward-Norm  (None, None, 768)    1536        Transformer-3-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-4-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-3-FeedForward-Norm[0]
                                                                 Transformer-3-FeedForward-Norm[0]
                                                                 Transformer-3-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-4-MultiHeadSelfAtte (None, None, 768)    0           Transformer-4-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-4-MultiHeadSelfAtte (None, None, 768)    0           Transformer-3-FeedForward-Norm[0]
                                                                 Transformer-4-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-4-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-4-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-4-FeedForward (Feed (None, None, 768)    4722432     Transformer-4-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-4-FeedForward-Dropo (None, None, 768)    0           Transformer-4-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-4-FeedForward-Add ( (None, None, 768)    0           Transformer-4-MultiHeadSelfAttent
                                                                 Transformer-4-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-4-FeedForward-Norm  (None, None, 768)    1536        Transformer-4-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-5-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-4-FeedForward-Norm[0]
                                                                 Transformer-4-FeedForward-Norm[0]
                                                                 Transformer-4-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-5-MultiHeadSelfAtte (None, None, 768)    0           Transformer-5-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-5-MultiHeadSelfAtte (None, None, 768)    0           Transformer-4-FeedForward-Norm[0]
                                                                 Transformer-5-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-5-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-5-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-5-FeedForward (Feed (None, None, 768)    4722432     Transformer-5-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-5-FeedForward-Dropo (None, None, 768)    0           Transformer-5-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-5-FeedForward-Add ( (None, None, 768)    0           Transformer-5-MultiHeadSelfAttent
                                                                 Transformer-5-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-5-FeedForward-Norm  (None, None, 768)    1536        Transformer-5-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-6-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-5-FeedForward-Norm[0]
                                                                 Transformer-5-FeedForward-Norm[0]
                                                                 Transformer-5-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-6-MultiHeadSelfAtte (None, None, 768)    0           Transformer-6-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-6-MultiHeadSelfAtte (None, None, 768)    0           Transformer-5-FeedForward-Norm[0]
                                                                 Transformer-6-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-6-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-6-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-6-FeedForward (Feed (None, None, 768)    4722432     Transformer-6-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-6-FeedForward-Dropo (None, None, 768)    0           Transformer-6-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-6-FeedForward-Add ( (None, None, 768)    0           Transformer-6-MultiHeadSelfAttent
                                                                 Transformer-6-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-6-FeedForward-Norm  (None, None, 768)    1536        Transformer-6-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-7-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-6-FeedForward-Norm[0]
                                                                 Transformer-6-FeedForward-Norm[0]
                                                                 Transformer-6-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-7-MultiHeadSelfAtte (None, None, 768)    0           Transformer-7-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-7-MultiHeadSelfAtte (None, None, 768)    0           Transformer-6-FeedForward-Norm[0]
                                                                 Transformer-7-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-7-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-7-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-7-FeedForward (Feed (None, None, 768)    4722432     Transformer-7-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-7-FeedForward-Dropo (None, None, 768)    0           Transformer-7-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-7-FeedForward-Add ( (None, None, 768)    0           Transformer-7-MultiHeadSelfAttent
                                                                 Transformer-7-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-7-FeedForward-Norm  (None, None, 768)    1536        Transformer-7-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-8-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-7-FeedForward-Norm[0]
                                                                 Transformer-7-FeedForward-Norm[0]
                                                                 Transformer-7-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-8-MultiHeadSelfAtte (None, None, 768)    0           Transformer-8-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-8-MultiHeadSelfAtte (None, None, 768)    0           Transformer-7-FeedForward-Norm[0]
                                                                 Transformer-8-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-8-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-8-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-8-FeedForward (Feed (None, None, 768)    4722432     Transformer-8-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-8-FeedForward-Dropo (None, None, 768)    0           Transformer-8-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-8-FeedForward-Add ( (None, None, 768)    0           Transformer-8-MultiHeadSelfAttent
                                                                 Transformer-8-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-8-FeedForward-Norm  (None, None, 768)    1536        Transformer-8-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-9-MultiHeadSelfAtte (None, None, 768)    2362368     Transformer-8-FeedForward-Norm[0]
                                                                 Transformer-8-FeedForward-Norm[0]
                                                                 Transformer-8-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-9-MultiHeadSelfAtte (None, None, 768)    0           Transformer-9-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-9-MultiHeadSelfAtte (None, None, 768)    0           Transformer-8-FeedForward-Norm[0]
                                                                 Transformer-9-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-9-MultiHeadSelfAtte (None, None, 768)    1536        Transformer-9-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-9-FeedForward (Feed (None, None, 768)    4722432     Transformer-9-MultiHeadSelfAttent
__________________________________________________________________________________________________
Transformer-9-FeedForward-Dropo (None, None, 768)    0           Transformer-9-FeedForward[0][0]  
__________________________________________________________________________________________________
Transformer-9-FeedForward-Add ( (None, None, 768)    0           Transformer-9-MultiHeadSelfAttent
                                                                 Transformer-9-FeedForward-Dropout
__________________________________________________________________________________________________
Transformer-9-FeedForward-Norm  (None, None, 768)    1536        Transformer-9-FeedForward-Add[0][
__________________________________________________________________________________________________
Transformer-10-MultiHeadSelfAtt (None, None, 768)    2362368     Transformer-9-FeedForward-Norm[0]
                                                                 Transformer-9-FeedForward-Norm[0]
                                                                 Transformer-9-FeedForward-Norm[0]
__________________________________________________________________________________________________
Transformer-10-MultiHeadSelfAtt (None, None, 768)    0           Transformer-10-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-10-MultiHeadSelfAtt (None, None, 768)    0           Transformer-9-FeedForward-Norm[0]
                                                                 Transformer-10-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-10-MultiHeadSelfAtt (None, None, 768)    1536        Transformer-10-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-10-FeedForward (Fee (None, None, 768)    4722432     Transformer-10-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-10-FeedForward-Drop (None, None, 768)    0           Transformer-10-FeedForward[0][0] 
__________________________________________________________________________________________________
Transformer-10-FeedForward-Add  (None, None, 768)    0           Transformer-10-MultiHeadSelfAtten
                                                                 Transformer-10-FeedForward-Dropou
__________________________________________________________________________________________________
Transformer-10-FeedForward-Norm (None, None, 768)    1536        Transformer-10-FeedForward-Add[0]
__________________________________________________________________________________________________
Transformer-11-MultiHeadSelfAtt (None, None, 768)    2362368     Transformer-10-FeedForward-Norm[0
                                                                 Transformer-10-FeedForward-Norm[0
                                                                 Transformer-10-FeedForward-Norm[0
__________________________________________________________________________________________________
Transformer-11-MultiHeadSelfAtt (None, None, 768)    0           Transformer-11-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-11-MultiHeadSelfAtt (None, None, 768)    0           Transformer-10-FeedForward-Norm[0
                                                                 Transformer-11-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-11-MultiHeadSelfAtt (None, None, 768)    1536        Transformer-11-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-11-FeedForward (Fee (None, None, 768)    4722432     Transformer-11-MultiHeadSelfAtten
__________________________________________________________________________________________________
Transformer-11-FeedForward-Drop (None, None, 768)    0           Transformer-11-FeedForward[0][0] 
__________________________________________________________________________________________________
Transformer-11-FeedForward-Add  (None, None, 768)    0           Transformer-11-MultiHeadSelfAtten
                                                                 Transformer-11-FeedForward-Dropou
__________________________________________________________________________________________________
Transformer-11-FeedForward-Norm (None, None, 768)    1536        Transformer-11-FeedForward-Add[0]
__________________________________________________________________________________________________
Pooler (Lambda)                 (None, 768)          0           Transformer-11-FeedForward-Norm[0
__________________________________________________________________________________________________
Pooler-Dense (Dense)            (None, 768)          590592      Pooler[0][0]                     
__________________________________________________________________________________________________
dropout_1 (Dropout)             (None, 768)          0           Pooler-Dense[0][0]               
__________________________________________________________________________________________________
dense_73 (Dense)                (None, 2)            1538        dropout_1[0][0]                  
==================================================================================================
Total params: 102,269,186
Trainable params: 102,269,186
Non-trainable params: 0
__________________________________________________________________________________________________
WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

Epoch 1/20
3125/3125 [==============================] - 755s 242ms/step - loss: 0.3337 - accuracy: 0.8556
val_acc: 0.8417, best_val_acc: 0.8417
Epoch 2/20
3125/3125 [==============================] - 747s 239ms/step - loss: 0.1676 - accuracy: 0.9366
val_acc: 0.8401, best_val_acc: 0.8417
Epoch 3/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0996 - accuracy: 0.9636
val_acc: 0.8306, best_val_acc: 0.8417
Epoch 4/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0684 - accuracy: 0.9758
val_acc: 0.8409, best_val_acc: 0.8417
Epoch 5/20
3125/3125 [==============================] - 745s 238ms/step - loss: 0.0521 - accuracy: 0.9819
val_acc: 0.8428, best_val_acc: 0.8428
Epoch 6/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0423 - accuracy: 0.9854
val_acc: 0.8386, best_val_acc: 0.8428
Epoch 7/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0368 - accuracy: 0.9874
val_acc: 0.8436, best_val_acc: 0.8436
Epoch 8/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0315 - accuracy: 0.9893
val_acc: 0.8419, best_val_acc: 0.8436
Epoch 9/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0288 - accuracy: 0.9904
val_acc: 0.8357, best_val_acc: 0.8436
Epoch 10/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0269 - accuracy: 0.9908
val_acc: 0.8373, best_val_acc: 0.8436
Epoch 11/20
3125/3125 [==============================] - 747s 239ms/step - loss: 0.0241 - accuracy: 0.9916
val_acc: 0.8448, best_val_acc: 0.8448
Epoch 12/20
3125/3125 [==============================] - 746s 239ms/step - loss: 0.0218 - accuracy: 0.9928
val_acc: 0.8414, best_val_acc: 0.8448
Epoch 13/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0212 - accuracy: 0.9932
val_acc: 0.8372, best_val_acc: 0.8448
Epoch 14/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0192 - accuracy: 0.9935
val_acc: 0.8411, best_val_acc: 0.8448
Epoch 15/20
3125/3125 [==============================] - 747s 239ms/step - loss: 0.0181 - accuracy: 0.9942
val_acc: 0.8399, best_val_acc: 0.8448
Epoch 16/20
3125/3125 [==============================] - 745s 239ms/step - loss: 0.0178 - accuracy: 0.9941
val_acc: 0.8475, best_val_acc: 0.8475
Epoch 17/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0160 - accuracy: 0.9948
val_acc: 0.8408, best_val_acc: 0.8475
Epoch 18/20
3125/3125 [==============================] - 745s 238ms/step - loss: 0.0172 - accuracy: 0.9943
val_acc: 0.8454, best_val_acc: 0.8475
Epoch 19/20
3125/3125 [==============================] - 747s 239ms/step - loss: 0.0152 - accuracy: 0.9952
val_acc: 0.8302, best_val_acc: 0.8475
Epoch 20/20
3125/3125 [==============================] - 744s 238ms/step - loss: 0.0159 - accuracy: 0.9951
val_acc: 0.8410, best_val_acc: 0.8475

<keras.callbacks.callbacks.History at 0x7fe2b0daafd0>