In [0]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/knowledge_graph_completion/wn18/main')
In [0]:
!pip install tensorflow-gpu==2.0.0-beta1
Collecting tensorflow-gpu==2.0.0-beta1
  Downloading https://files.pythonhosted.org/packages/2b/53/e18c5e7a2263d3581a979645a185804782e59b8e13f42b9c3c3cfb5bb503/tensorflow_gpu-2.0.0b1-cp36-cp36m-manylinux1_x86_64.whl (348.9MB)
     |████████████████████████████████| 348.9MB 56kB/s 
Collecting tb-nightly<1.14.0a20190604,>=1.14.0a20190603 (from tensorflow-gpu==2.0.0-beta1)
  Downloading https://files.pythonhosted.org/packages/a4/96/571b875cd81dda9d5dfa1422a4f9d749e67c0a8d4f4f0b33a4e5f5f35e27/tb_nightly-1.14.0a20190603-py3-none-any.whl (3.1MB)
     |████████████████████████████████| 3.1MB 39.3MB/s 
Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.11.2)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.16.4)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.0.8)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.33.4)
Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.1.7)
Collecting tf-estimator-nightly<1.14.0.dev2019060502,>=1.14.0.dev2019060501 (from tensorflow-gpu==2.0.0-beta1)
  Downloading https://files.pythonhosted.org/packages/32/dd/99c47dd007dcf10d63fd895611b063732646f23059c618a373e85019eb0e/tf_estimator_nightly-1.14.0.dev2019060501-py2.py3-none-any.whl (496kB)
     |████████████████████████████████| 501kB 41.3MB/s 
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.15.0)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (3.7.1)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.2.2)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.7.1)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.1.0)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (0.8.0)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-beta1) (1.12.0)
Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (41.0.1)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (3.1.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190604,>=1.14.0a20190603->tensorflow-gpu==2.0.0-beta1) (0.15.5)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-beta1) (2.8.0)
Installing collected packages: tb-nightly, tf-estimator-nightly, tensorflow-gpu
Successfully installed tb-nightly-1.14.0a20190603 tensorflow-gpu-2.0.0b1 tf-estimator-nightly-1.14.0.dev2019060501
In [0]:
import tensorflow as tf
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-beta1
GPU Enabled: True
In [0]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
"""
we use 1vN fast evaluation as purposed in ConvE paper:
"https://arxiv.org/abs/1707.01476"
sp2o is a dictionary that maps a pair of <subject, predicate>
to multiple possible corresponding <objects> in graph
"""
def make_sp2o(f_paths, e2idx, r2idx):
    sp2o = {}
    for f_path in f_paths:
      with open(f_path) as f:
        for line in f:
            line = line.rstrip()
            s, p, o = line.split()
            s, p, o = e2idx[s], r2idx[p], e2idx[o]
            if (s,p) not in sp2o:
                sp2o[(s,p)] = [o]
            else:
                if o not in sp2o[(s,p)]:
                    sp2o[(s,p)].append(o)
    return sp2o
In [0]:
def map_fn(x, y):
  i, v, s = y[0]
  one_hot = tf.SparseTensor(i, v, s)
  return x, (one_hot, y[1], y[2])


# stream data from text files
def data_generator(f_path, params, sp2o):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      s, p, o = line.split()
      s, p, o = params['e2idx'][s], params['r2idx'][p], params['e2idx'][o]
      sparse_i = [[x] for x in sp2o[(s, p)]]
      sparse_v = [1.] * len(sparse_i)
      sparse_s = [len(params['e2idx'])]
      yield ((s, p), ((sparse_i, sparse_v, sparse_s), o, len(sparse_i)))


def dataset(is_training, params, sp2o):
  _shapes = (([], []), (([None, 1], [None], [1]), [], []))
  _types = ((tf.int32, tf.int32),
            ((tf.int64, tf.float32, tf.int64), tf.int32, tf.int32))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  return ds
In [0]:
def update_metrics(scores, query, metrics):
  to_float = lambda x: tf.cast(x, tf.float32)
  
  _, i = tf.math.top_k(scores, sorted=True, k=scores.shape[1])
  query = tf.expand_dims(query, 1)
  is_query = to_float(tf.equal(i, query))
  r = tf.argmax(is_query, -1) + 1
  
  mrr = 1. / to_float(r)
  hits_10 = to_float(tf.less_equal(r, 10))
  hits_3 = to_float(tf.less_equal(r, 3))
  hits_1 = to_float(tf.less_equal(r, 1))
  
  metrics['mrr'].update_state(mrr)
  metrics['hits_10'].update_state(hits_10)
  metrics['hits_3'].update_state(hits_3)
  metrics['hits_1'].update_state(hits_1)
In [0]:
class Complex(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embed_e_real = tf.keras.layers.Embedding(input_dim=len(params['e2idx']),
                                                  output_dim=params['embed_dim'],
                                                  embeddings_initializer=tf.initializers.RandomUniform(),
                                                  name='Entity_Real')
    
    self.embed_e_img = tf.keras.layers.Embedding(input_dim=len(params['e2idx']),
                                                 output_dim=params['embed_dim'],
                                                 embeddings_initializer=tf.initializers.RandomUniform(),
                                                 name='Entity_Img')
    
    self.embed_rel_real = tf.keras.layers.Embedding(input_dim=len(params['r2idx']),
                                                    output_dim=params['embed_dim'],
                                                    embeddings_initializer=tf.initializers.RandomUniform(),
                                                    name='Relation_Real')
    
    self.embed_rel_img = tf.keras.layers.Embedding(input_dim=len(params['r2idx']),
                                                   output_dim=params['embed_dim'],
                                                   embeddings_initializer=tf.initializers.RandomUniform(),
                                                   name='Relation_Img')
    
    self.out_bias = self.add_weight(name='out_bias', shape=[len(params['e2idx'])])
  
  
  def call(self, inputs):
    s, p = inputs
    
    s_real = self.embed_e_real(s)
    p_real = self.embed_rel_real(p)
    s_img = self.embed_e_img(s)
    p_img = self.embed_rel_img(p)
    
    realrealreal = tf.matmul(s_real*p_real, self.embed_e_real.embeddings, transpose_b=True)
    realimgimg = tf.matmul(s_real*p_img, self.embed_e_img.embeddings, transpose_b=True)
    imgrealimg = tf.matmul(s_img*p_real, self.embed_e_img.embeddings, transpose_b=True)
    imgimgreal = tf.matmul(s_img*p_img, self.embed_e_real.embeddings, transpose_b=True)
    
    x = realrealreal + realimgimg + imgrealimg - imgimgreal
    x = tf.nn.bias_add(x, self.out_bias)
    return x
In [0]:
params = {
    'train_path': '../data/wn18/train.txt',
    'valid_path': '../data/wn18/valid.txt',
    'test_path': '../data/wn18/test.txt',
    'entity_path': '../vocab/entity.txt',
    'relation_path': '../vocab/relation.txt',
    'batch_size': 128,
    'embed_dim': 200,
    'num_samples': 141442,
    'lr': 3e-3,
    'num_patience': 3,
}
In [0]:
params['e2idx'] = get_vocab(params['entity_path'])
params['r2idx'] = get_vocab(params['relation_path'])
sp2o_tr = make_sp2o([params['train_path']], params['e2idx'], params['r2idx'])
sp2o_all = make_sp2o([params['train_path'],
                      params['test_path'],
                      params['valid_path']], params['e2idx'], params['r2idx'])
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True 
In [12]:
model = Complex(params)
model.build(input_shape=[[None], [None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.96)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

best_mrr = 0.
history_mrr = []

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=True, params=params, sp2o=sp2o_tr):
    with tf.GradientTape() as tape:
      logits = model((s, p))
      multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
      num_neg = len(params['e2idx']) - num_pos
      pos_weight = tf.expand_dims(tf.cast(num_neg/num_pos, tf.float32), 1)
      loss = tf.nn.weighted_cross_entropy_with_logits(labels=multi_o, logits=logits, pos_weight=pos_weight)
      loss = tf.reduce_mean(loss)
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  metrics = {
    'mrr': tf.metrics.Mean(),
    'hits_10': tf.metrics.Mean(),
    'hits_3': tf.metrics.Mean(),
    'hits_1': tf.metrics.Mean(),
  }
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=False, params=params, sp2o=sp2o_all):
    logits = model((s, p))
    multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
    # create masks for Filtered MRR
    o_one_hot = tf.one_hot(o, len(params['e2idx']))
    unwanted = multi_o - o_one_hot
    masks = tf.cast(tf.equal(unwanted, 0.), tf.float32)
    scores = tf.sigmoid(logits) * masks
    
    update_metrics(scores=scores, query=o, metrics=metrics)
  
  logger.info("MRR: {:.3f}| [email protected]: {:.3f} | [email protected]: {:.3f} | [email protected]: {:.3f}".format(
    metrics['mrr'].result().numpy(),
    metrics['hits_10'].result().numpy(),
    metrics['hits_3'].result().numpy(),
    metrics['hits_1'].result().numpy()))
  
  mrr = metrics['mrr'].result().numpy()
  history_mrr.append(mrr)
  
  if mrr > best_mrr:
    best_mrr = mrr
    # you can save model here
  logger.info("Best MRR: {:.3f}".format(best_mrr))
  
  if len(history_mrr) > params['num_patience'] and is_descending(history_mrr):
    logger.info("MRR not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0726 00:14:32.546877 139830849898368 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:505: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('Entity_Real/embeddings:0', TensorShape([40943, 200])),
 ('Entity_Img/embeddings:0', TensorShape([40943, 200])),
 ('Relation_Real/embeddings:0', TensorShape([18, 200])),
 ('Relation_Img/embeddings:0', TensorShape([18, 200])),
 ('out_bias:0', TensorShape([40943]))]
Reading ../data/wn18/train.txt
I0726 00:15:13.309617 139830849898368 interactiveshell.py:2882] Step 0 | Loss: 1.3858 | Spent: 40.8 secs | LR: 0.003000
I0726 00:15:25.789380 139830849898368 interactiveshell.py:2882] Step 50 | Loss: 1.3779 | Spent: 12.5 secs | LR: 0.002994
I0726 00:15:38.229571 139830849898368 interactiveshell.py:2882] Step 100 | Loss: 1.3657 | Spent: 12.4 secs | LR: 0.002988
I0726 00:15:50.725374 139830849898368 interactiveshell.py:2882] Step 150 | Loss: 1.3187 | Spent: 12.5 secs | LR: 0.002982
I0726 00:16:03.203264 139830849898368 interactiveshell.py:2882] Step 200 | Loss: 1.2276 | Spent: 12.5 secs | LR: 0.002976
I0726 00:16:15.632906 139830849898368 interactiveshell.py:2882] Step 250 | Loss: 1.0719 | Spent: 12.4 secs | LR: 0.002970
I0726 00:16:28.063027 139830849898368 interactiveshell.py:2882] Step 300 | Loss: 0.9416 | Spent: 12.4 secs | LR: 0.002963
I0726 00:16:40.468199 139830849898368 interactiveshell.py:2882] Step 350 | Loss: 0.9195 | Spent: 12.4 secs | LR: 0.002957
I0726 00:16:52.893753 139830849898368 interactiveshell.py:2882] Step 400 | Loss: 0.9095 | Spent: 12.4 secs | LR: 0.002951
I0726 00:17:05.346761 139830849898368 interactiveshell.py:2882] Step 450 | Loss: 0.7163 | Spent: 12.5 secs | LR: 0.002945
I0726 00:17:17.743800 139830849898368 interactiveshell.py:2882] Step 500 | Loss: 0.6726 | Spent: 12.4 secs | LR: 0.002939
I0726 00:17:30.197510 139830849898368 interactiveshell.py:2882] Step 550 | Loss: 0.6722 | Spent: 12.5 secs | LR: 0.002933
I0726 00:17:42.577505 139830849898368 interactiveshell.py:2882] Step 600 | Loss: 0.5887 | Spent: 12.4 secs | LR: 0.002927
I0726 00:17:55.026650 139830849898368 interactiveshell.py:2882] Step 650 | Loss: 0.5577 | Spent: 12.4 secs | LR: 0.002921
I0726 00:18:07.476079 139830849898368 interactiveshell.py:2882] Step 700 | Loss: 0.5180 | Spent: 12.4 secs | LR: 0.002915
I0726 00:18:19.962279 139830849898368 interactiveshell.py:2882] Step 750 | Loss: 0.4062 | Spent: 12.5 secs | LR: 0.002910
I0726 00:18:32.603715 139830849898368 interactiveshell.py:2882] Step 800 | Loss: 0.5187 | Spent: 12.6 secs | LR: 0.002904
I0726 00:18:45.049248 139830849898368 interactiveshell.py:2882] Step 850 | Loss: 0.3704 | Spent: 12.4 secs | LR: 0.002898
I0726 00:18:57.515890 139830849898368 interactiveshell.py:2882] Step 900 | Loss: 0.3857 | Spent: 12.5 secs | LR: 0.002892
I0726 00:19:10.006841 139830849898368 interactiveshell.py:2882] Step 950 | Loss: 0.3791 | Spent: 12.5 secs | LR: 0.002886
I0726 00:19:22.445520 139830849898368 interactiveshell.py:2882] Step 1000 | Loss: 0.3296 | Spent: 12.4 secs | LR: 0.002880
I0726 00:19:34.787384 139830849898368 interactiveshell.py:2882] Step 1050 | Loss: 0.3072 | Spent: 12.3 secs | LR: 0.002874
I0726 00:19:47.147488 139830849898368 interactiveshell.py:2882] Step 1100 | Loss: 0.2246 | Spent: 12.4 secs | LR: 0.002868
Reading ../data/wn18/test.txt
I0726 00:19:53.907614 139830849898368 interactiveshell.py:2882] MRR: 0.685| [email protected]: 0.824 | [email protected]: 0.751 | [email protected]: 0.601
I0726 00:19:53.911866 139830849898368 interactiveshell.py:2882] Best MRR: 0.685
Reading ../data/wn18/train.txt
I0726 00:20:44.938829 139830849898368 interactiveshell.py:2882] Step 1150 | Loss: 0.1216 | Spent: 57.8 secs | LR: 0.002862
I0726 00:20:57.396371 139830849898368 interactiveshell.py:2882] Step 1200 | Loss: 0.0895 | Spent: 12.5 secs | LR: 0.002857
I0726 00:21:09.873353 139830849898368 interactiveshell.py:2882] Step 1250 | Loss: 0.0906 | Spent: 12.5 secs | LR: 0.002851
I0726 00:21:22.355462 139830849898368 interactiveshell.py:2882] Step 1300 | Loss: 0.0719 | Spent: 12.5 secs | LR: 0.002845
I0726 00:21:34.724158 139830849898368 interactiveshell.py:2882] Step 1350 | Loss: 0.0593 | Spent: 12.4 secs | LR: 0.002839
I0726 00:21:47.141274 139830849898368 interactiveshell.py:2882] Step 1400 | Loss: 0.0948 | Spent: 12.4 secs | LR: 0.002833
I0726 00:21:59.675215 139830849898368 interactiveshell.py:2882] Step 1450 | Loss: 0.0838 | Spent: 12.5 secs | LR: 0.002828
I0726 00:22:12.241033 139830849898368 interactiveshell.py:2882] Step 1500 | Loss: 0.1531 | Spent: 12.6 secs | LR: 0.002822
I0726 00:22:24.675242 139830849898368 interactiveshell.py:2882] Step 1550 | Loss: 0.0327 | Spent: 12.4 secs | LR: 0.002816
I0726 00:22:37.093944 139830849898368 interactiveshell.py:2882] Step 1600 | Loss: 0.0342 | Spent: 12.4 secs | LR: 0.002810
I0726 00:22:49.445286 139830849898368 interactiveshell.py:2882] Step 1650 | Loss: 0.0884 | Spent: 12.3 secs | LR: 0.002805
I0726 00:23:01.745079 139830849898368 interactiveshell.py:2882] Step 1700 | Loss: 0.0387 | Spent: 12.3 secs | LR: 0.002799
I0726 00:23:14.041967 139830849898368 interactiveshell.py:2882] Step 1750 | Loss: 0.0682 | Spent: 12.3 secs | LR: 0.002793
I0726 00:23:26.347203 139830849898368 interactiveshell.py:2882] Step 1800 | Loss: 0.0395 | Spent: 12.3 secs | LR: 0.002787
I0726 00:23:39.036048 139830849898368 interactiveshell.py:2882] Step 1850 | Loss: 0.0516 | Spent: 12.7 secs | LR: 0.002782
I0726 00:23:51.447423 139830849898368 interactiveshell.py:2882] Step 1900 | Loss: 0.0264 | Spent: 12.4 secs | LR: 0.002776
I0726 00:24:03.899363 139830849898368 interactiveshell.py:2882] Step 1950 | Loss: 0.0586 | Spent: 12.5 secs | LR: 0.002770
I0726 00:24:16.404395 139830849898368 interactiveshell.py:2882] Step 2000 | Loss: 0.1065 | Spent: 12.5 secs | LR: 0.002765
I0726 00:24:28.842429 139830849898368 interactiveshell.py:2882] Step 2050 | Loss: 0.0673 | Spent: 12.4 secs | LR: 0.002759
I0726 00:24:41.354681 139830849898368 interactiveshell.py:2882] Step 2100 | Loss: 0.0228 | Spent: 12.5 secs | LR: 0.002754
I0726 00:24:53.773330 139830849898368 interactiveshell.py:2882] Step 2150 | Loss: 0.0198 | Spent: 12.4 secs | LR: 0.002748
I0726 00:25:06.404974 139830849898368 interactiveshell.py:2882] Step 2200 | Loss: 0.0280 | Spent: 12.6 secs | LR: 0.002742
Reading ../data/wn18/test.txt
I0726 00:25:14.763506 139830849898368 interactiveshell.py:2882] MRR: 0.869| [email protected]: 0.929 | [email protected]: 0.898 | [email protected]: 0.833
I0726 00:25:14.768368 139830849898368 interactiveshell.py:2882] Best MRR: 0.869
Reading ../data/wn18/train.txt
I0726 00:26:04.636107 139830849898368 interactiveshell.py:2882] Step 2250 | Loss: 0.0095 | Spent: 58.2 secs | LR: 0.002737
I0726 00:26:17.084999 139830849898368 interactiveshell.py:2882] Step 2300 | Loss: 0.0127 | Spent: 12.4 secs | LR: 0.002731
I0726 00:26:29.554997 139830849898368 interactiveshell.py:2882] Step 2350 | Loss: 0.0094 | Spent: 12.5 secs | LR: 0.002726
I0726 00:26:42.039398 139830849898368 interactiveshell.py:2882] Step 2400 | Loss: 0.0084 | Spent: 12.5 secs | LR: 0.002720
I0726 00:26:54.572520 139830849898368 interactiveshell.py:2882] Step 2450 | Loss: 0.0088 | Spent: 12.5 secs | LR: 0.002714
I0726 00:27:07.070028 139830849898368 interactiveshell.py:2882] Step 2500 | Loss: 0.0077 | Spent: 12.5 secs | LR: 0.002709
I0726 00:27:19.562482 139830849898368 interactiveshell.py:2882] Step 2550 | Loss: 0.0079 | Spent: 12.5 secs | LR: 0.002703
I0726 00:27:32.059894 139830849898368 interactiveshell.py:2882] Step 2600 | Loss: 0.0070 | Spent: 12.5 secs | LR: 0.002698
I0726 00:27:44.614809 139830849898368 interactiveshell.py:2882] Step 2650 | Loss: 0.0057 | Spent: 12.6 secs | LR: 0.002692
I0726 00:27:57.136033 139830849898368 interactiveshell.py:2882] Step 2700 | Loss: 0.0078 | Spent: 12.5 secs | LR: 0.002687
I0726 00:28:09.648807 139830849898368 interactiveshell.py:2882] Step 2750 | Loss: 0.0070 | Spent: 12.5 secs | LR: 0.002681
I0726 00:28:22.198885 139830849898368 interactiveshell.py:2882] Step 2800 | Loss: 0.0045 | Spent: 12.5 secs | LR: 0.002676
I0726 00:28:34.763340 139830849898368 interactiveshell.py:2882] Step 2850 | Loss: 0.0062 | Spent: 12.6 secs | LR: 0.002671
I0726 00:28:47.496386 139830849898368 interactiveshell.py:2882] Step 2900 | Loss: 0.0056 | Spent: 12.7 secs | LR: 0.002665
I0726 00:29:00.016962 139830849898368 interactiveshell.py:2882] Step 2950 | Loss: 0.0051 | Spent: 12.5 secs | LR: 0.002660
I0726 00:29:12.531445 139830849898368 interactiveshell.py:2882] Step 3000 | Loss: 0.0062 | Spent: 12.5 secs | LR: 0.002654
I0726 00:29:25.034653 139830849898368 interactiveshell.py:2882] Step 3050 | Loss: 0.0044 | Spent: 12.5 secs | LR: 0.002649
I0726 00:29:37.502150 139830849898368 interactiveshell.py:2882] Step 3100 | Loss: 0.0040 | Spent: 12.5 secs | LR: 0.002643
I0726 00:29:49.982738 139830849898368 interactiveshell.py:2882] Step 3150 | Loss: 0.0034 | Spent: 12.5 secs | LR: 0.002638
I0726 00:30:02.442935 139830849898368 interactiveshell.py:2882] Step 3200 | Loss: 0.0037 | Spent: 12.5 secs | LR: 0.002633
I0726 00:30:15.073267 139830849898368 interactiveshell.py:2882] Step 3250 | Loss: 0.0033 | Spent: 12.6 secs | LR: 0.002627
I0726 00:30:27.606940 139830849898368 interactiveshell.py:2882] Step 3300 | Loss: 0.0033 | Spent: 12.5 secs | LR: 0.002622
Reading ../data/wn18/test.txt
I0726 00:30:37.507411 139830849898368 interactiveshell.py:2882] MRR: 0.898| [email protected]: 0.940 | [email protected]: 0.921 | [email protected]: 0.872
I0726 00:30:37.512624 139830849898368 interactiveshell.py:2882] Best MRR: 0.898
Reading ../data/wn18/train.txt
I0726 00:31:25.783589 139830849898368 interactiveshell.py:2882] Step 3350 | Loss: 0.0023 | Spent: 58.2 secs | LR: 0.002617
I0726 00:31:38.260471 139830849898368 interactiveshell.py:2882] Step 3400 | Loss: 0.0026 | Spent: 12.5 secs | LR: 0.002611
I0726 00:31:50.751790 139830849898368 interactiveshell.py:2882] Step 3450 | Loss: 0.0029 | Spent: 12.5 secs | LR: 0.002606
I0726 00:32:03.239024 139830849898368 interactiveshell.py:2882] Step 3500 | Loss: 0.0029 | Spent: 12.5 secs | LR: 0.002601
I0726 00:32:15.679461 139830849898368 interactiveshell.py:2882] Step 3550 | Loss: 0.0022 | Spent: 12.4 secs | LR: 0.002595
I0726 00:32:28.236600 139830849898368 interactiveshell.py:2882] Step 3600 | Loss: 0.0025 | Spent: 12.6 secs | LR: 0.002590
I0726 00:32:40.738658 139830849898368 interactiveshell.py:2882] Step 3650 | Loss: 0.0021 | Spent: 12.5 secs | LR: 0.002585
I0726 00:32:53.263463 139830849898368 interactiveshell.py:2882] Step 3700 | Loss: 0.0020 | Spent: 12.5 secs | LR: 0.002579
I0726 00:33:05.762552 139830849898368 interactiveshell.py:2882] Step 3750 | Loss: 0.0024 | Spent: 12.5 secs | LR: 0.002574
I0726 00:33:18.235314 139830849898368 interactiveshell.py:2882] Step 3800 | Loss: 0.0015 | Spent: 12.5 secs | LR: 0.002569
I0726 00:33:30.740544 139830849898368 interactiveshell.py:2882] Step 3850 | Loss: 0.0019 | Spent: 12.5 secs | LR: 0.002564
I0726 00:33:43.236084 139830849898368 interactiveshell.py:2882] Step 3900 | Loss: 0.0016 | Spent: 12.5 secs | LR: 0.002558
I0726 00:33:55.926439 139830849898368 interactiveshell.py:2882] Step 3950 | Loss: 0.0024 | Spent: 12.7 secs | LR: 0.002553
I0726 00:34:08.573990 139830849898368 interactiveshell.py:2882] Step 4000 | Loss: 0.0016 | Spent: 12.6 secs | LR: 0.002548
I0726 00:34:21.104792 139830849898368 interactiveshell.py:2882] Step 4050 | Loss: 0.0025 | Spent: 12.5 secs | LR: 0.002543
I0726 00:34:33.609894 139830849898368 interactiveshell.py:2882] Step 4100 | Loss: 0.0014 | Spent: 12.5 secs | LR: 0.002538
I0726 00:34:46.036811 139830849898368 interactiveshell.py:2882] Step 4150 | Loss: 0.0014 | Spent: 12.4 secs | LR: 0.002532
I0726 00:34:58.537425 139830849898368 interactiveshell.py:2882] Step 4200 | Loss: 0.0016 | Spent: 12.5 secs | LR: 0.002527
I0726 00:35:11.001888 139830849898368 interactiveshell.py:2882] Step 4250 | Loss: 0.0014 | Spent: 12.5 secs | LR: 0.002522
I0726 00:35:23.569116 139830849898368 interactiveshell.py:2882] Step 4300 | Loss: 0.0012 | Spent: 12.6 secs | LR: 0.002517
I0726 00:35:36.011733 139830849898368 interactiveshell.py:2882] Step 4350 | Loss: 0.0016 | Spent: 12.4 secs | LR: 0.002512
I0726 00:35:48.501270 139830849898368 interactiveshell.py:2882] Step 4400 | Loss: 0.0026 | Spent: 12.5 secs | LR: 0.002507
Reading ../data/wn18/test.txt
I0726 00:35:59.800635 139830849898368 interactiveshell.py:2882] MRR: 0.917| [email protected]: 0.946 | [email protected]: 0.932 | [email protected]: 0.898
I0726 00:35:59.808879 139830849898368 interactiveshell.py:2882] Best MRR: 0.917
Reading ../data/wn18/train.txt
I0726 00:36:46.161354 139830849898368 interactiveshell.py:2882] Step 4450 | Loss: 0.0010 | Spent: 57.7 secs | LR: 0.002502
I0726 00:36:58.612127 139830849898368 interactiveshell.py:2882] Step 4500 | Loss: 0.0011 | Spent: 12.4 secs | LR: 0.002497
I0726 00:37:11.160741 139830849898368 interactiveshell.py:2882] Step 4550 | Loss: 0.0014 | Spent: 12.5 secs | LR: 0.002491
I0726 00:37:23.684003 139830849898368 interactiveshell.py:2882] Step 4600 | Loss: 0.0013 | Spent: 12.5 secs | LR: 0.002486
I0726 00:37:36.103784 139830849898368 interactiveshell.py:2882] Step 4650 | Loss: 0.0009 | Spent: 12.4 secs | LR: 0.002481
I0726 00:37:48.557719 139830849898368 interactiveshell.py:2882] Step 4700 | Loss: 0.0009 | Spent: 12.5 secs | LR: 0.002476
I0726 00:38:01.032455 139830849898368 interactiveshell.py:2882] Step 4750 | Loss: 0.0010 | Spent: 12.5 secs | LR: 0.002471
I0726 00:38:13.567363 139830849898368 interactiveshell.py:2882] Step 4800 | Loss: 0.0010 | Spent: 12.5 secs | LR: 0.002466
I0726 00:38:26.006003 139830849898368 interactiveshell.py:2882] Step 4850 | Loss: 0.0010 | Spent: 12.4 secs | LR: 0.002461
I0726 00:38:38.464464 139830849898368 interactiveshell.py:2882] Step 4900 | Loss: 0.0009 | Spent: 12.5 secs | LR: 0.002456
I0726 00:38:50.845068 139830849898368 interactiveshell.py:2882] Step 4950 | Loss: 0.0009 | Spent: 12.4 secs | LR: 0.002451
I0726 00:39:03.343152 139830849898368 interactiveshell.py:2882] Step 5000 | Loss: 0.0007 | Spent: 12.5 secs | LR: 0.002446
I0726 00:39:15.987538 139830849898368 interactiveshell.py:2882] Step 5050 | Loss: 0.0009 | Spent: 12.6 secs | LR: 0.002441
I0726 00:39:28.444747 139830849898368 interactiveshell.py:2882] Step 5100 | Loss: 0.0009 | Spent: 12.5 secs | LR: 0.002436
I0726 00:39:40.908010 139830849898368 interactiveshell.py:2882] Step 5150 | Loss: 0.0009 | Spent: 12.5 secs | LR: 0.002431
I0726 00:39:53.370702 139830849898368 interactiveshell.py:2882] Step 5200 | Loss: 0.0010 | Spent: 12.5 secs | LR: 0.002426
I0726 00:40:05.807497 139830849898368 interactiveshell.py:2882] Step 5250 | Loss: 0.0008 | Spent: 12.4 secs | LR: 0.002421
I0726 00:40:18.248280 139830849898368 interactiveshell.py:2882] Step 5300 | Loss: 0.0008 | Spent: 12.4 secs | LR: 0.002416
I0726 00:40:30.789010 139830849898368 interactiveshell.py:2882] Step 5350 | Loss: 0.0009 | Spent: 12.5 secs | LR: 0.002411
I0726 00:40:43.236534 139830849898368 interactiveshell.py:2882] Step 5400 | Loss: 0.0007 | Spent: 12.4 secs | LR: 0.002407
I0726 00:40:55.665714 139830849898368 interactiveshell.py:2882] Step 5450 | Loss: 0.0008 | Spent: 12.4 secs | LR: 0.002402
I0726 00:41:08.213079 139830849898368 interactiveshell.py:2882] Step 5500 | Loss: 0.0007 | Spent: 12.5 secs | LR: 0.002397
Reading ../data/wn18/test.txt
I0726 00:41:20.938351 139830849898368 interactiveshell.py:2882] MRR: 0.923| [email protected]: 0.947 | [email protected]: 0.935 | [email protected]: 0.908
I0726 00:41:20.947011 139830849898368 interactiveshell.py:2882] Best MRR: 0.923
Reading ../data/wn18/train.txt
I0726 00:42:06.135334 139830849898368 interactiveshell.py:2882] Step 5550 | Loss: 0.0005 | Spent: 57.9 secs | LR: 0.002392
I0726 00:42:18.621170 139830849898368 interactiveshell.py:2882] Step 5600 | Loss: 0.0006 | Spent: 12.5 secs | LR: 0.002387
I0726 00:42:31.052489 139830849898368 interactiveshell.py:2882] Step 5650 | Loss: 0.0006 | Spent: 12.4 secs | LR: 0.002382
I0726 00:42:43.500790 139830849898368 interactiveshell.py:2882] Step 5700 | Loss: 0.0006 | Spent: 12.4 secs | LR: 0.002377
I0726 00:42:55.930796 139830849898368 interactiveshell.py:2882] Step 5750 | Loss: 0.0005 | Spent: 12.4 secs | LR: 0.002372
I0726 00:43:08.392253 139830849898368 interactiveshell.py:2882] Step 5800 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002368
I0726 00:43:20.814773 139830849898368 interactiveshell.py:2882] Step 5850 | Loss: 0.0004 | Spent: 12.4 secs | LR: 0.002363
I0726 00:43:33.324902 139830849898368 interactiveshell.py:2882] Step 5900 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002358
I0726 00:43:45.840377 139830849898368 interactiveshell.py:2882] Step 5950 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002353
I0726 00:43:58.295000 139830849898368 interactiveshell.py:2882] Step 6000 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002348
I0726 00:44:10.749198 139830849898368 interactiveshell.py:2882] Step 6050 | Loss: 0.0004 | Spent: 12.5 secs | LR: 0.002343
I0726 00:44:23.406784 139830849898368 interactiveshell.py:2882] Step 6100 | Loss: 0.0005 | Spent: 12.7 secs | LR: 0.002339
I0726 00:44:35.891355 139830849898368 interactiveshell.py:2882] Step 6150 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002334
I0726 00:44:48.413989 139830849898368 interactiveshell.py:2882] Step 6200 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002329
I0726 00:45:00.849756 139830849898368 interactiveshell.py:2882] Step 6250 | Loss: 0.0005 | Spent: 12.4 secs | LR: 0.002324
I0726 00:45:13.264910 139830849898368 interactiveshell.py:2882] Step 6300 | Loss: 0.0005 | Spent: 12.4 secs | LR: 0.002320
I0726 00:45:25.713924 139830849898368 interactiveshell.py:2882] Step 6350 | Loss: 0.0005 | Spent: 12.4 secs | LR: 0.002315
I0726 00:45:38.212860 139830849898368 interactiveshell.py:2882] Step 6400 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002310
I0726 00:45:50.657805 139830849898368 interactiveshell.py:2882] Step 6450 | Loss: 0.0004 | Spent: 12.4 secs | LR: 0.002306
I0726 00:46:03.121655 139830849898368 interactiveshell.py:2882] Step 6500 | Loss: 0.0004 | Spent: 12.5 secs | LR: 0.002301
I0726 00:46:15.466275 139830849898368 interactiveshell.py:2882] Step 6550 | Loss: 0.0003 | Spent: 12.3 secs | LR: 0.002296
I0726 00:46:27.846889 139830849898368 interactiveshell.py:2882] Step 6600 | Loss: 0.0005 | Spent: 12.4 secs | LR: 0.002291
Reading ../data/wn18/test.txt
I0726 00:46:41.958081 139830849898368 interactiveshell.py:2882] MRR: 0.929| [email protected]: 0.949 | [email protected]: 0.939 | [email protected]: 0.916
I0726 00:46:41.969484 139830849898368 interactiveshell.py:2882] Best MRR: 0.929
Reading ../data/wn18/train.txt
I0726 00:47:25.367014 139830849898368 interactiveshell.py:2882] Step 6650 | Loss: 0.0003 | Spent: 57.5 secs | LR: 0.002287
I0726 00:47:37.790865 139830849898368 interactiveshell.py:2882] Step 6700 | Loss: 0.0004 | Spent: 12.4 secs | LR: 0.002282
I0726 00:47:50.236618 139830849898368 interactiveshell.py:2882] Step 6750 | Loss: 0.0003 | Spent: 12.4 secs | LR: 0.002277
I0726 00:48:02.703660 139830849898368 interactiveshell.py:2882] Step 6800 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002273
I0726 00:48:15.119761 139830849898368 interactiveshell.py:2882] Step 6850 | Loss: 0.0003 | Spent: 12.4 secs | LR: 0.002268
I0726 00:48:27.621771 139830849898368 interactiveshell.py:2882] Step 6900 | Loss: 0.0005 | Spent: 12.5 secs | LR: 0.002264
I0726 00:48:40.121664 139830849898368 interactiveshell.py:2882] Step 6950 | Loss: 0.0004 | Spent: 12.5 secs | LR: 0.002259
I0726 00:48:52.638270 139830849898368 interactiveshell.py:2882] Step 7000 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002254
I0726 00:49:05.130889 139830849898368 interactiveshell.py:2882] Step 7050 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002250
I0726 00:49:17.521052 139830849898368 interactiveshell.py:2882] Step 7100 | Loss: 0.0003 | Spent: 12.4 secs | LR: 0.002245
I0726 00:49:30.184077 139830849898368 interactiveshell.py:2882] Step 7150 | Loss: 0.0003 | Spent: 12.7 secs | LR: 0.002241
I0726 00:49:42.665784 139830849898368 interactiveshell.py:2882] Step 7200 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002236
I0726 00:49:55.149352 139830849898368 interactiveshell.py:2882] Step 7250 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002231
I0726 00:50:07.547487 139830849898368 interactiveshell.py:2882] Step 7300 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002227
I0726 00:50:20.006062 139830849898368 interactiveshell.py:2882] Step 7350 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002222
I0726 00:50:32.485266 139830849898368 interactiveshell.py:2882] Step 7400 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002218
I0726 00:50:44.964736 139830849898368 interactiveshell.py:2882] Step 7450 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002213
I0726 00:50:57.444120 139830849898368 interactiveshell.py:2882] Step 7500 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002209
I0726 00:51:09.945990 139830849898368 interactiveshell.py:2882] Step 7550 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002204
I0726 00:51:22.346810 139830849898368 interactiveshell.py:2882] Step 7600 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002200
I0726 00:51:34.868115 139830849898368 interactiveshell.py:2882] Step 7650 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002195
I0726 00:51:47.349719 139830849898368 interactiveshell.py:2882] Step 7700 | Loss: 0.0003 | Spent: 12.5 secs | LR: 0.002191
Reading ../data/wn18/test.txt
I0726 00:52:02.932290 139830849898368 interactiveshell.py:2882] MRR: 0.932| [email protected]: 0.949 | [email protected]: 0.942 | [email protected]: 0.920
I0726 00:52:02.940547 139830849898368 interactiveshell.py:2882] Best MRR: 0.932
Reading ../data/wn18/train.txt
I0726 00:52:45.389431 139830849898368 interactiveshell.py:2882] Step 7750 | Loss: 0.0002 | Spent: 58.0 secs | LR: 0.002186
I0726 00:52:57.869483 139830849898368 interactiveshell.py:2882] Step 7800 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002182
I0726 00:53:10.356826 139830849898368 interactiveshell.py:2882] Step 7850 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002177
I0726 00:53:22.828546 139830849898368 interactiveshell.py:2882] Step 7900 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002173
I0726 00:53:35.341006 139830849898368 interactiveshell.py:2882] Step 7950 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002169
I0726 00:53:47.779379 139830849898368 interactiveshell.py:2882] Step 8000 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002164
I0726 00:54:00.231698 139830849898368 interactiveshell.py:2882] Step 8050 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002160
I0726 00:54:12.643695 139830849898368 interactiveshell.py:2882] Step 8100 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002155
I0726 00:54:25.148658 139830849898368 interactiveshell.py:2882] Step 8150 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002151
I0726 00:54:37.747953 139830849898368 interactiveshell.py:2882] Step 8200 | Loss: 0.0002 | Spent: 12.6 secs | LR: 0.002147
I0726 00:54:50.238249 139830849898368 interactiveshell.py:2882] Step 8250 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002142
I0726 00:55:02.543337 139830849898368 interactiveshell.py:2882] Step 8300 | Loss: 0.0002 | Spent: 12.3 secs | LR: 0.002138
I0726 00:55:14.910876 139830849898368 interactiveshell.py:2882] Step 8350 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002133
I0726 00:55:27.341779 139830849898368 interactiveshell.py:2882] Step 8400 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002129
I0726 00:55:39.846895 139830849898368 interactiveshell.py:2882] Step 8450 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002125
I0726 00:55:52.282403 139830849898368 interactiveshell.py:2882] Step 8500 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002120
I0726 00:56:04.697173 139830849898368 interactiveshell.py:2882] Step 8550 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002116
I0726 00:56:17.085093 139830849898368 interactiveshell.py:2882] Step 8600 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.002112
I0726 00:56:29.568407 139830849898368 interactiveshell.py:2882] Step 8650 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002107
I0726 00:56:42.090277 139830849898368 interactiveshell.py:2882] Step 8700 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002103
I0726 00:56:54.542803 139830849898368 interactiveshell.py:2882] Step 8750 | Loss: 0.0002 | Spent: 12.5 secs | LR: 0.002099
I0726 00:57:07.088361 139830849898368 interactiveshell.py:2882] Step 8800 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002095
Reading ../data/wn18/test.txt
I0726 00:57:24.335604 139830849898368 interactiveshell.py:2882] MRR: 0.934| [email protected]: 0.949 | [email protected]: 0.943 | [email protected]: 0.923
I0726 00:57:24.339437 139830849898368 interactiveshell.py:2882] Best MRR: 0.934
Reading ../data/wn18/train.txt
I0726 00:58:05.345171 139830849898368 interactiveshell.py:2882] Step 8850 | Loss: 0.0002 | Spent: 58.3 secs | LR: 0.002090
I0726 00:58:17.785409 139830849898368 interactiveshell.py:2882] Step 8900 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.002086
I0726 00:58:30.307535 139830849898368 interactiveshell.py:2882] Step 8950 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002082
I0726 00:58:42.709766 139830849898368 interactiveshell.py:2882] Step 9000 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.002078
I0726 00:58:55.167193 139830849898368 interactiveshell.py:2882] Step 9050 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002073
I0726 00:59:07.641722 139830849898368 interactiveshell.py:2882] Step 9100 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002069
I0726 00:59:20.114513 139830849898368 interactiveshell.py:2882] Step 9150 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002065
I0726 00:59:32.596481 139830849898368 interactiveshell.py:2882] Step 9200 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002061
I0726 00:59:45.114586 139830849898368 interactiveshell.py:2882] Step 9250 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002057
I0726 00:59:57.740283 139830849898368 interactiveshell.py:2882] Step 9300 | Loss: 0.0001 | Spent: 12.6 secs | LR: 0.002052
I0726 01:00:10.206969 139830849898368 interactiveshell.py:2882] Step 9350 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002048
I0726 01:00:22.682372 139830849898368 interactiveshell.py:2882] Step 9400 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002044
I0726 01:00:35.181816 139830849898368 interactiveshell.py:2882] Step 9450 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002040
I0726 01:00:47.733799 139830849898368 interactiveshell.py:2882] Step 9500 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002036
I0726 01:01:00.251925 139830849898368 interactiveshell.py:2882] Step 9550 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002031
I0726 01:01:12.760015 139830849898368 interactiveshell.py:2882] Step 9600 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002027
I0726 01:01:25.254607 139830849898368 interactiveshell.py:2882] Step 9650 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002023
I0726 01:01:37.867125 139830849898368 interactiveshell.py:2882] Step 9700 | Loss: 0.0001 | Spent: 12.6 secs | LR: 0.002019
I0726 01:01:50.430602 139830849898368 interactiveshell.py:2882] Step 9750 | Loss: 0.0001 | Spent: 12.6 secs | LR: 0.002015
I0726 01:02:02.889183 139830849898368 interactiveshell.py:2882] Step 9800 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002011
I0726 01:02:15.413430 139830849898368 interactiveshell.py:2882] Step 9850 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002007
I0726 01:02:27.926951 139830849898368 interactiveshell.py:2882] Step 9900 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.002003
I0726 01:02:40.452672 139830849898368 interactiveshell.py:2882] Step 9950 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001999
Reading ../data/wn18/test.txt
I0726 01:02:46.627884 139830849898368 interactiveshell.py:2882] MRR: 0.937| [email protected]: 0.950 | [email protected]: 0.945 | [email protected]: 0.928
I0726 01:02:46.636787 139830849898368 interactiveshell.py:2882] Best MRR: 0.937
Reading ../data/wn18/train.txt
I0726 01:03:38.483527 139830849898368 interactiveshell.py:2882] Step 10000 | Loss: 0.0001 | Spent: 58.0 secs | LR: 0.001994
I0726 01:03:51.022470 139830849898368 interactiveshell.py:2882] Step 10050 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001990
I0726 01:04:03.459896 139830849898368 interactiveshell.py:2882] Step 10100 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001986
I0726 01:04:15.823320 139830849898368 interactiveshell.py:2882] Step 10150 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001982
I0726 01:04:28.286228 139830849898368 interactiveshell.py:2882] Step 10200 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001978
I0726 01:04:40.698867 139830849898368 interactiveshell.py:2882] Step 10250 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001974
I0726 01:04:53.171001 139830849898368 interactiveshell.py:2882] Step 10300 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001970
I0726 01:05:05.826331 139830849898368 interactiveshell.py:2882] Step 10350 | Loss: 0.0001 | Spent: 12.7 secs | LR: 0.001966
I0726 01:05:18.209717 139830849898368 interactiveshell.py:2882] Step 10400 | Loss: 0.0002 | Spent: 12.4 secs | LR: 0.001962
I0726 01:05:30.688757 139830849898368 interactiveshell.py:2882] Step 10450 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001958
I0726 01:05:43.178032 139830849898368 interactiveshell.py:2882] Step 10500 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001954
I0726 01:05:55.697512 139830849898368 interactiveshell.py:2882] Step 10550 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001950
I0726 01:06:08.165353 139830849898368 interactiveshell.py:2882] Step 10600 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001946
I0726 01:06:20.558545 139830849898368 interactiveshell.py:2882] Step 10650 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001942
I0726 01:06:32.970876 139830849898368 interactiveshell.py:2882] Step 10700 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001938
I0726 01:06:45.419219 139830849898368 interactiveshell.py:2882] Step 10750 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001934
I0726 01:06:57.878738 139830849898368 interactiveshell.py:2882] Step 10800 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001930
I0726 01:07:10.302548 139830849898368 interactiveshell.py:2882] Step 10850 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001926
I0726 01:07:22.674716 139830849898368 interactiveshell.py:2882] Step 10900 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001923
I0726 01:07:35.071172 139830849898368 interactiveshell.py:2882] Step 10950 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001919
I0726 01:07:47.368353 139830849898368 interactiveshell.py:2882] Step 11000 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001915
I0726 01:07:59.782285 139830849898368 interactiveshell.py:2882] Step 11050 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001911
Reading ../data/wn18/test.txt
I0726 01:08:07.540784 139830849898368 interactiveshell.py:2882] MRR: 0.938| [email protected]: 0.950 | [email protected]: 0.946 | [email protected]: 0.929
I0726 01:08:07.548888 139830849898368 interactiveshell.py:2882] Best MRR: 0.938
Reading ../data/wn18/train.txt
I0726 01:08:57.248836 139830849898368 interactiveshell.py:2882] Step 11100 | Loss: 0.0001 | Spent: 57.5 secs | LR: 0.001907
I0726 01:09:09.762110 139830849898368 interactiveshell.py:2882] Step 11150 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001903
I0726 01:09:22.208655 139830849898368 interactiveshell.py:2882] Step 11200 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001899
I0726 01:09:34.678363 139830849898368 interactiveshell.py:2882] Step 11250 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001895
I0726 01:09:47.066646 139830849898368 interactiveshell.py:2882] Step 11300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001891
I0726 01:09:59.559002 139830849898368 interactiveshell.py:2882] Step 11350 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001888
I0726 01:10:12.235810 139830849898368 interactiveshell.py:2882] Step 11400 | Loss: 0.0001 | Spent: 12.7 secs | LR: 0.001884
I0726 01:10:24.695999 139830849898368 interactiveshell.py:2882] Step 11450 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001880
I0726 01:10:37.083269 139830849898368 interactiveshell.py:2882] Step 11500 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001876
I0726 01:10:49.359424 139830849898368 interactiveshell.py:2882] Step 11550 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001872
I0726 01:11:01.880425 139830849898368 interactiveshell.py:2882] Step 11600 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001868
I0726 01:11:14.231323 139830849898368 interactiveshell.py:2882] Step 11650 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001865
I0726 01:11:26.533878 139830849898368 interactiveshell.py:2882] Step 11700 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001861
I0726 01:11:38.820000 139830849898368 interactiveshell.py:2882] Step 11750 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001857
I0726 01:11:51.144244 139830849898368 interactiveshell.py:2882] Step 11800 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001853
I0726 01:12:03.501285 139830849898368 interactiveshell.py:2882] Step 11850 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001849
I0726 01:12:15.886878 139830849898368 interactiveshell.py:2882] Step 11900 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001846
I0726 01:12:28.333135 139830849898368 interactiveshell.py:2882] Step 11950 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001842
I0726 01:12:40.751386 139830849898368 interactiveshell.py:2882] Step 12000 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001838
I0726 01:12:53.143711 139830849898368 interactiveshell.py:2882] Step 12050 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001834
I0726 01:13:05.545046 139830849898368 interactiveshell.py:2882] Step 12100 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001831
I0726 01:13:17.962070 139830849898368 interactiveshell.py:2882] Step 12150 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001827
Reading ../data/wn18/test.txt
I0726 01:13:27.144026 139830849898368 interactiveshell.py:2882] MRR: 0.939| [email protected]: 0.950 | [email protected]: 0.947 | [email protected]: 0.930
I0726 01:13:27.147767 139830849898368 interactiveshell.py:2882] Best MRR: 0.939
Reading ../data/wn18/train.txt
I0726 01:14:14.934577 139830849898368 interactiveshell.py:2882] Step 12200 | Loss: 0.0001 | Spent: 57.0 secs | LR: 0.001823
I0726 01:14:27.336954 139830849898368 interactiveshell.py:2882] Step 12250 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001819
I0726 01:14:39.705554 139830849898368 interactiveshell.py:2882] Step 12300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001816
I0726 01:14:52.118105 139830849898368 interactiveshell.py:2882] Step 12350 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001812
I0726 01:15:04.480068 139830849898368 interactiveshell.py:2882] Step 12400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001808
I0726 01:15:16.905614 139830849898368 interactiveshell.py:2882] Step 12450 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001805
I0726 01:15:29.517977 139830849898368 interactiveshell.py:2882] Step 12500 | Loss: 0.0001 | Spent: 12.6 secs | LR: 0.001801
I0726 01:15:41.929823 139830849898368 interactiveshell.py:2882] Step 12550 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001797
I0726 01:15:54.305085 139830849898368 interactiveshell.py:2882] Step 12600 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001794
I0726 01:16:06.797850 139830849898368 interactiveshell.py:2882] Step 12650 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001790
I0726 01:16:19.062616 139830849898368 interactiveshell.py:2882] Step 12700 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001786
I0726 01:16:31.356074 139830849898368 interactiveshell.py:2882] Step 12750 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001783
I0726 01:16:43.624044 139830849898368 interactiveshell.py:2882] Step 12800 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001779
I0726 01:16:55.897016 139830849898368 interactiveshell.py:2882] Step 12850 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001775
I0726 01:17:08.195725 139830849898368 interactiveshell.py:2882] Step 12900 | Loss: 0.0001 | Spent: 12.3 secs | LR: 0.001772
I0726 01:17:20.435055 139830849898368 interactiveshell.py:2882] Step 12950 | Loss: 0.0001 | Spent: 12.2 secs | LR: 0.001768
I0726 01:17:32.890059 139830849898368 interactiveshell.py:2882] Step 13000 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001765
I0726 01:17:45.296319 139830849898368 interactiveshell.py:2882] Step 13050 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001761
I0726 01:17:57.724352 139830849898368 interactiveshell.py:2882] Step 13100 | Loss: 0.0001 | Spent: 12.4 secs | LR: 0.001757
I0726 01:18:10.169596 139830849898368 interactiveshell.py:2882] Step 13150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001754
I0726 01:18:22.631194 139830849898368 interactiveshell.py:2882] Step 13200 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001750
I0726 01:18:35.108068 139830849898368 interactiveshell.py:2882] Step 13250 | Loss: 0.0001 | Spent: 12.5 secs | LR: 0.001747
Reading ../data/wn18/test.txt
I0726 01:18:45.754950 139830849898368 interactiveshell.py:2882] MRR: 0.940| [email protected]: 0.951 | [email protected]: 0.947 | [email protected]: 0.932
I0726 01:18:45.763474 139830849898368 interactiveshell.py:2882] Best MRR: 0.940
Reading ../data/wn18/train.txt
I0726 01:19:32.419957 139830849898368 interactiveshell.py:2882] Step 13300 | Loss: 0.0000 | Spent: 57.3 secs | LR: 0.001743
I0726 01:19:44.840202 139830849898368 interactiveshell.py:2882] Step 13350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001740
I0726 01:19:57.237730 139830849898368 interactiveshell.py:2882] Step 13400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001736
I0726 01:20:09.552878 139830849898368 interactiveshell.py:2882] Step 13450 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001732
I0726 01:20:21.897521 139830849898368 interactiveshell.py:2882] Step 13500 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001729
I0726 01:20:34.589808 139830849898368 interactiveshell.py:2882] Step 13550 | Loss: 0.0000 | Spent: 12.7 secs | LR: 0.001725
I0726 01:20:47.021683 139830849898368 interactiveshell.py:2882] Step 13600 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001722
I0726 01:20:59.477566 139830849898368 interactiveshell.py:2882] Step 13650 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001718
I0726 01:21:12.004158 139830849898368 interactiveshell.py:2882] Step 13700 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001715
I0726 01:21:24.331457 139830849898368 interactiveshell.py:2882] Step 13750 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001711
I0726 01:21:36.752445 139830849898368 interactiveshell.py:2882] Step 13800 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001708
I0726 01:21:49.156939 139830849898368 interactiveshell.py:2882] Step 13850 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001704
I0726 01:22:01.584831 139830849898368 interactiveshell.py:2882] Step 13900 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001701
I0726 01:22:13.994924 139830849898368 interactiveshell.py:2882] Step 13950 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001697
I0726 01:22:26.395888 139830849898368 interactiveshell.py:2882] Step 14000 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001694
I0726 01:22:38.861906 139830849898368 interactiveshell.py:2882] Step 14050 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001691
I0726 01:22:51.375340 139830849898368 interactiveshell.py:2882] Step 14100 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001687
I0726 01:23:03.838430 139830849898368 interactiveshell.py:2882] Step 14150 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001684
I0726 01:23:16.324431 139830849898368 interactiveshell.py:2882] Step 14200 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001680
I0726 01:23:28.803190 139830849898368 interactiveshell.py:2882] Step 14250 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001677
I0726 01:23:41.231396 139830849898368 interactiveshell.py:2882] Step 14300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001673
I0726 01:23:53.681960 139830849898368 interactiveshell.py:2882] Step 14350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001670
Reading ../data/wn18/test.txt
I0726 01:24:06.085587 139830849898368 interactiveshell.py:2882] MRR: 0.941| [email protected]: 0.950 | [email protected]: 0.948 | [email protected]: 0.933
I0726 01:24:06.091294 139830849898368 interactiveshell.py:2882] Best MRR: 0.941
Reading ../data/wn18/train.txt
I0726 01:24:51.630141 139830849898368 interactiveshell.py:2882] Step 14400 | Loss: 0.0000 | Spent: 57.9 secs | LR: 0.001667
I0726 01:25:04.168757 139830849898368 interactiveshell.py:2882] Step 14450 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001663
I0726 01:25:16.667810 139830849898368 interactiveshell.py:2882] Step 14500 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001660
I0726 01:25:29.143358 139830849898368 interactiveshell.py:2882] Step 14550 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001656
I0726 01:25:41.813661 139830849898368 interactiveshell.py:2882] Step 14600 | Loss: 0.0000 | Spent: 12.7 secs | LR: 0.001653
I0726 01:25:54.430075 139830849898368 interactiveshell.py:2882] Step 14650 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001650
I0726 01:26:07.008574 139830849898368 interactiveshell.py:2882] Step 14700 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001646
I0726 01:26:19.559000 139830849898368 interactiveshell.py:2882] Step 14750 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001643
I0726 01:26:32.014799 139830849898368 interactiveshell.py:2882] Step 14800 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001640
I0726 01:26:44.596860 139830849898368 interactiveshell.py:2882] Step 14850 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001636
I0726 01:26:57.091743 139830849898368 interactiveshell.py:2882] Step 14900 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001633
I0726 01:27:09.578064 139830849898368 interactiveshell.py:2882] Step 14950 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001630
I0726 01:27:22.061287 139830849898368 interactiveshell.py:2882] Step 15000 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001626
I0726 01:27:34.578789 139830849898368 interactiveshell.py:2882] Step 15050 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001623
I0726 01:27:47.009426 139830849898368 interactiveshell.py:2882] Step 15100 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001620
I0726 01:27:59.444378 139830849898368 interactiveshell.py:2882] Step 15150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001616
I0726 01:28:11.867072 139830849898368 interactiveshell.py:2882] Step 15200 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001613
I0726 01:28:24.270075 139830849898368 interactiveshell.py:2882] Step 15250 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001610
I0726 01:28:36.710439 139830849898368 interactiveshell.py:2882] Step 15300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001606
I0726 01:28:49.234961 139830849898368 interactiveshell.py:2882] Step 15350 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001603
I0726 01:29:01.690078 139830849898368 interactiveshell.py:2882] Step 15400 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001600
I0726 01:29:14.098044 139830849898368 interactiveshell.py:2882] Step 15450 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001597
Reading ../data/wn18/test.txt
I0726 01:29:27.693936 139830849898368 interactiveshell.py:2882] MRR: 0.942| [email protected]: 0.950 | [email protected]: 0.947 | [email protected]: 0.936
I0726 01:29:27.699543 139830849898368 interactiveshell.py:2882] Best MRR: 0.942
Reading ../data/wn18/train.txt
I0726 01:30:11.673147 139830849898368 interactiveshell.py:2882] Step 15500 | Loss: 0.0000 | Spent: 57.6 secs | LR: 0.001593
I0726 01:30:23.927935 139830849898368 interactiveshell.py:2882] Step 15550 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001590
I0726 01:30:36.318478 139830849898368 interactiveshell.py:2882] Step 15600 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001587
I0726 01:30:48.780548 139830849898368 interactiveshell.py:2882] Step 15650 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001584
I0726 01:31:01.396547 139830849898368 interactiveshell.py:2882] Step 15700 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001580
I0726 01:31:13.787827 139830849898368 interactiveshell.py:2882] Step 15750 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001577
I0726 01:31:26.310373 139830849898368 interactiveshell.py:2882] Step 15800 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001574
I0726 01:31:38.674285 139830849898368 interactiveshell.py:2882] Step 15850 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001571
I0726 01:31:51.099660 139830849898368 interactiveshell.py:2882] Step 15900 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001568
I0726 01:32:03.556997 139830849898368 interactiveshell.py:2882] Step 15950 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001564
I0726 01:32:16.001345 139830849898368 interactiveshell.py:2882] Step 16000 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001561
I0726 01:32:28.427618 139830849898368 interactiveshell.py:2882] Step 16050 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001558
I0726 01:32:40.877456 139830849898368 interactiveshell.py:2882] Step 16100 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001555
I0726 01:32:53.288670 139830849898368 interactiveshell.py:2882] Step 16150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001552
I0726 01:33:05.720242 139830849898368 interactiveshell.py:2882] Step 16200 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001549
I0726 01:33:18.129845 139830849898368 interactiveshell.py:2882] Step 16250 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001545
I0726 01:33:30.556677 139830849898368 interactiveshell.py:2882] Step 16300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001542
I0726 01:33:42.955990 139830849898368 interactiveshell.py:2882] Step 16350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001539
I0726 01:33:55.344896 139830849898368 interactiveshell.py:2882] Step 16400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001536
I0726 01:34:07.782530 139830849898368 interactiveshell.py:2882] Step 16450 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001533
I0726 01:34:20.144982 139830849898368 interactiveshell.py:2882] Step 16500 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001530
I0726 01:34:32.609336 139830849898368 interactiveshell.py:2882] Step 16550 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001527
Reading ../data/wn18/test.txt
I0726 01:34:47.740786 139830849898368 interactiveshell.py:2882] MRR: 0.941| [email protected]: 0.950 | [email protected]: 0.946 | [email protected]: 0.934
I0726 01:34:47.748382 139830849898368 interactiveshell.py:2882] Best MRR: 0.942
Reading ../data/wn18/train.txt
I0726 01:35:30.195164 139830849898368 interactiveshell.py:2882] Step 16600 | Loss: 0.0000 | Spent: 57.6 secs | LR: 0.001523
I0726 01:35:42.603405 139830849898368 interactiveshell.py:2882] Step 16650 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001520
I0726 01:35:54.958020 139830849898368 interactiveshell.py:2882] Step 16700 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001517
I0726 01:36:07.632351 139830849898368 interactiveshell.py:2882] Step 16750 | Loss: 0.0000 | Spent: 12.7 secs | LR: 0.001514
I0726 01:36:19.990075 139830849898368 interactiveshell.py:2882] Step 16800 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001511
I0726 01:36:32.569251 139830849898368 interactiveshell.py:2882] Step 16850 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001508
I0726 01:36:44.994293 139830849898368 interactiveshell.py:2882] Step 16900 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001505
I0726 01:36:57.389389 139830849898368 interactiveshell.py:2882] Step 16950 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001502
I0726 01:37:09.799444 139830849898368 interactiveshell.py:2882] Step 17000 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001499
I0726 01:37:22.201654 139830849898368 interactiveshell.py:2882] Step 17050 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001496
I0726 01:37:34.700779 139830849898368 interactiveshell.py:2882] Step 17100 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001493
I0726 01:37:47.079824 139830849898368 interactiveshell.py:2882] Step 17150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001490
I0726 01:37:59.472947 139830849898368 interactiveshell.py:2882] Step 17200 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001487
I0726 01:38:11.915735 139830849898368 interactiveshell.py:2882] Step 17250 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001484
I0726 01:38:24.275284 139830849898368 interactiveshell.py:2882] Step 17300 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001481
I0726 01:38:36.683388 139830849898368 interactiveshell.py:2882] Step 17350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001477
I0726 01:38:49.092177 139830849898368 interactiveshell.py:2882] Step 17400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001474
I0726 01:39:01.456367 139830849898368 interactiveshell.py:2882] Step 17450 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001471
I0726 01:39:13.820870 139830849898368 interactiveshell.py:2882] Step 17500 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001468
I0726 01:39:26.223116 139830849898368 interactiveshell.py:2882] Step 17550 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001465
I0726 01:39:38.652700 139830849898368 interactiveshell.py:2882] Step 17600 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001462
I0726 01:39:51.048085 139830849898368 interactiveshell.py:2882] Step 17650 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001460
Reading ../data/wn18/test.txt
I0726 01:40:07.489699 139830849898368 interactiveshell.py:2882] MRR: 0.941| [email protected]: 0.950 | [email protected]: 0.946 | [email protected]: 0.935
I0726 01:40:07.497766 139830849898368 interactiveshell.py:2882] Best MRR: 0.942
Reading ../data/wn18/train.txt
I0726 01:40:48.129178 139830849898368 interactiveshell.py:2882] Step 17700 | Loss: 0.0000 | Spent: 57.1 secs | LR: 0.001457
I0726 01:41:00.568784 139830849898368 interactiveshell.py:2882] Step 17750 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001454
I0726 01:41:13.107477 139830849898368 interactiveshell.py:2882] Step 17800 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001451
I0726 01:41:25.617860 139830849898368 interactiveshell.py:2882] Step 17850 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001448
I0726 01:41:38.181080 139830849898368 interactiveshell.py:2882] Step 17900 | Loss: 0.0000 | Spent: 12.6 secs | LR: 0.001445
I0726 01:41:50.550322 139830849898368 interactiveshell.py:2882] Step 17950 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001442
I0726 01:42:02.963787 139830849898368 interactiveshell.py:2882] Step 18000 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001439
I0726 01:42:15.359393 139830849898368 interactiveshell.py:2882] Step 18050 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001436
I0726 01:42:27.787104 139830849898368 interactiveshell.py:2882] Step 18100 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001433
I0726 01:42:40.178790 139830849898368 interactiveshell.py:2882] Step 18150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001430
I0726 01:42:52.575128 139830849898368 interactiveshell.py:2882] Step 18200 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001427
I0726 01:43:04.959094 139830849898368 interactiveshell.py:2882] Step 18250 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001424
I0726 01:43:17.287070 139830849898368 interactiveshell.py:2882] Step 18300 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001421
I0726 01:43:29.669784 139830849898368 interactiveshell.py:2882] Step 18350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001418
I0726 01:43:42.084837 139830849898368 interactiveshell.py:2882] Step 18400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001416
I0726 01:43:54.452327 139830849898368 interactiveshell.py:2882] Step 18450 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001413
I0726 01:44:06.841444 139830849898368 interactiveshell.py:2882] Step 18500 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001410
I0726 01:44:19.196258 139830849898368 interactiveshell.py:2882] Step 18550 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001407
I0726 01:44:31.487292 139830849898368 interactiveshell.py:2882] Step 18600 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001404
I0726 01:44:43.904318 139830849898368 interactiveshell.py:2882] Step 18650 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001401
I0726 01:44:56.280069 139830849898368 interactiveshell.py:2882] Step 18700 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001398
I0726 01:45:08.636086 139830849898368 interactiveshell.py:2882] Step 18750 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001395
I0726 01:45:20.943012 139830849898368 interactiveshell.py:2882] Step 18800 | Loss: 0.0000 | Spent: 12.3 secs | LR: 0.001393
Reading ../data/wn18/test.txt
I0726 01:45:26.435588 139830849898368 interactiveshell.py:2882] MRR: 0.941| [email protected]: 0.949 | [email protected]: 0.946 | [email protected]: 0.935
I0726 01:45:26.439271 139830849898368 interactiveshell.py:2882] Best MRR: 0.942
Reading ../data/wn18/train.txt
I0726 01:46:17.608955 139830849898368 interactiveshell.py:2882] Step 18850 | Loss: 0.0000 | Spent: 56.7 secs | LR: 0.001390
I0726 01:46:30.350728 139830849898368 interactiveshell.py:2882] Step 18900 | Loss: 0.0000 | Spent: 12.7 secs | LR: 0.001387
I0726 01:46:42.877482 139830849898368 interactiveshell.py:2882] Step 18950 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001384
I0726 01:46:55.325151 139830849898368 interactiveshell.py:2882] Step 19000 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001381
I0726 01:47:07.772522 139830849898368 interactiveshell.py:2882] Step 19050 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001378
I0726 01:47:20.204020 139830849898368 interactiveshell.py:2882] Step 19100 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001376
I0726 01:47:32.615381 139830849898368 interactiveshell.py:2882] Step 19150 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001373
I0726 01:47:45.129026 139830849898368 interactiveshell.py:2882] Step 19200 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001370
I0726 01:47:57.558561 139830849898368 interactiveshell.py:2882] Step 19250 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001367
I0726 01:48:10.033493 139830849898368 interactiveshell.py:2882] Step 19300 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001364
I0726 01:48:22.484457 139830849898368 interactiveshell.py:2882] Step 19350 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001362
I0726 01:48:34.921515 139830849898368 interactiveshell.py:2882] Step 19400 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001359
I0726 01:48:47.288103 139830849898368 interactiveshell.py:2882] Step 19450 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001356
I0726 01:48:59.641593 139830849898368 interactiveshell.py:2882] Step 19500 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001353
I0726 01:49:12.178711 139830849898368 interactiveshell.py:2882] Step 19550 | Loss: 0.0000 | Spent: 12.5 secs | LR: 0.001351
I0726 01:49:24.599696 139830849898368 interactiveshell.py:2882] Step 19600 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001348
I0726 01:49:36.994649 139830849898368 interactiveshell.py:2882] Step 19650 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001345
I0726 01:49:49.396643 139830849898368 interactiveshell.py:2882] Step 19700 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001342
I0726 01:50:01.814740 139830849898368 interactiveshell.py:2882] Step 19750 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001340
I0726 01:50:14.236103 139830849898368 interactiveshell.py:2882] Step 19800 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001337
I0726 01:50:26.610101 139830849898368 interactiveshell.py:2882] Step 19850 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001334
I0726 01:50:38.981051 139830849898368 interactiveshell.py:2882] Step 19900 | Loss: 0.0000 | Spent: 12.4 secs | LR: 0.001331
Reading ../data/wn18/test.txt
I0726 01:50:46.075441 139830849898368 interactiveshell.py:2882] MRR: 0.941| [email protected]: 0.950 | [email protected]: 0.946 | [email protected]: 0.935
I0726 01:50:46.082938 139830849898368 interactiveshell.py:2882] Best MRR: 0.942
Reading ../data/wn18/train.txt
I0726 01:51:36.469737 139830849898368 interactiveshell.py:2882] Step 19950 | Loss: 0.0000 | Spent: 57.5 secs | LR: 0.001329
---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-12-41552e1219ab> in <module>()
     28     optim.lr.assign(decay_lr(global_step))
     29     grads = tape.gradient(loss, model.trainable_variables)
---> 30     optim.apply_gradients(zip(grads, model.trainable_variables))
     31 
     32     if global_step % 50 == 0:

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py in apply_gradients(self, grads_and_vars, name)
    440           self._distributed_apply,
    441           args=(grads_and_vars,),
--> 442           kwargs={"name": name})
    443 
    444   def _distributed_apply(self, distribution, grads_and_vars, name):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py in merge_call(self, merge_fn, args, kwargs)
   1837     if kwargs is None:
   1838       kwargs = {}
-> 1839     return self._merge_call(merge_fn, args, kwargs)
   1840 
   1841   def _merge_call(self, merge_fn, args, kwargs):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py in _merge_call(self, merge_fn, args, kwargs)
   1844         distribution_strategy_context._CrossReplicaThreadMode(self._strategy))  # pylint: disable=protected-access
   1845     try:
-> 1846       return merge_fn(self._strategy, *args, **kwargs)
   1847     finally:
   1848       _pop_per_thread_mode()

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py in _distributed_apply(self, distribution, grads_and_vars, name)
    474           update_ops.extend(
    475               distribution.extended.update(
--> 476                   var, apply_grad_to_update_var, args=(grad,), group=False))
    477 
    478       any_symbolic = any(isinstance(i, ops.Operation) or

/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py in update(self, var, fn, args, kwargs, group)
   1456       kwargs = {}
   1457     with self._container_strategy().scope():
-> 1458       return self._update(var, fn, args, kwargs, group)
   1459 
   1460   def _update(self, var, fn, args, kwargs, group):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py in _update(self, var, fn, args, kwargs, group)
   2011     # The implementations of _update() and _update_non_slot() are identical
   2012     # except _update() passes `var` as the first argument to `fn()`.
-> 2013     return self._update_non_slot(var, fn, (var,) + tuple(args), kwargs, group)
   2014 
   2015   def _update_non_slot(self, colocate_with, fn, args, kwargs, should_group):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/distribute/distribute_lib.py in _update_non_slot(self, colocate_with, fn, args, kwargs, should_group)
   2017     # once that value is used for something.
   2018     with UpdateContext(colocate_with):
-> 2019       result = fn(*args, **kwargs)
   2020       if should_group:
   2021         return result

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py in apply_grad_to_update_var(var, grad)
    459         return self._resource_apply_sparse_duplicate_indices(
    460             grad.values, var, grad.indices)
--> 461       update_op = self._resource_apply_dense(grad, var)
    462       if var.constraint is not None:
    463         with ops.control_dependencies([update_op]):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/optimizer_v2/adam.py in _resource_apply_dense(self, grad, var)
    183           epsilon_t,
    184           grad,
--> 185           use_locking=self._use_locking)
    186     else:
    187       vhat = self.get_slot(var, 'vhat')

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/gen_training_ops.py in resource_apply_adam(var, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad, use_locking, use_nesterov, name)
   1369         "ResourceApplyAdam", name, _ctx._post_execution_callbacks, var, m, v,
   1370         beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
-> 1371         "use_locking", use_locking, "use_nesterov", use_nesterov)
   1372       return _result
   1373     except _core._FallbackException:

KeyboardInterrupt: