In [1]:
"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need these
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/knowledge_graph_completion/wn18/main')
Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
In [2]:
!pip install tensorflow-gpu==2.0.0-alpha0 
Requirement already satisfied: tensorflow-gpu==2.0.0-alpha0 in /usr/local/lib/python3.6/dist-packages (2.0.0a0)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)
Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)
Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)
Requirement already satisfied: tb-nightly<1.14.0a20190302,>=1.14.0a20190301 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0a20190301)
Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)
Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)
Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)
Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.11.0)
Requirement already satisfied: tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.0.dev2019030115)
Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.2)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)
Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)
Requirement already satisfied: google-pasta>=0.1.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.1.5)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (40.9.0)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)
Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)
In [3]:
import tensorflow as tf
import pprint
import logging
import time

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())
TensorFlow Version 2.0.0-alpha0
GPU Enabled: True
In [0]:
def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx
In [0]:
"""
we use 1vN fast evaluation as purposed in ConvE paper:
"https://arxiv.org/abs/1707.01476"
sp2o is a dictionary that maps a pair of <subject, predicate>
to multiple possible corresponding <objects> in graph
"""
def make_sp2o(f_paths, e2idx, r2idx):
    sp2o = {}
    for f_path in f_paths:
      with open(f_path) as f:
        for line in f:
            line = line.rstrip()
            s, p, o = line.split()
            s, p, o = e2idx[s], r2idx[p], e2idx[o]
            if (s,p) not in sp2o:
                sp2o[(s,p)] = [o]
            else:
                if o not in sp2o[(s,p)]:
                    sp2o[(s,p)].append(o)
    return sp2o
In [0]:
def map_fn(x, y):
  i, v, s = y[0]
  one_hot = tf.SparseTensor(i, v, s)
  return x, (one_hot, y[1], y[2])


# stream data from text files
def data_generator(f_path, params, sp2o):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      line = line.rstrip()
      s, p, o = line.split()
      s, p, o = params['e2idx'][s], params['r2idx'][p], params['e2idx'][o]
      sparse_i = [[x] for x in sp2o[(s, p)]]
      sparse_v = [1.] * len(sparse_i)
      sparse_s = [len(params['e2idx'])]
      yield ((s, p), ((sparse_i, sparse_v, sparse_s), o, len(sparse_i)))


def dataset(is_training, params, sp2o):
  _shapes = (([], []), (([None, 1], [None], [1]), [], []))
  _types = ((tf.int32, tf.int32),
            ((tf.int64, tf.float32, tf.int64), tf.int32, tf.int32))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['num_samples'])
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params, sp2o),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.map(map_fn)
    ds = ds.batch(params['batch_size'])
  
  return ds
In [0]:
def update_metrics(scores, query, metrics):
  to_float = lambda x: tf.cast(x, tf.float32)
  
  _, i = tf.math.top_k(scores, sorted=True, k=scores.shape[1])
  query = tf.expand_dims(query, 1)
  is_query = to_float(tf.equal(i, query))
  r = tf.argmax(is_query, -1) + 1
  
  mrr = 1. / to_float(r)
  hits_10 = to_float(tf.less_equal(r, 10))
  hits_3 = to_float(tf.less_equal(r, 3))
  hits_1 = to_float(tf.less_equal(r, 1))
  
  metrics['mrr'].update_state(mrr)
  metrics['hits_10'].update_state(hits_10)
  metrics['hits_3'].update_state(hits_3)
  metrics['hits_1'].update_state(hits_1)
In [0]:
class DistMult(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embed_ent = tf.keras.layers.Embedding(input_dim=len(params['e2idx']),
                                               output_dim=params['embed_dim'],
                                               embeddings_initializer=tf.initializers.RandomUniform(),
                                               name='Entity')
    self.embed_rel = tf.keras.layers.Embedding(input_dim=len(params['r2idx']),
                                               output_dim=params['embed_dim'],
                                               embeddings_initializer=tf.initializers.RandomUniform(),
                                               name='Relation')
  
  def call(self, inputs):
    s, p = inputs
    s = self.embed_ent(s)
    p = self.embed_rel(p)
    x = tf.matmul(s * p, self.embed_ent.embeddings, transpose_b=True)
    return x
In [0]:
params = {
    'train_path': '../data/wn18/train.txt',
    'valid_path': '../data/wn18/valid.txt',
    'test_path': '../data/wn18/test.txt',
    'entity_path': '../vocab/entity.txt',
    'relation_path': '../vocab/relation.txt',
    'batch_size': 128,
    'embed_dim': 200,
    'num_samples': 141442,
    'lr': 3e-3,
    'num_patience': 3,
}
In [0]:
params['e2idx']  = get_vocab(params['entity_path'])
params['r2idx'] = get_vocab(params['relation_path'])
sp2o_tr = make_sp2o([params['train_path']], params['e2idx'], params['r2idx'])
sp2o_all = make_sp2o([params['train_path'],
                      params['test_path'],
                      params['valid_path']], params['e2idx'], params['r2idx'])
In [0]:
def is_descending(history: list):
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True 
In [12]:
model = DistMult(params)
model.build(input_shape=[[None], [None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.96)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

best_mrr = 0.
history_mrr = []

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.setLevel(logging.INFO)


while True:
  # TRAINING
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=True, params=params, sp2o=sp2o_tr):
    with tf.GradientTape() as tape:
      logits = model((s, p))
      multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
      num_neg = len(params['e2idx']) - num_pos
      pos_weight = tf.expand_dims(tf.cast(num_neg/num_pos, tf.float32), 1)
      loss = tf.nn.weighted_cross_entropy_with_logits(targets=multi_o, logits=logits, pos_weight=pos_weight)
      loss = tf.reduce_mean(loss)
    
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, model.trainable_variables)
    optim.apply_gradients(zip(grads, model.trainable_variables))

    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
          global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    global_step += 1
  
  # EVALUATION
  metrics = {
    'mrr': tf.metrics.Mean(),
    'hits_10': tf.metrics.Mean(),
    'hits_3': tf.metrics.Mean(),
    'hits_1': tf.metrics.Mean(),
  }
  for ((s, p), (multi_o, o, num_pos)) in dataset(is_training=False, params=params, sp2o=sp2o_all):
    logits = model((s, p))
    multi_o = tf.sparse.to_dense(multi_o, validate_indices=False)
    # create masks for Filtered MRR
    o_one_hot = tf.one_hot(o, len(params['e2idx']))
    unwanted = multi_o - o_one_hot
    masks = tf.cast(tf.equal(unwanted, 0.), tf.float32)
    scores = tf.sigmoid(logits) * masks
    
    update_metrics(scores=scores, query=o, metrics=metrics)
  
  logger.info("MRR: {:.3f}| [email protected]: {:.3f} | [email protected]: {:.3f} | [email protected]: {:.3f}".format(
    metrics['mrr'].result().numpy(),
    metrics['hits_10'].result().numpy(),
    metrics['hits_3'].result().numpy(),
    metrics['hits_1'].result().numpy()))
  
  mrr = metrics['mrr'].result().numpy()
  history_mrr.append(mrr)
  
  if mrr > best_mrr:
    best_mrr = mrr
    # you can save model here
  logger.info("Best MRR: {:.3f}".format(best_mrr))
  
  if len(history_mrr) > params['num_patience'] and is_descending(history_mrr):
    logger.info("MRR not improved over {} epochs, Early Stop".format(params['num_patience']))
    break
WARNING: Logging before flag parsing goes to stderr.
W0419 02:45:29.234123 140363059820416 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py:410: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.
Instructions for updating:
tf.py_func is deprecated in TF V2. Instead, there are two
    options available in V2.
    - tf.py_function takes a python function which manipulates tf eager
    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to
    an ndarray (just call tensor.numpy()) but having access to eager tensors
    means `tf.py_function`s can use accelerators such as GPUs as well as
    being differentiable using a gradient tape.
    - tf.numpy_function maintains the semantics of the deprecated tf.py_func
    (it is not differentiable, and manipulates numpy arrays). It drops the
    stateful argument making all functions stateful.
    
[('Entity/embeddings:0', TensorShape([40943, 200])),
 ('Relation/embeddings:0', TensorShape([18, 200]))]
Reading ../data/wn18/train.txt
I0419 02:45:53.790063 140363059820416 interactiveshell.py:2882] Step 0 | Loss: 1.3855 | Spent: 24.6 secs | LR: 0.003000
I0419 02:45:55.924041 140363059820416 interactiveshell.py:2882] Step 50 | Loss: 1.3857 | Spent: 2.1 secs | LR: 0.002994
I0419 02:45:58.058234 140363059820416 interactiveshell.py:2882] Step 100 | Loss: 1.3845 | Spent: 2.1 secs | LR: 0.002988
I0419 02:46:00.185530 140363059820416 interactiveshell.py:2882] Step 150 | Loss: 1.3740 | Spent: 2.1 secs | LR: 0.002982
I0419 02:46:02.306592 140363059820416 interactiveshell.py:2882] Step 200 | Loss: 1.3110 | Spent: 2.1 secs | LR: 0.002976
I0419 02:46:04.441041 140363059820416 interactiveshell.py:2882] Step 250 | Loss: 1.2251 | Spent: 2.1 secs | LR: 0.002970
I0419 02:46:06.568732 140363059820416 interactiveshell.py:2882] Step 300 | Loss: 1.1427 | Spent: 2.1 secs | LR: 0.002963
I0419 02:46:08.724704 140363059820416 interactiveshell.py:2882] Step 350 | Loss: 1.0426 | Spent: 2.2 secs | LR: 0.002957
I0419 02:46:10.869930 140363059820416 interactiveshell.py:2882] Step 400 | Loss: 0.9648 | Spent: 2.1 secs | LR: 0.002951
I0419 02:46:13.017762 140363059820416 interactiveshell.py:2882] Step 450 | Loss: 1.0101 | Spent: 2.1 secs | LR: 0.002945
I0419 02:46:15.150521 140363059820416 interactiveshell.py:2882] Step 500 | Loss: 0.8092 | Spent: 2.1 secs | LR: 0.002939
I0419 02:46:17.301398 140363059820416 interactiveshell.py:2882] Step 550 | Loss: 0.7173 | Spent: 2.1 secs | LR: 0.002933
I0419 02:46:19.447122 140363059820416 interactiveshell.py:2882] Step 600 | Loss: 0.7011 | Spent: 2.1 secs | LR: 0.002927
I0419 02:46:21.585452 140363059820416 interactiveshell.py:2882] Step 650 | Loss: 0.7359 | Spent: 2.1 secs | LR: 0.002921
I0419 02:46:23.719987 140363059820416 interactiveshell.py:2882] Step 700 | Loss: 0.6895 | Spent: 2.1 secs | LR: 0.002915
I0419 02:46:25.868222 140363059820416 interactiveshell.py:2882] Step 750 | Loss: 0.5740 | Spent: 2.1 secs | LR: 0.002910
I0419 02:46:28.003318 140363059820416 interactiveshell.py:2882] Step 800 | Loss: 0.6470 | Spent: 2.1 secs | LR: 0.002904
I0419 02:46:30.146912 140363059820416 interactiveshell.py:2882] Step 850 | Loss: 0.6495 | Spent: 2.1 secs | LR: 0.002898
I0419 02:46:32.295840 140363059820416 interactiveshell.py:2882] Step 900 | Loss: 0.4021 | Spent: 2.1 secs | LR: 0.002892
I0419 02:46:34.440737 140363059820416 interactiveshell.py:2882] Step 950 | Loss: 0.5676 | Spent: 2.1 secs | LR: 0.002886
I0419 02:46:36.585713 140363059820416 interactiveshell.py:2882] Step 1000 | Loss: 0.4147 | Spent: 2.1 secs | LR: 0.002880
I0419 02:46:38.756368 140363059820416 interactiveshell.py:2882] Step 1050 | Loss: 0.4684 | Spent: 2.2 secs | LR: 0.002874
I0419 02:46:41.098708 140363059820416 interactiveshell.py:2882] Step 1100 | Loss: 0.3574 | Spent: 2.3 secs | LR: 0.002868
Reading ../data/wn18/test.txt
I0419 02:46:43.659612 140363059820416 interactiveshell.py:2882] MRR: 0.442| [email protected]: 0.702 | [email protected]: 0.519 | [email protected]: 0.313
I0419 02:46:43.661445 140363059820416 interactiveshell.py:2882] Best MRR: 0.442
Reading ../data/wn18/train.txt
I0419 02:47:13.397497 140363059820416 interactiveshell.py:2882] Step 1150 | Loss: 0.2037 | Spent: 32.3 secs | LR: 0.002862
I0419 02:47:15.514809 140363059820416 interactiveshell.py:2882] Step 1200 | Loss: 0.1545 | Spent: 2.1 secs | LR: 0.002857
I0419 02:47:17.639808 140363059820416 interactiveshell.py:2882] Step 1250 | Loss: 0.1337 | Spent: 2.1 secs | LR: 0.002851
I0419 02:47:19.779346 140363059820416 interactiveshell.py:2882] Step 1300 | Loss: 0.1231 | Spent: 2.1 secs | LR: 0.002845
I0419 02:47:21.912069 140363059820416 interactiveshell.py:2882] Step 1350 | Loss: 0.1442 | Spent: 2.1 secs | LR: 0.002839
I0419 02:47:24.037053 140363059820416 interactiveshell.py:2882] Step 1400 | Loss: 0.1231 | Spent: 2.1 secs | LR: 0.002833
I0419 02:47:26.162008 140363059820416 interactiveshell.py:2882] Step 1450 | Loss: 0.1732 | Spent: 2.1 secs | LR: 0.002828
I0419 02:47:28.295783 140363059820416 interactiveshell.py:2882] Step 1500 | Loss: 0.0875 | Spent: 2.1 secs | LR: 0.002822
I0419 02:47:30.424008 140363059820416 interactiveshell.py:2882] Step 1550 | Loss: 0.2169 | Spent: 2.1 secs | LR: 0.002816
I0419 02:47:32.559748 140363059820416 interactiveshell.py:2882] Step 1600 | Loss: 0.1545 | Spent: 2.1 secs | LR: 0.002810
I0419 02:47:34.685464 140363059820416 interactiveshell.py:2882] Step 1650 | Loss: 0.0682 | Spent: 2.1 secs | LR: 0.002805
I0419 02:47:36.812308 140363059820416 interactiveshell.py:2882] Step 1700 | Loss: 0.1072 | Spent: 2.1 secs | LR: 0.002799
I0419 02:47:38.969979 140363059820416 interactiveshell.py:2882] Step 1750 | Loss: 0.0998 | Spent: 2.2 secs | LR: 0.002793
I0419 02:47:41.103491 140363059820416 interactiveshell.py:2882] Step 1800 | Loss: 0.0542 | Spent: 2.1 secs | LR: 0.002787
I0419 02:47:43.244930 140363059820416 interactiveshell.py:2882] Step 1850 | Loss: 0.0641 | Spent: 2.1 secs | LR: 0.002782
I0419 02:47:45.383041 140363059820416 interactiveshell.py:2882] Step 1900 | Loss: 0.0569 | Spent: 2.1 secs | LR: 0.002776
I0419 02:47:47.515271 140363059820416 interactiveshell.py:2882] Step 1950 | Loss: 0.0466 | Spent: 2.1 secs | LR: 0.002770
I0419 02:47:49.652942 140363059820416 interactiveshell.py:2882] Step 2000 | Loss: 0.0406 | Spent: 2.1 secs | LR: 0.002765
I0419 02:47:51.785445 140363059820416 interactiveshell.py:2882] Step 2050 | Loss: 0.0570 | Spent: 2.1 secs | LR: 0.002759
I0419 02:47:53.930615 140363059820416 interactiveshell.py:2882] Step 2100 | Loss: 0.0511 | Spent: 2.1 secs | LR: 0.002754
I0419 02:47:56.070626 140363059820416 interactiveshell.py:2882] Step 2150 | Loss: 0.0358 | Spent: 2.1 secs | LR: 0.002748
I0419 02:47:58.220481 140363059820416 interactiveshell.py:2882] Step 2200 | Loss: 0.0319 | Spent: 2.1 secs | LR: 0.002742
Reading ../data/wn18/test.txt
I0419 02:48:01.006850 140363059820416 interactiveshell.py:2882] MRR: 0.699| [email protected]: 0.899 | [email protected]: 0.819 | [email protected]: 0.564
I0419 02:48:01.012896 140363059820416 interactiveshell.py:2882] Best MRR: 0.699
Reading ../data/wn18/train.txt
I0419 02:48:28.844053 140363059820416 interactiveshell.py:2882] Step 2250 | Loss: 0.0227 | Spent: 30.6 secs | LR: 0.002737
I0419 02:48:30.969196 140363059820416 interactiveshell.py:2882] Step 2300 | Loss: 0.0178 | Spent: 2.1 secs | LR: 0.002731
I0419 02:48:33.094399 140363059820416 interactiveshell.py:2882] Step 2350 | Loss: 0.0204 | Spent: 2.1 secs | LR: 0.002726
I0419 02:48:35.234266 140363059820416 interactiveshell.py:2882] Step 2400 | Loss: 0.0196 | Spent: 2.1 secs | LR: 0.002720
I0419 02:48:37.367260 140363059820416 interactiveshell.py:2882] Step 2450 | Loss: 0.0169 | Spent: 2.1 secs | LR: 0.002714
I0419 02:48:39.505978 140363059820416 interactiveshell.py:2882] Step 2500 | Loss: 0.0166 | Spent: 2.1 secs | LR: 0.002709
I0419 02:48:41.660887 140363059820416 interactiveshell.py:2882] Step 2550 | Loss: 0.0134 | Spent: 2.2 secs | LR: 0.002703
I0419 02:48:43.804085 140363059820416 interactiveshell.py:2882] Step 2600 | Loss: 0.0136 | Spent: 2.1 secs | LR: 0.002698
I0419 02:48:45.940457 140363059820416 interactiveshell.py:2882] Step 2650 | Loss: 0.0251 | Spent: 2.1 secs | LR: 0.002692
I0419 02:48:48.078678 140363059820416 interactiveshell.py:2882] Step 2700 | Loss: 0.0121 | Spent: 2.1 secs | LR: 0.002687
I0419 02:48:50.227930 140363059820416 interactiveshell.py:2882] Step 2750 | Loss: 0.0135 | Spent: 2.1 secs | LR: 0.002681
I0419 02:48:52.351986 140363059820416 interactiveshell.py:2882] Step 2800 | Loss: 0.0114 | Spent: 2.1 secs | LR: 0.002676
I0419 02:48:54.491408 140363059820416 interactiveshell.py:2882] Step 2850 | Loss: 0.0094 | Spent: 2.1 secs | LR: 0.002671
I0419 02:48:56.622768 140363059820416 interactiveshell.py:2882] Step 2900 | Loss: 0.0114 | Spent: 2.1 secs | LR: 0.002665
I0419 02:48:58.762818 140363059820416 interactiveshell.py:2882] Step 2950 | Loss: 0.0147 | Spent: 2.1 secs | LR: 0.002660
I0419 02:49:00.930675 140363059820416 interactiveshell.py:2882] Step 3000 | Loss: 0.0076 | Spent: 2.2 secs | LR: 0.002654
I0419 02:49:03.078073 140363059820416 interactiveshell.py:2882] Step 3050 | Loss: 0.0113 | Spent: 2.1 secs | LR: 0.002649
I0419 02:49:05.223946 140363059820416 interactiveshell.py:2882] Step 3100 | Loss: 0.0094 | Spent: 2.1 secs | LR: 0.002643
I0419 02:49:07.382345 140363059820416 interactiveshell.py:2882] Step 3150 | Loss: 0.0075 | Spent: 2.2 secs | LR: 0.002638
I0419 02:49:09.537650 140363059820416 interactiveshell.py:2882] Step 3200 | Loss: 0.0075 | Spent: 2.2 secs | LR: 0.002633
I0419 02:49:11.678789 140363059820416 interactiveshell.py:2882] Step 3250 | Loss: 0.0080 | Spent: 2.1 secs | LR: 0.002627
I0419 02:49:13.816834 140363059820416 interactiveshell.py:2882] Step 3300 | Loss: 0.0075 | Spent: 2.1 secs | LR: 0.002622
Reading ../data/wn18/test.txt
I0419 02:49:16.556100 140363059820416 interactiveshell.py:2882] MRR: 0.768| [email protected]: 0.930 | [email protected]: 0.875 | [email protected]: 0.655
I0419 02:49:16.560619 140363059820416 interactiveshell.py:2882] Best MRR: 0.768
Reading ../data/wn18/train.txt
I0419 02:49:45.059677 140363059820416 interactiveshell.py:2882] Step 3350 | Loss: 0.0061 | Spent: 31.2 secs | LR: 0.002617
I0419 02:49:47.203936 140363059820416 interactiveshell.py:2882] Step 3400 | Loss: 0.0059 | Spent: 2.1 secs | LR: 0.002611
I0419 02:49:49.359980 140363059820416 interactiveshell.py:2882] Step 3450 | Loss: 0.0054 | Spent: 2.2 secs | LR: 0.002606
I0419 02:49:51.499655 140363059820416 interactiveshell.py:2882] Step 3500 | Loss: 0.0074 | Spent: 2.1 secs | LR: 0.002601
I0419 02:49:53.649007 140363059820416 interactiveshell.py:2882] Step 3550 | Loss: 0.0050 | Spent: 2.1 secs | LR: 0.002595
I0419 02:49:55.790234 140363059820416 interactiveshell.py:2882] Step 3600 | Loss: 0.0055 | Spent: 2.1 secs | LR: 0.002590
I0419 02:49:57.946289 140363059820416 interactiveshell.py:2882] Step 3650 | Loss: 0.0045 | Spent: 2.2 secs | LR: 0.002585
I0419 02:50:00.102192 140363059820416 interactiveshell.py:2882] Step 3700 | Loss: 0.0052 | Spent: 2.2 secs | LR: 0.002579
I0419 02:50:02.248070 140363059820416 interactiveshell.py:2882] Step 3750 | Loss: 0.0046 | Spent: 2.1 secs | LR: 0.002574
I0419 02:50:04.404830 140363059820416 interactiveshell.py:2882] Step 3800 | Loss: 0.0041 | Spent: 2.2 secs | LR: 0.002569
I0419 02:50:06.574953 140363059820416 interactiveshell.py:2882] Step 3850 | Loss: 0.0037 | Spent: 2.2 secs | LR: 0.002564
I0419 02:50:08.731313 140363059820416 interactiveshell.py:2882] Step 3900 | Loss: 0.0043 | Spent: 2.2 secs | LR: 0.002558
I0419 02:50:10.902874 140363059820416 interactiveshell.py:2882] Step 3950 | Loss: 0.0051 | Spent: 2.2 secs | LR: 0.002553
I0419 02:50:13.098269 140363059820416 interactiveshell.py:2882] Step 4000 | Loss: 0.0038 | Spent: 2.2 secs | LR: 0.002548
I0419 02:50:15.251761 140363059820416 interactiveshell.py:2882] Step 4050 | Loss: 0.0031 | Spent: 2.2 secs | LR: 0.002543
I0419 02:50:17.412610 140363059820416 interactiveshell.py:2882] Step 4100 | Loss: 0.0029 | Spent: 2.2 secs | LR: 0.002538
I0419 02:50:19.578361 140363059820416 interactiveshell.py:2882] Step 4150 | Loss: 0.0033 | Spent: 2.2 secs | LR: 0.002532
I0419 02:50:21.746677 140363059820416 interactiveshell.py:2882] Step 4200 | Loss: 0.0029 | Spent: 2.2 secs | LR: 0.002527
I0419 02:50:23.909059 140363059820416 interactiveshell.py:2882] Step 4250 | Loss: 0.0032 | Spent: 2.2 secs | LR: 0.002522
I0419 02:50:26.063997 140363059820416 interactiveshell.py:2882] Step 4300 | Loss: 0.0036 | Spent: 2.2 secs | LR: 0.002517
I0419 02:50:28.218514 140363059820416 interactiveshell.py:2882] Step 4350 | Loss: 0.0026 | Spent: 2.2 secs | LR: 0.002512
I0419 02:50:30.384402 140363059820416 interactiveshell.py:2882] Step 4400 | Loss: 0.0036 | Spent: 2.2 secs | LR: 0.002507
Reading ../data/wn18/test.txt
I0419 02:50:33.402980 140363059820416 interactiveshell.py:2882] MRR: 0.787| [email protected]: 0.941 | [email protected]: 0.901 | [email protected]: 0.670
I0419 02:50:33.408011 140363059820416 interactiveshell.py:2882] Best MRR: 0.787
Reading ../data/wn18/train.txt
I0419 02:51:01.010845 140363059820416 interactiveshell.py:2882] Step 4450 | Loss: 0.0028 | Spent: 30.6 secs | LR: 0.002502
I0419 02:51:03.167533 140363059820416 interactiveshell.py:2882] Step 4500 | Loss: 0.0024 | Spent: 2.2 secs | LR: 0.002497
I0419 02:51:05.307936 140363059820416 interactiveshell.py:2882] Step 4550 | Loss: 0.0023 | Spent: 2.1 secs | LR: 0.002491
I0419 02:51:07.460746 140363059820416 interactiveshell.py:2882] Step 4600 | Loss: 0.0025 | Spent: 2.2 secs | LR: 0.002486
I0419 02:51:09.626403 140363059820416 interactiveshell.py:2882] Step 4650 | Loss: 0.0019 | Spent: 2.2 secs | LR: 0.002481
I0419 02:51:11.796262 140363059820416 interactiveshell.py:2882] Step 4700 | Loss: 0.0017 | Spent: 2.2 secs | LR: 0.002476
I0419 02:51:13.951525 140363059820416 interactiveshell.py:2882] Step 4750 | Loss: 0.0022 | Spent: 2.2 secs | LR: 0.002471
I0419 02:51:16.119417 140363059820416 interactiveshell.py:2882] Step 4800 | Loss: 0.0025 | Spent: 2.2 secs | LR: 0.002466
I0419 02:51:18.288289 140363059820416 interactiveshell.py:2882] Step 4850 | Loss: 0.0021 | Spent: 2.2 secs | LR: 0.002461
I0419 02:51:20.456620 140363059820416 interactiveshell.py:2882] Step 4900 | Loss: 0.0019 | Spent: 2.2 secs | LR: 0.002456
I0419 02:51:22.623989 140363059820416 interactiveshell.py:2882] Step 4950 | Loss: 0.0018 | Spent: 2.2 secs | LR: 0.002451
I0419 02:51:24.779106 140363059820416 interactiveshell.py:2882] Step 5000 | Loss: 0.0018 | Spent: 2.2 secs | LR: 0.002446
I0419 02:51:26.955220 140363059820416 interactiveshell.py:2882] Step 5050 | Loss: 0.0015 | Spent: 2.2 secs | LR: 0.002441
I0419 02:51:29.128236 140363059820416 interactiveshell.py:2882] Step 5100 | Loss: 0.0019 | Spent: 2.2 secs | LR: 0.002436
I0419 02:51:31.296869 140363059820416 interactiveshell.py:2882] Step 5150 | Loss: 0.0018 | Spent: 2.2 secs | LR: 0.002431
I0419 02:51:33.461788 140363059820416 interactiveshell.py:2882] Step 5200 | Loss: 0.0018 | Spent: 2.2 secs | LR: 0.002426
I0419 02:51:35.616897 140363059820416 interactiveshell.py:2882] Step 5250 | Loss: 0.0017 | Spent: 2.2 secs | LR: 0.002421
I0419 02:51:37.794231 140363059820416 interactiveshell.py:2882] Step 5300 | Loss: 0.0019 | Spent: 2.2 secs | LR: 0.002416
I0419 02:51:39.955643 140363059820416 interactiveshell.py:2882] Step 5350 | Loss: 0.0017 | Spent: 2.2 secs | LR: 0.002411
I0419 02:51:42.136529 140363059820416 interactiveshell.py:2882] Step 5400 | Loss: 0.0020 | Spent: 2.2 secs | LR: 0.002407
I0419 02:51:44.277934 140363059820416 interactiveshell.py:2882] Step 5450 | Loss: 0.0015 | Spent: 2.1 secs | LR: 0.002402
I0419 02:51:46.433987 140363059820416 interactiveshell.py:2882] Step 5500 | Loss: 0.0016 | Spent: 2.2 secs | LR: 0.002397
Reading ../data/wn18/test.txt
I0419 02:51:49.668187 140363059820416 interactiveshell.py:2882] MRR: 0.796| [email protected]: 0.944 | [email protected]: 0.915 | [email protected]: 0.676
I0419 02:51:49.672987 140363059820416 interactiveshell.py:2882] Best MRR: 0.796
Reading ../data/wn18/train.txt
I0419 02:52:18.927590 140363059820416 interactiveshell.py:2882] Step 5550 | Loss: 0.0014 | Spent: 32.5 secs | LR: 0.002392
I0419 02:52:21.358655 140363059820416 interactiveshell.py:2882] Step 5600 | Loss: 0.0012 | Spent: 2.4 secs | LR: 0.002387
I0419 02:52:23.517156 140363059820416 interactiveshell.py:2882] Step 5650 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002382
I0419 02:52:25.668445 140363059820416 interactiveshell.py:2882] Step 5700 | Loss: 0.0012 | Spent: 2.1 secs | LR: 0.002377
I0419 02:52:27.827868 140363059820416 interactiveshell.py:2882] Step 5750 | Loss: 0.0014 | Spent: 2.2 secs | LR: 0.002372
I0419 02:52:29.998427 140363059820416 interactiveshell.py:2882] Step 5800 | Loss: 0.0013 | Spent: 2.2 secs | LR: 0.002368
I0419 02:52:32.160956 140363059820416 interactiveshell.py:2882] Step 5850 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002363
I0419 02:52:34.322265 140363059820416 interactiveshell.py:2882] Step 5900 | Loss: 0.0013 | Spent: 2.2 secs | LR: 0.002358
I0419 02:52:36.474354 140363059820416 interactiveshell.py:2882] Step 5950 | Loss: 0.0011 | Spent: 2.2 secs | LR: 0.002353
I0419 02:52:38.625435 140363059820416 interactiveshell.py:2882] Step 6000 | Loss: 0.0009 | Spent: 2.1 secs | LR: 0.002348
I0419 02:52:40.784707 140363059820416 interactiveshell.py:2882] Step 6050 | Loss: 0.0011 | Spent: 2.2 secs | LR: 0.002343
I0419 02:52:42.958753 140363059820416 interactiveshell.py:2882] Step 6100 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002339
I0419 02:52:45.122363 140363059820416 interactiveshell.py:2882] Step 6150 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002334
I0419 02:52:47.288640 140363059820416 interactiveshell.py:2882] Step 6200 | Loss: 0.0011 | Spent: 2.2 secs | LR: 0.002329
I0419 02:52:49.458727 140363059820416 interactiveshell.py:2882] Step 6250 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002324
I0419 02:52:51.636865 140363059820416 interactiveshell.py:2882] Step 6300 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002320
I0419 02:52:53.803817 140363059820416 interactiveshell.py:2882] Step 6350 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002315
I0419 02:52:55.966971 140363059820416 interactiveshell.py:2882] Step 6400 | Loss: 0.0013 | Spent: 2.2 secs | LR: 0.002310
I0419 02:52:58.118885 140363059820416 interactiveshell.py:2882] Step 6450 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002306
I0419 02:53:00.285209 140363059820416 interactiveshell.py:2882] Step 6500 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002301
I0419 02:53:02.454388 140363059820416 interactiveshell.py:2882] Step 6550 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.002296
I0419 02:53:04.616501 140363059820416 interactiveshell.py:2882] Step 6600 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002291
Reading ../data/wn18/test.txt
I0419 02:53:08.135100 140363059820416 interactiveshell.py:2882] MRR: 0.796| [email protected]: 0.945 | [email protected]: 0.918 | [email protected]: 0.671
I0419 02:53:08.139744 140363059820416 interactiveshell.py:2882] Best MRR: 0.796
Reading ../data/wn18/train.txt
I0419 02:53:35.355734 140363059820416 interactiveshell.py:2882] Step 6650 | Loss: 0.0007 | Spent: 30.7 secs | LR: 0.002287
I0419 02:53:37.505830 140363059820416 interactiveshell.py:2882] Step 6700 | Loss: 0.0008 | Spent: 2.1 secs | LR: 0.002282
I0419 02:53:39.654364 140363059820416 interactiveshell.py:2882] Step 6750 | Loss: 0.0010 | Spent: 2.1 secs | LR: 0.002277
I0419 02:53:41.815752 140363059820416 interactiveshell.py:2882] Step 6800 | Loss: 0.0008 | Spent: 2.2 secs | LR: 0.002273
I0419 02:53:43.988122 140363059820416 interactiveshell.py:2882] Step 6850 | Loss: 0.0008 | Spent: 2.2 secs | LR: 0.002268
I0419 02:53:46.137506 140363059820416 interactiveshell.py:2882] Step 6900 | Loss: 0.0007 | Spent: 2.1 secs | LR: 0.002264
I0419 02:53:48.303936 140363059820416 interactiveshell.py:2882] Step 6950 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002259
I0419 02:53:50.477420 140363059820416 interactiveshell.py:2882] Step 7000 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002254
I0419 02:53:52.635059 140363059820416 interactiveshell.py:2882] Step 7050 | Loss: 0.0008 | Spent: 2.2 secs | LR: 0.002250
I0419 02:53:54.805500 140363059820416 interactiveshell.py:2882] Step 7100 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002245
I0419 02:53:56.976144 140363059820416 interactiveshell.py:2882] Step 7150 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002241
I0419 02:53:59.153827 140363059820416 interactiveshell.py:2882] Step 7200 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002236
I0419 02:54:01.332658 140363059820416 interactiveshell.py:2882] Step 7250 | Loss: 0.0008 | Spent: 2.2 secs | LR: 0.002231
I0419 02:54:03.504027 140363059820416 interactiveshell.py:2882] Step 7300 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.002227
I0419 02:54:05.670512 140363059820416 interactiveshell.py:2882] Step 7350 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002222
I0419 02:54:07.842494 140363059820416 interactiveshell.py:2882] Step 7400 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002218
I0419 02:54:10.010387 140363059820416 interactiveshell.py:2882] Step 7450 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002213
I0419 02:54:12.210930 140363059820416 interactiveshell.py:2882] Step 7500 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002209
I0419 02:54:14.388682 140363059820416 interactiveshell.py:2882] Step 7550 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002204
I0419 02:54:16.567562 140363059820416 interactiveshell.py:2882] Step 7600 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002200
I0419 02:54:18.729226 140363059820416 interactiveshell.py:2882] Step 7650 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002195
I0419 02:54:20.885736 140363059820416 interactiveshell.py:2882] Step 7700 | Loss: 0.0008 | Spent: 2.2 secs | LR: 0.002191
Reading ../data/wn18/test.txt
I0419 02:54:24.643871 140363059820416 interactiveshell.py:2882] MRR: 0.796| [email protected]: 0.946 | [email protected]: 0.921 | [email protected]: 0.670
I0419 02:54:24.646815 140363059820416 interactiveshell.py:2882] Best MRR: 0.796
Reading ../data/wn18/train.txt
I0419 02:54:52.752736 140363059820416 interactiveshell.py:2882] Step 7750 | Loss: 0.0005 | Spent: 31.9 secs | LR: 0.002186
I0419 02:54:54.943394 140363059820416 interactiveshell.py:2882] Step 7800 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002182
I0419 02:54:57.096084 140363059820416 interactiveshell.py:2882] Step 7850 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002177
I0419 02:54:59.239016 140363059820416 interactiveshell.py:2882] Step 7900 | Loss: 0.0004 | Spent: 2.1 secs | LR: 0.002173
I0419 02:55:01.402220 140363059820416 interactiveshell.py:2882] Step 7950 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002169
I0419 02:55:03.581848 140363059820416 interactiveshell.py:2882] Step 8000 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002164
I0419 02:55:05.743314 140363059820416 interactiveshell.py:2882] Step 8050 | Loss: 0.0006 | Spent: 2.2 secs | LR: 0.002160
I0419 02:55:07.902896 140363059820416 interactiveshell.py:2882] Step 8100 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002155
I0419 02:55:10.072140 140363059820416 interactiveshell.py:2882] Step 8150 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002151
I0419 02:55:12.260429 140363059820416 interactiveshell.py:2882] Step 8200 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002147
I0419 02:55:14.430076 140363059820416 interactiveshell.py:2882] Step 8250 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002142
I0419 02:55:16.592216 140363059820416 interactiveshell.py:2882] Step 8300 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002138
I0419 02:55:18.749949 140363059820416 interactiveshell.py:2882] Step 8350 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002133
I0419 02:55:20.909818 140363059820416 interactiveshell.py:2882] Step 8400 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002129
I0419 02:55:23.075429 140363059820416 interactiveshell.py:2882] Step 8450 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002125
I0419 02:55:25.243824 140363059820416 interactiveshell.py:2882] Step 8500 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002120
I0419 02:55:27.417875 140363059820416 interactiveshell.py:2882] Step 8550 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002116
I0419 02:55:29.586658 140363059820416 interactiveshell.py:2882] Step 8600 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002112
I0419 02:55:31.748242 140363059820416 interactiveshell.py:2882] Step 8650 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002107
I0419 02:55:33.912075 140363059820416 interactiveshell.py:2882] Step 8700 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002103
I0419 02:55:36.074780 140363059820416 interactiveshell.py:2882] Step 8750 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002099
I0419 02:55:38.230186 140363059820416 interactiveshell.py:2882] Step 8800 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002095
Reading ../data/wn18/test.txt
I0419 02:55:42.252061 140363059820416 interactiveshell.py:2882] MRR: 0.801| [email protected]: 0.947 | [email protected]: 0.924 | [email protected]: 0.677
I0419 02:55:42.256847 140363059820416 interactiveshell.py:2882] Best MRR: 0.801
Reading ../data/wn18/train.txt
I0419 02:56:08.363279 140363059820416 interactiveshell.py:2882] Step 8850 | Loss: 0.0004 | Spent: 30.1 secs | LR: 0.002090
I0419 02:56:10.773043 140363059820416 interactiveshell.py:2882] Step 8900 | Loss: 0.0004 | Spent: 2.4 secs | LR: 0.002086
I0419 02:56:13.042905 140363059820416 interactiveshell.py:2882] Step 8950 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.002082
I0419 02:56:15.192581 140363059820416 interactiveshell.py:2882] Step 9000 | Loss: 0.0004 | Spent: 2.1 secs | LR: 0.002078
I0419 02:56:17.369715 140363059820416 interactiveshell.py:2882] Step 9050 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.002073
I0419 02:56:19.540726 140363059820416 interactiveshell.py:2882] Step 9100 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.002069
I0419 02:56:21.705480 140363059820416 interactiveshell.py:2882] Step 9150 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002065
I0419 02:56:23.876171 140363059820416 interactiveshell.py:2882] Step 9200 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.002061
I0419 02:56:26.050717 140363059820416 interactiveshell.py:2882] Step 9250 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002057
I0419 02:56:28.223433 140363059820416 interactiveshell.py:2882] Step 9300 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002052
I0419 02:56:30.396097 140363059820416 interactiveshell.py:2882] Step 9350 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002048
I0419 02:56:32.580787 140363059820416 interactiveshell.py:2882] Step 9400 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002044
I0419 02:56:34.769355 140363059820416 interactiveshell.py:2882] Step 9450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.002040
I0419 02:56:36.940620 140363059820416 interactiveshell.py:2882] Step 9500 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002036
I0419 02:56:39.109008 140363059820416 interactiveshell.py:2882] Step 9550 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002031
I0419 02:56:41.276306 140363059820416 interactiveshell.py:2882] Step 9600 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.002027
I0419 02:56:43.451981 140363059820416 interactiveshell.py:2882] Step 9650 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002023
I0419 02:56:45.633667 140363059820416 interactiveshell.py:2882] Step 9700 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002019
I0419 02:56:47.809221 140363059820416 interactiveshell.py:2882] Step 9750 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002015
I0419 02:56:49.976487 140363059820416 interactiveshell.py:2882] Step 9800 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002011
I0419 02:56:52.156462 140363059820416 interactiveshell.py:2882] Step 9850 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.002007
I0419 02:56:54.328870 140363059820416 interactiveshell.py:2882] Step 9900 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.002003
I0419 02:56:56.492269 140363059820416 interactiveshell.py:2882] Step 9950 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001999
Reading ../data/wn18/test.txt
I0419 02:56:58.652274 140363059820416 interactiveshell.py:2882] MRR: 0.802| [email protected]: 0.948 | [email protected]: 0.928 | [email protected]: 0.677
I0419 02:56:58.657151 140363059820416 interactiveshell.py:2882] Best MRR: 0.802
Reading ../data/wn18/train.txt
I0419 02:57:26.162566 140363059820416 interactiveshell.py:2882] Step 10000 | Loss: 0.0003 | Spent: 29.7 secs | LR: 0.001994
I0419 02:57:28.637170 140363059820416 interactiveshell.py:2882] Step 10050 | Loss: 0.0003 | Spent: 2.5 secs | LR: 0.001990
I0419 02:57:31.075687 140363059820416 interactiveshell.py:2882] Step 10100 | Loss: 0.0004 | Spent: 2.4 secs | LR: 0.001986
I0419 02:57:33.342544 140363059820416 interactiveshell.py:2882] Step 10150 | Loss: 0.0004 | Spent: 2.3 secs | LR: 0.001982
I0419 02:57:35.493125 140363059820416 interactiveshell.py:2882] Step 10200 | Loss: 0.0003 | Spent: 2.1 secs | LR: 0.001978
I0419 02:57:37.656020 140363059820416 interactiveshell.py:2882] Step 10250 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001974
I0419 02:57:39.814240 140363059820416 interactiveshell.py:2882] Step 10300 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001970
I0419 02:57:41.972389 140363059820416 interactiveshell.py:2882] Step 10350 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001966
I0419 02:57:44.154650 140363059820416 interactiveshell.py:2882] Step 10400 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001962
I0419 02:57:46.347544 140363059820416 interactiveshell.py:2882] Step 10450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001958
I0419 02:57:48.528957 140363059820416 interactiveshell.py:2882] Step 10500 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001954
I0419 02:57:50.714782 140363059820416 interactiveshell.py:2882] Step 10550 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001950
I0419 02:57:52.873775 140363059820416 interactiveshell.py:2882] Step 10600 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001946
I0419 02:57:55.043900 140363059820416 interactiveshell.py:2882] Step 10650 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001942
I0419 02:57:57.219841 140363059820416 interactiveshell.py:2882] Step 10700 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001938
I0419 02:57:59.389407 140363059820416 interactiveshell.py:2882] Step 10750 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001934
I0419 02:58:01.572642 140363059820416 interactiveshell.py:2882] Step 10800 | Loss: 0.0009 | Spent: 2.2 secs | LR: 0.001930
I0419 02:58:03.745435 140363059820416 interactiveshell.py:2882] Step 10850 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001926
I0419 02:58:05.922014 140363059820416 interactiveshell.py:2882] Step 10900 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001923
I0419 02:58:08.090964 140363059820416 interactiveshell.py:2882] Step 10950 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001919
I0419 02:58:10.268933 140363059820416 interactiveshell.py:2882] Step 11000 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001915
I0419 02:58:12.457862 140363059820416 interactiveshell.py:2882] Step 11050 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001911
Reading ../data/wn18/test.txt
I0419 02:58:14.920981 140363059820416 interactiveshell.py:2882] MRR: 0.804| [email protected]: 0.947 | [email protected]: 0.924 | [email protected]: 0.682
I0419 02:58:14.925927 140363059820416 interactiveshell.py:2882] Best MRR: 0.804
Reading ../data/wn18/train.txt
I0419 02:58:41.230911 140363059820416 interactiveshell.py:2882] Step 11100 | Loss: 0.0003 | Spent: 28.8 secs | LR: 0.001907
I0419 02:58:43.463304 140363059820416 interactiveshell.py:2882] Step 11150 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001903
I0419 02:58:45.821015 140363059820416 interactiveshell.py:2882] Step 11200 | Loss: 0.0004 | Spent: 2.4 secs | LR: 0.001899
I0419 02:58:48.154664 140363059820416 interactiveshell.py:2882] Step 11250 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001895
I0419 02:58:50.500640 140363059820416 interactiveshell.py:2882] Step 11300 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001891
I0419 02:58:52.819999 140363059820416 interactiveshell.py:2882] Step 11350 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001888
I0419 02:58:54.992840 140363059820416 interactiveshell.py:2882] Step 11400 | Loss: 0.0007 | Spent: 2.2 secs | LR: 0.001884
I0419 02:58:57.152767 140363059820416 interactiveshell.py:2882] Step 11450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001880
I0419 02:58:59.309727 140363059820416 interactiveshell.py:2882] Step 11500 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001876
I0419 02:59:01.479920 140363059820416 interactiveshell.py:2882] Step 11550 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001872
I0419 02:59:03.649392 140363059820416 interactiveshell.py:2882] Step 11600 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001868
I0419 02:59:05.810759 140363059820416 interactiveshell.py:2882] Step 11650 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001865
I0419 02:59:07.979834 140363059820416 interactiveshell.py:2882] Step 11700 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001861
I0419 02:59:10.150264 140363059820416 interactiveshell.py:2882] Step 11750 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001857
I0419 02:59:12.304775 140363059820416 interactiveshell.py:2882] Step 11800 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001853
I0419 02:59:14.492609 140363059820416 interactiveshell.py:2882] Step 11850 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001849
I0419 02:59:16.669432 140363059820416 interactiveshell.py:2882] Step 11900 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001846
I0419 02:59:18.842114 140363059820416 interactiveshell.py:2882] Step 11950 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001842
I0419 02:59:21.005463 140363059820416 interactiveshell.py:2882] Step 12000 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001838
I0419 02:59:23.171277 140363059820416 interactiveshell.py:2882] Step 12050 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001834
I0419 02:59:25.331317 140363059820416 interactiveshell.py:2882] Step 12100 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001831
I0419 02:59:27.498985 140363059820416 interactiveshell.py:2882] Step 12150 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001827
Reading ../data/wn18/test.txt
I0419 02:59:30.156344 140363059820416 interactiveshell.py:2882] MRR: 0.807| [email protected]: 0.948 | [email protected]: 0.924 | [email protected]: 0.688
I0419 02:59:30.161260 140363059820416 interactiveshell.py:2882] Best MRR: 0.807
Reading ../data/wn18/train.txt
I0419 02:59:57.363820 140363059820416 interactiveshell.py:2882] Step 12200 | Loss: 0.0003 | Spent: 29.9 secs | LR: 0.001823
I0419 02:59:59.542338 140363059820416 interactiveshell.py:2882] Step 12250 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001819
I0419 03:00:01.701362 140363059820416 interactiveshell.py:2882] Step 12300 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001816
I0419 03:00:03.939495 140363059820416 interactiveshell.py:2882] Step 12350 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001812
I0419 03:00:06.288684 140363059820416 interactiveshell.py:2882] Step 12400 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001808
I0419 03:00:08.636584 140363059820416 interactiveshell.py:2882] Step 12450 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001805
I0419 03:00:10.991637 140363059820416 interactiveshell.py:2882] Step 12500 | Loss: 0.0003 | Spent: 2.4 secs | LR: 0.001801
I0419 03:00:13.313622 140363059820416 interactiveshell.py:2882] Step 12550 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001797
I0419 03:00:15.479029 140363059820416 interactiveshell.py:2882] Step 12600 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001794
I0419 03:00:17.649770 140363059820416 interactiveshell.py:2882] Step 12650 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001790
I0419 03:00:19.813188 140363059820416 interactiveshell.py:2882] Step 12700 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001786
I0419 03:00:21.977955 140363059820416 interactiveshell.py:2882] Step 12750 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001783
I0419 03:00:24.156149 140363059820416 interactiveshell.py:2882] Step 12800 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001779
I0419 03:00:26.322770 140363059820416 interactiveshell.py:2882] Step 12850 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001775
I0419 03:00:28.501571 140363059820416 interactiveshell.py:2882] Step 12900 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001772
I0419 03:00:30.664682 140363059820416 interactiveshell.py:2882] Step 12950 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001768
I0419 03:00:32.851950 140363059820416 interactiveshell.py:2882] Step 13000 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001765
I0419 03:00:35.019503 140363059820416 interactiveshell.py:2882] Step 13050 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001761
I0419 03:00:37.180516 140363059820416 interactiveshell.py:2882] Step 13100 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001757
I0419 03:00:39.349562 140363059820416 interactiveshell.py:2882] Step 13150 | Loss: 0.0005 | Spent: 2.2 secs | LR: 0.001754
I0419 03:00:41.515085 140363059820416 interactiveshell.py:2882] Step 13200 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001750
I0419 03:00:43.678015 140363059820416 interactiveshell.py:2882] Step 13250 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001747
Reading ../data/wn18/test.txt
I0419 03:00:46.672073 140363059820416 interactiveshell.py:2882] MRR: 0.808| [email protected]: 0.949 | [email protected]: 0.927 | [email protected]: 0.687
I0419 03:00:46.678630 140363059820416 interactiveshell.py:2882] Best MRR: 0.808
Reading ../data/wn18/train.txt
I0419 03:01:12.565281 140363059820416 interactiveshell.py:2882] Step 13300 | Loss: 0.0002 | Spent: 28.9 secs | LR: 0.001743
I0419 03:01:14.714180 140363059820416 interactiveshell.py:2882] Step 13350 | Loss: 0.0003 | Spent: 2.1 secs | LR: 0.001740
I0419 03:01:16.867226 140363059820416 interactiveshell.py:2882] Step 13400 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001736
I0419 03:01:19.010283 140363059820416 interactiveshell.py:2882] Step 13450 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001732
I0419 03:01:21.177013 140363059820416 interactiveshell.py:2882] Step 13500 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001729
I0419 03:01:23.329146 140363059820416 interactiveshell.py:2882] Step 13550 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001725
I0419 03:01:25.672645 140363059820416 interactiveshell.py:2882] Step 13600 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001722
I0419 03:01:28.025345 140363059820416 interactiveshell.py:2882] Step 13650 | Loss: 0.0003 | Spent: 2.4 secs | LR: 0.001718
I0419 03:01:30.365940 140363059820416 interactiveshell.py:2882] Step 13700 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001715
I0419 03:01:32.701982 140363059820416 interactiveshell.py:2882] Step 13750 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001711
I0419 03:01:34.939349 140363059820416 interactiveshell.py:2882] Step 13800 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001708
I0419 03:01:37.098617 140363059820416 interactiveshell.py:2882] Step 13850 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001704
I0419 03:01:39.247241 140363059820416 interactiveshell.py:2882] Step 13900 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001701
I0419 03:01:41.409868 140363059820416 interactiveshell.py:2882] Step 13950 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001697
I0419 03:01:43.575518 140363059820416 interactiveshell.py:2882] Step 14000 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001694
I0419 03:01:45.751561 140363059820416 interactiveshell.py:2882] Step 14050 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001691
I0419 03:01:47.911783 140363059820416 interactiveshell.py:2882] Step 14100 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001687
I0419 03:01:50.096114 140363059820416 interactiveshell.py:2882] Step 14150 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001684
I0419 03:01:52.261914 140363059820416 interactiveshell.py:2882] Step 14200 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001680
I0419 03:01:54.425772 140363059820416 interactiveshell.py:2882] Step 14250 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001677
I0419 03:01:56.589558 140363059820416 interactiveshell.py:2882] Step 14300 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001673
I0419 03:01:58.746105 140363059820416 interactiveshell.py:2882] Step 14350 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001670
Reading ../data/wn18/test.txt
I0419 03:02:01.929275 140363059820416 interactiveshell.py:2882] MRR: 0.810| [email protected]: 0.949 | [email protected]: 0.926 | [email protected]: 0.692
I0419 03:02:01.934513 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:02:27.826611 140363059820416 interactiveshell.py:2882] Step 14400 | Loss: 0.0002 | Spent: 29.1 secs | LR: 0.001667
I0419 03:02:29.987325 140363059820416 interactiveshell.py:2882] Step 14450 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001663
I0419 03:02:32.123877 140363059820416 interactiveshell.py:2882] Step 14500 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001660
I0419 03:02:34.467497 140363059820416 interactiveshell.py:2882] Step 14550 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001656
I0419 03:02:36.842844 140363059820416 interactiveshell.py:2882] Step 14600 | Loss: 0.0003 | Spent: 2.4 secs | LR: 0.001653
I0419 03:02:39.261272 140363059820416 interactiveshell.py:2882] Step 14650 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001650
I0419 03:02:41.688321 140363059820416 interactiveshell.py:2882] Step 14700 | Loss: 0.0004 | Spent: 2.4 secs | LR: 0.001646
I0419 03:02:43.982613 140363059820416 interactiveshell.py:2882] Step 14750 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001643
I0419 03:02:46.324828 140363059820416 interactiveshell.py:2882] Step 14800 | Loss: 0.0004 | Spent: 2.3 secs | LR: 0.001640
I0419 03:02:48.680406 140363059820416 interactiveshell.py:2882] Step 14850 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001636
I0419 03:02:51.023689 140363059820416 interactiveshell.py:2882] Step 14900 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001633
I0419 03:02:53.366230 140363059820416 interactiveshell.py:2882] Step 14950 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001630
I0419 03:02:55.575829 140363059820416 interactiveshell.py:2882] Step 15000 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001626
I0419 03:02:57.734235 140363059820416 interactiveshell.py:2882] Step 15050 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001623
I0419 03:02:59.907406 140363059820416 interactiveshell.py:2882] Step 15100 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001620
I0419 03:03:02.078325 140363059820416 interactiveshell.py:2882] Step 15150 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001616
I0419 03:03:04.244468 140363059820416 interactiveshell.py:2882] Step 15200 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001613
I0419 03:03:06.389238 140363059820416 interactiveshell.py:2882] Step 15250 | Loss: 0.0003 | Spent: 2.1 secs | LR: 0.001610
I0419 03:03:08.557773 140363059820416 interactiveshell.py:2882] Step 15300 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001606
I0419 03:03:10.724197 140363059820416 interactiveshell.py:2882] Step 15350 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001603
I0419 03:03:12.891543 140363059820416 interactiveshell.py:2882] Step 15400 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001600
I0419 03:03:15.056863 140363059820416 interactiveshell.py:2882] Step 15450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001597
Reading ../data/wn18/test.txt
I0419 03:03:18.524386 140363059820416 interactiveshell.py:2882] MRR: 0.807| [email protected]: 0.949 | [email protected]: 0.926 | [email protected]: 0.688
I0419 03:03:18.529479 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:03:44.047668 140363059820416 interactiveshell.py:2882] Step 15500 | Loss: 0.0003 | Spent: 29.0 secs | LR: 0.001593
I0419 03:03:46.213158 140363059820416 interactiveshell.py:2882] Step 15550 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001590
I0419 03:03:48.367478 140363059820416 interactiveshell.py:2882] Step 15600 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001587
I0419 03:03:50.509772 140363059820416 interactiveshell.py:2882] Step 15650 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001584
I0419 03:03:52.662824 140363059820416 interactiveshell.py:2882] Step 15700 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001580
I0419 03:03:54.817422 140363059820416 interactiveshell.py:2882] Step 15750 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001577
I0419 03:03:56.971871 140363059820416 interactiveshell.py:2882] Step 15800 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001574
I0419 03:03:59.146476 140363059820416 interactiveshell.py:2882] Step 15850 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001571
I0419 03:04:01.300762 140363059820416 interactiveshell.py:2882] Step 15900 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001568
I0419 03:04:03.464588 140363059820416 interactiveshell.py:2882] Step 15950 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001564
I0419 03:04:05.743123 140363059820416 interactiveshell.py:2882] Step 16000 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001561
I0419 03:04:08.086133 140363059820416 interactiveshell.py:2882] Step 16050 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001558
I0419 03:04:10.436881 140363059820416 interactiveshell.py:2882] Step 16100 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001555
I0419 03:04:12.798673 140363059820416 interactiveshell.py:2882] Step 16150 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001552
I0419 03:04:15.072909 140363059820416 interactiveshell.py:2882] Step 16200 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001549
I0419 03:04:17.239684 140363059820416 interactiveshell.py:2882] Step 16250 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001545
I0419 03:04:19.390938 140363059820416 interactiveshell.py:2882] Step 16300 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001542
I0419 03:04:21.553794 140363059820416 interactiveshell.py:2882] Step 16350 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001539
I0419 03:04:23.727125 140363059820416 interactiveshell.py:2882] Step 16400 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001536
I0419 03:04:25.909080 140363059820416 interactiveshell.py:2882] Step 16450 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001533
I0419 03:04:28.096504 140363059820416 interactiveshell.py:2882] Step 16500 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001530
I0419 03:04:30.283957 140363059820416 interactiveshell.py:2882] Step 16550 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001527
Reading ../data/wn18/test.txt
I0419 03:04:34.009741 140363059820416 interactiveshell.py:2882] MRR: 0.804| [email protected]: 0.947 | [email protected]: 0.922 | [email protected]: 0.683
I0419 03:04:34.013939 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:05:00.217474 140363059820416 interactiveshell.py:2882] Step 16600 | Loss: 0.0002 | Spent: 29.9 secs | LR: 0.001523
I0419 03:05:02.405017 140363059820416 interactiveshell.py:2882] Step 16650 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001520
I0419 03:05:04.558930 140363059820416 interactiveshell.py:2882] Step 16700 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001517
I0419 03:05:06.707412 140363059820416 interactiveshell.py:2882] Step 16750 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001514
I0419 03:05:08.867673 140363059820416 interactiveshell.py:2882] Step 16800 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001511
I0419 03:05:11.033065 140363059820416 interactiveshell.py:2882] Step 16850 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001508
I0419 03:05:13.199657 140363059820416 interactiveshell.py:2882] Step 16900 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001505
I0419 03:05:15.355546 140363059820416 interactiveshell.py:2882] Step 16950 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001502
I0419 03:05:17.528534 140363059820416 interactiveshell.py:2882] Step 17000 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001499
I0419 03:05:19.704082 140363059820416 interactiveshell.py:2882] Step 17050 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001496
I0419 03:05:21.858234 140363059820416 interactiveshell.py:2882] Step 17100 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001493
I0419 03:05:24.026916 140363059820416 interactiveshell.py:2882] Step 17150 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001490
I0419 03:05:26.317033 140363059820416 interactiveshell.py:2882] Step 17200 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001487
I0419 03:05:28.669998 140363059820416 interactiveshell.py:2882] Step 17250 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001484
I0419 03:05:31.026062 140363059820416 interactiveshell.py:2882] Step 17300 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001481
I0419 03:05:33.375689 140363059820416 interactiveshell.py:2882] Step 17350 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001477
I0419 03:05:35.644837 140363059820416 interactiveshell.py:2882] Step 17400 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001474
I0419 03:05:37.824846 140363059820416 interactiveshell.py:2882] Step 17450 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001471
I0419 03:05:39.990617 140363059820416 interactiveshell.py:2882] Step 17500 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001468
I0419 03:05:42.152621 140363059820416 interactiveshell.py:2882] Step 17550 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001465
I0419 03:05:44.307726 140363059820416 interactiveshell.py:2882] Step 17600 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001462
I0419 03:05:46.471768 140363059820416 interactiveshell.py:2882] Step 17650 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001460
Reading ../data/wn18/test.txt
I0419 03:05:50.392532 140363059820416 interactiveshell.py:2882] MRR: 0.804| [email protected]: 0.947 | [email protected]: 0.924 | [email protected]: 0.682
I0419 03:05:50.394960 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:06:15.079569 140363059820416 interactiveshell.py:2882] Step 17700 | Loss: 0.0004 | Spent: 28.6 secs | LR: 0.001457
I0419 03:06:17.265018 140363059820416 interactiveshell.py:2882] Step 17750 | Loss: 0.0001 | Spent: 2.2 secs | LR: 0.001454
I0419 03:06:19.413024 140363059820416 interactiveshell.py:2882] Step 17800 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001451
I0419 03:06:21.564203 140363059820416 interactiveshell.py:2882] Step 17850 | Loss: 0.0002 | Spent: 2.1 secs | LR: 0.001448
I0419 03:06:23.718847 140363059820416 interactiveshell.py:2882] Step 17900 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001445
I0419 03:06:25.896656 140363059820416 interactiveshell.py:2882] Step 17950 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001442
I0419 03:06:28.048960 140363059820416 interactiveshell.py:2882] Step 18000 | Loss: 0.0001 | Spent: 2.2 secs | LR: 0.001439
I0419 03:06:30.204495 140363059820416 interactiveshell.py:2882] Step 18050 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001436
I0419 03:06:32.364904 140363059820416 interactiveshell.py:2882] Step 18100 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001433
I0419 03:06:34.538787 140363059820416 interactiveshell.py:2882] Step 18150 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001430
I0419 03:06:36.710727 140363059820416 interactiveshell.py:2882] Step 18200 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001427
I0419 03:06:38.908045 140363059820416 interactiveshell.py:2882] Step 18250 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001424
I0419 03:06:41.068333 140363059820416 interactiveshell.py:2882] Step 18300 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001421
I0419 03:06:43.228185 140363059820416 interactiveshell.py:2882] Step 18350 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001418
I0419 03:06:45.435175 140363059820416 interactiveshell.py:2882] Step 18400 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001416
I0419 03:06:47.800668 140363059820416 interactiveshell.py:2882] Step 18450 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001413
I0419 03:06:50.150009 140363059820416 interactiveshell.py:2882] Step 18500 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001410
I0419 03:06:52.508491 140363059820416 interactiveshell.py:2882] Step 18550 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001407
I0419 03:06:54.856913 140363059820416 interactiveshell.py:2882] Step 18600 | Loss: 0.0003 | Spent: 2.3 secs | LR: 0.001404
I0419 03:06:57.049504 140363059820416 interactiveshell.py:2882] Step 18650 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001401
I0419 03:06:59.213945 140363059820416 interactiveshell.py:2882] Step 18700 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001398
I0419 03:07:01.378394 140363059820416 interactiveshell.py:2882] Step 18750 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001395
I0419 03:07:03.549765 140363059820416 interactiveshell.py:2882] Step 18800 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001393
Reading ../data/wn18/test.txt
I0419 03:07:05.613154 140363059820416 interactiveshell.py:2882] MRR: 0.799| [email protected]: 0.948 | [email protected]: 0.923 | [email protected]: 0.671
I0419 03:07:05.616983 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:07:32.352391 140363059820416 interactiveshell.py:2882] Step 18850 | Loss: 0.0002 | Spent: 28.8 secs | LR: 0.001390
I0419 03:07:34.530455 140363059820416 interactiveshell.py:2882] Step 18900 | Loss: 0.0001 | Spent: 2.2 secs | LR: 0.001387
I0419 03:07:36.684872 140363059820416 interactiveshell.py:2882] Step 18950 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001384
I0419 03:07:38.839009 140363059820416 interactiveshell.py:2882] Step 19000 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001381
I0419 03:07:41.010899 140363059820416 interactiveshell.py:2882] Step 19050 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001378
I0419 03:07:43.196222 140363059820416 interactiveshell.py:2882] Step 19100 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001376
I0419 03:07:45.600788 140363059820416 interactiveshell.py:2882] Step 19150 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001373
I0419 03:07:48.042237 140363059820416 interactiveshell.py:2882] Step 19200 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001370
I0419 03:07:50.468606 140363059820416 interactiveshell.py:2882] Step 19250 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001367
I0419 03:07:52.897455 140363059820416 interactiveshell.py:2882] Step 19300 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001364
I0419 03:07:55.113472 140363059820416 interactiveshell.py:2882] Step 19350 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001362
I0419 03:07:57.293524 140363059820416 interactiveshell.py:2882] Step 19400 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001359
I0419 03:07:59.471431 140363059820416 interactiveshell.py:2882] Step 19450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001356
I0419 03:08:01.640886 140363059820416 interactiveshell.py:2882] Step 19500 | Loss: 0.0010 | Spent: 2.2 secs | LR: 0.001353
I0419 03:08:03.825306 140363059820416 interactiveshell.py:2882] Step 19550 | Loss: 0.0004 | Spent: 2.2 secs | LR: 0.001351
I0419 03:08:06.067859 140363059820416 interactiveshell.py:2882] Step 19600 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001348
I0419 03:08:08.435000 140363059820416 interactiveshell.py:2882] Step 19650 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001345
I0419 03:08:10.797613 140363059820416 interactiveshell.py:2882] Step 19700 | Loss: 0.0004 | Spent: 2.4 secs | LR: 0.001342
I0419 03:08:13.152040 140363059820416 interactiveshell.py:2882] Step 19750 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001340
I0419 03:08:15.506778 140363059820416 interactiveshell.py:2882] Step 19800 | Loss: 0.0002 | Spent: 2.4 secs | LR: 0.001337
I0419 03:08:17.673060 140363059820416 interactiveshell.py:2882] Step 19850 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001334
I0419 03:08:19.851164 140363059820416 interactiveshell.py:2882] Step 19900 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001331
Reading ../data/wn18/test.txt
I0419 03:08:22.219443 140363059820416 interactiveshell.py:2882] MRR: 0.798| [email protected]: 0.949 | [email protected]: 0.926 | [email protected]: 0.668
I0419 03:08:22.224124 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
Reading ../data/wn18/train.txt
I0419 03:08:49.069505 140363059820416 interactiveshell.py:2882] Step 19950 | Loss: 0.0002 | Spent: 29.2 secs | LR: 0.001329
I0419 03:08:51.235039 140363059820416 interactiveshell.py:2882] Step 20000 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001326
I0419 03:08:53.394015 140363059820416 interactiveshell.py:2882] Step 20050 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001323
I0419 03:08:55.569463 140363059820416 interactiveshell.py:2882] Step 20100 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001321
I0419 03:08:57.789127 140363059820416 interactiveshell.py:2882] Step 20150 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001318
I0419 03:08:59.960206 140363059820416 interactiveshell.py:2882] Step 20200 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001315
I0419 03:09:02.139737 140363059820416 interactiveshell.py:2882] Step 20250 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001313
I0419 03:09:04.325982 140363059820416 interactiveshell.py:2882] Step 20300 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001310
I0419 03:09:06.510616 140363059820416 interactiveshell.py:2882] Step 20350 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001307
I0419 03:09:08.681034 140363059820416 interactiveshell.py:2882] Step 20400 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001305
I0419 03:09:10.867643 140363059820416 interactiveshell.py:2882] Step 20450 | Loss: 0.0003 | Spent: 2.2 secs | LR: 0.001302
I0419 03:09:13.046737 140363059820416 interactiveshell.py:2882] Step 20500 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001299
I0419 03:09:15.234053 140363059820416 interactiveshell.py:2882] Step 20550 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001297
I0419 03:09:17.418440 140363059820416 interactiveshell.py:2882] Step 20600 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001294
I0419 03:09:19.615988 140363059820416 interactiveshell.py:2882] Step 20650 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001291
I0419 03:09:21.796715 140363059820416 interactiveshell.py:2882] Step 20700 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001289
I0419 03:09:23.975501 140363059820416 interactiveshell.py:2882] Step 20750 | Loss: 0.0001 | Spent: 2.2 secs | LR: 0.001286
I0419 03:09:26.199164 140363059820416 interactiveshell.py:2882] Step 20800 | Loss: 0.0002 | Spent: 2.2 secs | LR: 0.001283
I0419 03:09:28.545131 140363059820416 interactiveshell.py:2882] Step 20850 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001281
I0419 03:09:30.888232 140363059820416 interactiveshell.py:2882] Step 20900 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001278
I0419 03:09:33.229095 140363059820416 interactiveshell.py:2882] Step 20950 | Loss: 0.0002 | Spent: 2.3 secs | LR: 0.001276
I0419 03:09:35.584407 140363059820416 interactiveshell.py:2882] Step 21000 | Loss: 0.0001 | Spent: 2.4 secs | LR: 0.001273
Reading ../data/wn18/test.txt
I0419 03:09:38.220379 140363059820416 interactiveshell.py:2882] MRR: 0.795| [email protected]: 0.947 | [email protected]: 0.923 | [email protected]: 0.665
I0419 03:09:38.225623 140363059820416 interactiveshell.py:2882] Best MRR: 0.810
I0419 03:09:38.226562 140363059820416 interactiveshell.py:2882] MRR not improved over 3 epochs, Early Stop