"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow1/semantic_parsing/tree_slu/main')

Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).

!pip install tensorflow-hub

Requirement already satisfied: tensorflow-hub in /usr/local/lib/python3.6/dist-packages (0.7.0)
Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-hub) (1.12.0)
Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-hub) (3.10.0)
Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-hub) (1.17.3)
Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.4.0->tensorflow-hub) (41.4.0)

%tensorflow_version 1.x
import tensorflow as tf
import tensorflow_hub as hub

import numpy as np
import pprint
import logging
import time
import nltk
import os
import random

from pathlib import Path

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 1.15.0
GPU Enabled: True

# stream data from text files
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      text_raw, text_tokenized, label = line.split('\t')
      text_tokenized = text_tokenized.lower().split()
      label = label.replace('[', '[ ').lower().split()
      #source = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in text_tokenized]
      target = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in label]
      target_in = [1] + target
      target_out = target + [2]
      yield (text_tokenized, (target_in, target_out))

def dataset(is_training, params):
  _shapes = ([None], ([None], [None]))
  _types = (tf.string, (tf.int32, tf.int32))
  _pads = ('<pad>', (0, 0))
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['train_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['eval_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds

def clip_grads(loss):
    variables = tf.trainable_variables()
    pprint.pprint(variables)
    grads = tf.gradients(loss, variables)
    clipped_grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    return zip(clipped_grads, variables)


def rnn_cell():
    def cell_fn():
        cell = tf.nn.rnn_cell.LSTMCell(params['rnn_units'],
                                       initializer=tf.orthogonal_initializer())
        return cell
    if params['dec_layers'] > 1:
      cells = []
      for i in range(params['dec_layers']):
        if i == params['dec_layers'] - 1:
          cells.append(cell_fn())
        else:
          cells.append(tf.nn.rnn_cell.ResidualWrapper(cell_fn(), residual_fn=lambda i,o: tf.concat((i,o), -1)))
      return tf.nn.rnn_cell.MultiRNNCell(cells)
    else:
      return cell_fn()

  
def dec_cell(enc_out, enc_seq_len):
    attn1 = tf.contrib.seq2seq.BahdanauAttention(
        num_units = params['rnn_units'],
        memory = enc_out,
        memory_sequence_length = enc_seq_len,
        normalize=False)
    
    attn2 = tf.contrib.seq2seq.LuongAttention(
        num_units = params['rnn_units'],
        memory = enc_out,
        memory_sequence_length = enc_seq_len,
        scale=False)
    
    return tf.contrib.seq2seq.AttentionWrapper(
        cell = rnn_cell(),
        attention_mechanism = (attn1, attn2),
        attention_layer_size = (params['rnn_units'], params['rnn_units']))

class TiedDense(tf.layers.Layer):
  def __init__(self, tied_embed, out_dim):
    super().__init__()
    self.tied_embed = tied_embed
    self.out_dim = out_dim
  
  def build(self, input_shape):
    self.bias = self.add_weight(name='bias',
                                shape=[self.out_dim],
                                trainable=True)
    self.proj_W = self.add_weight(name='proj_W',
                                  shape=[2*params['rnn_units'], params['rnn_units']],
                                  trainable=True)
    self.proj_b = self.add_weight(name='proj_b',
                                  shape=[params['rnn_units']],
                                  trainable=True)
    super().build(input_shape)
  
  def call(self, inputs):
    x = params['activation'](tf.nn.bias_add(tf.matmul(inputs, self.proj_W), self.proj_b))
    x = tf.matmul(x, self.tied_embed, transpose_b=True)
    x = tf.nn.bias_add(x, self.bias)
    return x
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.out_dim)

def forward(raw_texts, labels, mode):
    vocab = tf.contrib.lookup.index_table_from_file(
        params['vocab_tgt_path'], num_oov_buckets=1)
    words = vocab.lookup(raw_texts)
    words_len = tf.count_nonzero(words, 1, dtype=tf.int32)
    
    is_training = (mode == tf.estimator.ModeKeys.TRAIN)
    batch_sz = tf.shape(words)[0]
    
    
    with tf.variable_scope('Embedding'):
        elmo = hub.Module("https://tfhub.dev/google/elmo/2", trainable=False)
        e = elmo(inputs={'tokens':raw_texts, 'sequence_len':words_len,}, signature="tokens", as_dict=True)['lstm_outputs1']
        e = tf.layers.dropout(e, params['dropout_rate'], training=is_training)
        e = tf.layers.dense(e, params['rnn_units'], params['activation'])
        
        embedding = tf.Variable(np.load('../vocab/word.npy'),
                                dtype=tf.float32,
                                name='glove')
        x = tf.nn.embedding_lookup(embedding, words)
        x = tf.concat((x, e), -1)
        x = tf.layers.dropout(x, params['dropout_rate'], training=is_training)
    
    
    with tf.variable_scope('Encoder'):
        t = tf.transpose(x, perm=[1, 0, 2])  # Need time-major
        lstm_cell_fw = tf.contrib.rnn.LSTMBlockFusedCell(params['rnn_units'])
        lstm_cell_bw = tf.contrib.rnn.LSTMBlockFusedCell(params['rnn_units'])
        lstm_cell_bw = tf.contrib.rnn.TimeReversedFusedRNN(lstm_cell_bw)
        o_fw, s_fw = lstm_cell_fw(t, dtype=tf.float32, sequence_length=words_len)
        o_bw, s_bw = lstm_cell_bw(t, dtype=tf.float32, sequence_length=words_len)
        enc_out = tf.concat([o_fw, o_bw], axis=-1)
        enc_out = tf.transpose(enc_out, perm=[1, 0, 2])

        enc_state = tf.layers.dense(tf.concat((s_fw.h, s_bw.h), -1), params['rnn_units'], params['activation'], name='state_fc')
        enc_state = tf.nn.rnn_cell.LSTMStateTuple(c=enc_state, h=enc_state)
        if params['dec_layers'] > 1:
          enc_state = tuple(params['dec_layers'] * [enc_state])
    
    
    with tf.variable_scope('Decoder'):
        output_proj = TiedDense(embedding, len(params['tgt2idx'])+1)
        
        if is_training or (mode == tf.estimator.ModeKeys.EVAL):
            dec_inputs, dec_outputs = labels
            dec_seq_len = tf.count_nonzero(dec_inputs, 1, dtype=tf.int32)
            dec_inputs = tf.nn.embedding_lookup(embedding, dec_inputs)
            dec_inputs = tf.layers.dropout(dec_inputs, params['dropout_rate'], training=is_training)
            cell = dec_cell(enc_out, words_len)
            
            init_state = cell.zero_state(batch_sz, tf.float32).clone(
                cell_state=enc_state)
            
            helper = tf.contrib.seq2seq.TrainingHelper(
                inputs = dec_inputs,
                sequence_length = dec_seq_len)
            decoder = tf.contrib.seq2seq.BasicDecoder(
                cell = cell,
                helper = helper,
                initial_state = init_state,
                output_layer = output_proj)
            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
                decoder = decoder,
                maximum_iterations = tf.reduce_max(dec_seq_len))
            
            return decoder_output.rnn_output
        else:
            enc_out_t = tf.contrib.seq2seq.tile_batch(enc_out, params['beam_width'])
            enc_state_t = tf.contrib.seq2seq.tile_batch(enc_state, params['beam_width'])
            enc_seq_len_t = tf.contrib.seq2seq.tile_batch(words_len, params['beam_width'])
            
            cell = dec_cell(enc_out_t, enc_seq_len_t)
            
            init_state = cell.zero_state(batch_sz*params['beam_width'], tf.float32).clone(
                cell_state=enc_state_t)
            
            decoder = tf.contrib.seq2seq.BeamSearchDecoder(
                cell = cell,
                embedding = embedding,
                start_tokens = tf.tile(tf.constant([1], tf.int32), [batch_sz]),
                end_token = 2,
                initial_state = init_state,
                beam_width = params['beam_width'],
                output_layer = output_proj)
            decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
                decoder = decoder,
                maximum_iterations = 80,)
            
            return decoder_output.predicted_ids[:, :, 0]

def model_fn(features, labels, mode, params):
    logits_or_ids = forward(features, labels, mode)
    
    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode, predictions=logits_or_ids)
    
    dec_inputs, dec_outputs = labels
    loss_op = tf.losses.softmax_cross_entropy(onehot_labels = tf.one_hot(dec_outputs, len(params['tgt2idx'])+1),
                                              logits = logits_or_ids,
                                              weights = tf.to_float(tf.sign(dec_outputs)),
                                              label_smoothing = .2)  
      
    if mode == tf.estimator.ModeKeys.TRAIN:
        global_step=tf.train.get_or_create_global_step()
        
        decay_lr = tf.train.exponential_decay(
            params['lr'], global_step, 1000, .96)
        
        train_op = tf.train.AdamOptimizer(decay_lr).apply_gradients(
            clip_grads(loss_op), global_step=global_step)
        
        hook = tf.train.LoggingTensorHook({'lr': decay_lr}, every_n_iter=100)
        
        return tf.estimator.EstimatorSpec(
            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[hook],)

def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx

params = {
    'model_dir': '../model/lstm_seq2seq',
    'log_path': '../log/lstm_seq2seq.txt',
    'train_path': '../data/train.tsv',
    'test_path': '../data/test.tsv',
    'vocab_src_path': '../vocab/source.txt',
    'vocab_tgt_path': '../vocab/target.txt',
    'model_path': '../model/',
    'dropout_rate': 0.2,
    'rnn_units': 300,
    'dec_layers': 1,
    'beam_width': 10,
    'activation': tf.nn.relu,
    'lr': 4e-4,
    'clip_norm': .1,
    'buffer_size': 31279,
    'train_batch_size': 32,
    'eval_batch_size': 128,
    'num_patience': 5,
}

params['tgt2idx'] = get_vocab(params['vocab_tgt_path'])
params['idx2tgt'] = {idx: tgt for tgt, idx in params['tgt2idx'].items()}

def is_descending(history: list) -> bool:
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True

def minimal_test(estimator):
  test_str = [['what', 'times', 'are', 'the', 'nutcracker', 'show', 'playing', 'near', 'me']]
  predicted = list(estimator.predict(tf.estimator.inputs.numpy_input_fn(
    x = np.array(test_str), shuffle = False)))
  print('-'*12)
  print('minimal test')
  print('utterance:', ' '.join(test_str[0]))
  predicted = ' '.join([params['idx2tgt'].get(idx, len(params['idx2tgt'])) for idx in predicted[0]])
  predicted = predicted.replace('<end>', '').strip()
  print('parsed:', predicted)
  print()
  try:
    nltk.tree.Tree.fromstring(predicted.replace('[ ', '(').replace(' ]', ')')).pretty_print()
  except:
    pass
  print('-'*12)

# Create directory if not exist
Path(os.path.dirname(params['log_path'])).mkdir(exist_ok=True)
Path(params['model_dir']).mkdir(exist_ok=True, parents=True)

# Logging
logger = logging.getLogger('tensorflow')
logger.propagate = False
logger.setLevel(logging.INFO)
fh = logging.FileHandler(params['log_path'])
logger.addHandler(fh)

# Create an estimator
eval_steps = params['buffer_size']//params['train_batch_size'] + 1
estimator = tf.estimator.Estimator(
  model_fn=model_fn,
  model_dir=params['model_dir'],
  config=tf.estimator.RunConfig(save_checkpoints_steps=eval_steps),
  params=params)

best_acc = .0
history_acc = []
tf.enable_eager_execution()

while True:
  estimator.train(input_fn=lambda: dataset(is_training=True, params=params))

  minimal_test(estimator)

  # Evaluation
  labels = [label for _, (_, label) in dataset(is_training=False, params=params)]
  seq_lens = [tf.argmax(tf.cast(tf.equal(l, 2), tf.int32), axis=1) for l in labels]
  labels = [j for i in labels for j in i.numpy()]
  seq_lens = [j for i in seq_lens for j in i.numpy()]

  preds = list(estimator.predict(input_fn=lambda: dataset(is_training=False, params=params)))

  assert len(preds) == len(labels)

  res = [np.all(p[:seq_len+1] == l[:seq_len+1]) for p, l, seq_len in zip(preds, labels, seq_lens)]
  acc = np.asarray(res).mean()
  logger.info("Evaluation: Testing (Exact Match) Accuracy: {:.3f}".format(acc))
  history_acc.append(acc)

  if acc > best_acc:
    best_acc = acc
  logger.info("Best (Exact Match) Accuracy: {:.3f}".format(best_acc))

  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing (Exact Match) Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break

INFO:tensorflow:Using config: {'_model_dir': '../model/lstm_seq2seq', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 978, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true
graph_options {
  rewrite_options {
    meta_optimizer_iterations: ONE
  }
}
, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f05efdae6a0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
INFO:tensorflow:Calling model_fn.
WARNING:tensorflow:
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.
Instructions for updating:
reduction_indices is deprecated, use axis instead
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
WARNING:tensorflow:From <ipython-input-8-c909d6f1c681>:14: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.dropout instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.__call__` method instead.
WARNING:tensorflow:From <ipython-input-8-c909d6f1c681>:15: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.
Instructions for updating:
Use keras.layers.Dense instead.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/rnn/python/ops/lstm_ops.py:597: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `layer.add_weight` method instead.
WARNING:tensorflow:From <ipython-input-6-135f635cf6e4>:12: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:962: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/seq2seq/python/ops/attention_wrapper.py:2078: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From <ipython-input-9-8e660c2a5c11>:10: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.cast` instead.
[<tf.Variable 'Embedding/dense/kernel:0' shape=(1024, 300) dtype=float32_ref>,
 <tf.Variable 'Embedding/dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Embedding/glove:0' shape=(8692, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer_1/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(1200, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer_1/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(8692,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_W:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_b:0' shape=(300,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 0 into ../model/lstm_seq2seq/model.ckpt.
Reading ../data/train.tsv
INFO:tensorflow:loss = 9.545226, step = 0
INFO:tensorflow:lr = 0.0004
INFO:tensorflow:global_step/sec: 2.08373
INFO:tensorflow:loss = 4.537199, step = 100 (47.993 sec)
INFO:tensorflow:lr = 0.00039837044 (47.992 sec)
INFO:tensorflow:global_step/sec: 2.17141
INFO:tensorflow:loss = 3.5616326, step = 200 (46.059 sec)
INFO:tensorflow:lr = 0.00039674752 (46.057 sec)
INFO:tensorflow:global_step/sec: 2.11963
INFO:tensorflow:loss = 3.1313076, step = 300 (47.173 sec)
INFO:tensorflow:lr = 0.0003951312 (47.173 sec)
INFO:tensorflow:global_step/sec: 2.19324
INFO:tensorflow:loss = 3.0543175, step = 400 (45.596 sec)
INFO:tensorflow:lr = 0.00039352148 (45.595 sec)
INFO:tensorflow:global_step/sec: 2.19026
INFO:tensorflow:loss = 2.8855112, step = 500 (45.658 sec)
INFO:tensorflow:lr = 0.00039191832 (45.660 sec)
INFO:tensorflow:global_step/sec: 2.21216
INFO:tensorflow:loss = 2.8380878, step = 600 (45.205 sec)
INFO:tensorflow:lr = 0.00039032171 (45.205 sec)
INFO:tensorflow:global_step/sec: 2.23568
INFO:tensorflow:loss = 2.7726312, step = 700 (44.727 sec)
INFO:tensorflow:lr = 0.00038873157 (44.726 sec)
INFO:tensorflow:global_step/sec: 2.21924
INFO:tensorflow:loss = 2.631123, step = 800 (45.061 sec)
INFO:tensorflow:lr = 0.0003871479 (45.061 sec)
INFO:tensorflow:global_step/sec: 2.21488
INFO:tensorflow:loss = 2.6776147, step = 900 (45.149 sec)
INFO:tensorflow:lr = 0.00038557075 (45.150 sec)
INFO:tensorflow:Saving checkpoints for 978 into ../model/lstm_seq2seq/model.ckpt.
INFO:tensorflow:Loss for final step: 2.602205.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_estimator/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/seq2seq/python/ops/beam_search_decoder.py:971: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.cast` instead.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-978
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/monitored_session.py:882: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:unsupported what times are the nutcracker show playing near me ]

                   in:unsupported                      
  _______________________|___________________________   
what times are the   nutcracker   show playing near  me

------------
Reading ../data/test.tsv
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-978
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing (Exact Match) Accuracy: 0.500
INFO:tensorflow:Best (Exact Match) Accuracy: 0.500
INFO:tensorflow:Calling model_fn.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
[<tf.Variable 'Embedding/dense/kernel:0' shape=(1024, 300) dtype=float32_ref>,
 <tf.Variable 'Embedding/dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Embedding/glove:0' shape=(8692, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer_1/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(1200, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer_1/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(8692,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_W:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_b:0' shape=(300,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-978
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/saver.py:1069: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file utilities to get mtimes.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 978 into ../model/lstm_seq2seq/model.ckpt.
Reading ../data/train.tsv
INFO:tensorflow:loss = 2.6306496, step = 978
INFO:tensorflow:lr = 0.000384345
INFO:tensorflow:global_step/sec: 2.08052
INFO:tensorflow:loss = 2.5160456, step = 1078 (48.067 sec)
INFO:tensorflow:lr = 0.0003827792 (48.068 sec)
INFO:tensorflow:global_step/sec: 2.21039
INFO:tensorflow:loss = 2.5345397, step = 1178 (45.240 sec)
INFO:tensorflow:lr = 0.00038121984 (45.242 sec)
INFO:tensorflow:global_step/sec: 2.21609
INFO:tensorflow:loss = 2.5588732, step = 1278 (45.125 sec)
INFO:tensorflow:lr = 0.00037966677 (45.123 sec)
INFO:tensorflow:global_step/sec: 2.22595
INFO:tensorflow:loss = 2.4706988, step = 1378 (44.927 sec)
INFO:tensorflow:lr = 0.00037812005 (44.927 sec)
INFO:tensorflow:global_step/sec: 2.20183
INFO:tensorflow:loss = 2.4629943, step = 1478 (45.416 sec)
INFO:tensorflow:lr = 0.00037657964 (45.417 sec)
INFO:tensorflow:global_step/sec: 2.22839
INFO:tensorflow:loss = 2.5238068, step = 1578 (44.878 sec)
INFO:tensorflow:lr = 0.0003750455 (44.877 sec)
INFO:tensorflow:global_step/sec: 2.18223
INFO:tensorflow:loss = 2.541678, step = 1678 (45.820 sec)
INFO:tensorflow:lr = 0.0003735176 (45.823 sec)
INFO:tensorflow:global_step/sec: 2.2391
INFO:tensorflow:loss = 2.485625, step = 1778 (44.663 sec)
INFO:tensorflow:lr = 0.00037199593 (44.661 sec)
INFO:tensorflow:global_step/sec: 2.20416
INFO:tensorflow:loss = 2.4658086, step = 1878 (45.367 sec)
INFO:tensorflow:lr = 0.00037048047 (45.367 sec)
INFO:tensorflow:Saving checkpoints for 1956 into ../model/lstm_seq2seq/model.ckpt.
INFO:tensorflow:Loss for final step: 2.474984.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-1956
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-1956
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing (Exact Match) Accuracy: 0.656
INFO:tensorflow:Best (Exact Match) Accuracy: 0.656
INFO:tensorflow:Calling model_fn.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
[<tf.Variable 'Embedding/dense/kernel:0' shape=(1024, 300) dtype=float32_ref>,
 <tf.Variable 'Embedding/dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Embedding/glove:0' shape=(8692, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer_1/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(1200, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer_1/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(8692,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_W:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_b:0' shape=(300,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-1956
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1956 into ../model/lstm_seq2seq/model.ckpt.
Reading ../data/train.tsv
INFO:tensorflow:loss = 2.4931223, step = 1956
INFO:tensorflow:lr = 0.0003693027
INFO:tensorflow:global_step/sec: 2.15286
INFO:tensorflow:loss = 2.465508, step = 2056 (46.455 sec)
INFO:tensorflow:lr = 0.0003677982 (46.451 sec)
INFO:tensorflow:global_step/sec: 2.16443
INFO:tensorflow:loss = 2.4741223, step = 2156 (46.200 sec)
INFO:tensorflow:lr = 0.00036629982 (46.200 sec)
INFO:tensorflow:global_step/sec: 2.26136
INFO:tensorflow:loss = 2.4377806, step = 2256 (44.223 sec)
INFO:tensorflow:lr = 0.00036480758 (44.222 sec)
INFO:tensorflow:global_step/sec: 2.20505
INFO:tensorflow:loss = 2.4605105, step = 2356 (45.351 sec)
INFO:tensorflow:lr = 0.00036332142 (45.354 sec)
INFO:tensorflow:global_step/sec: 2.19774
INFO:tensorflow:loss = 2.4478672, step = 2456 (45.498 sec)
INFO:tensorflow:lr = 0.00036184126 (45.494 sec)
INFO:tensorflow:global_step/sec: 2.19507
INFO:tensorflow:loss = 2.486076, step = 2556 (45.560 sec)
INFO:tensorflow:lr = 0.00036036715 (45.560 sec)
INFO:tensorflow:global_step/sec: 2.1689
INFO:tensorflow:loss = 2.4467924, step = 2656 (46.106 sec)
INFO:tensorflow:lr = 0.0003588991 (46.106 sec)
INFO:tensorflow:global_step/sec: 2.22423
INFO:tensorflow:loss = 2.434639, step = 2756 (44.957 sec)
INFO:tensorflow:lr = 0.00035743695 (44.956 sec)
INFO:tensorflow:global_step/sec: 2.21487
INFO:tensorflow:loss = 2.4393866, step = 2856 (45.153 sec)
INFO:tensorflow:lr = 0.00035598082 (45.155 sec)
INFO:tensorflow:Saving checkpoints for 2934 into ../model/lstm_seq2seq/model.ckpt.
INFO:tensorflow:Loss for final step: 2.4379032.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-2934
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show ] [ sl:category_event playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                          
  ________________________|_______________________________________________________                          
 |     |    |             |                     |                            sl:location                   
 |     |    |             |                     |                                 |                         
 |     |    |             |                     |                          in:get_location                 
 |     |    |             |                     |                 ________________|_______________          
 |     |    |      sl:category_even      sl:category_even sl:search_radius                 sl:location_user
 |     |    |             t                     t                |                                |        
 |     |    |    _________|__________           |                |                                |         
what times are the    nutcracker    show     playing            near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-2934
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing (Exact Match) Accuracy: 0.671
INFO:tensorflow:Best (Exact Match) Accuracy: 0.671
INFO:tensorflow:Calling model_fn.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
[<tf.Variable 'Embedding/dense/kernel:0' shape=(1024, 300) dtype=float32_ref>,
 <tf.Variable 'Embedding/dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Embedding/glove:0' shape=(8692, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer_1/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(1200, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer_1/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(8692,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_W:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_b:0' shape=(300,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-2934
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 2934 into ../model/lstm_seq2seq/model.ckpt.
Reading ../data/train.tsv
INFO:tensorflow:loss = 2.4148135, step = 2934
INFO:tensorflow:lr = 0.00035484915
INFO:tensorflow:global_step/sec: 2.13728
INFO:tensorflow:loss = 2.4266083, step = 3034 (46.794 sec)
INFO:tensorflow:lr = 0.0003534035 (46.791 sec)
INFO:tensorflow:global_step/sec: 2.13758
INFO:tensorflow:loss = 2.3987186, step = 3134 (46.780 sec)
INFO:tensorflow:lr = 0.0003519638 (46.780 sec)
INFO:tensorflow:global_step/sec: 2.23556
INFO:tensorflow:loss = 2.421347, step = 3234 (44.734 sec)
INFO:tensorflow:lr = 0.00035052994 (44.733 sec)
INFO:tensorflow:global_step/sec: 2.14919
INFO:tensorflow:loss = 2.4059613, step = 3334 (46.528 sec)
INFO:tensorflow:lr = 0.00034910193 (46.528 sec)
INFO:tensorflow:global_step/sec: 2.18005
INFO:tensorflow:loss = 2.410464, step = 3434 (45.870 sec)
INFO:tensorflow:lr = 0.0003476797 (45.871 sec)
INFO:tensorflow:global_step/sec: 2.23121
INFO:tensorflow:loss = 2.391828, step = 3534 (44.818 sec)
INFO:tensorflow:lr = 0.00034626332 (44.817 sec)
INFO:tensorflow:global_step/sec: 2.19801
INFO:tensorflow:loss = 2.4041214, step = 3634 (45.497 sec)
INFO:tensorflow:lr = 0.00034485265 (45.498 sec)
INFO:tensorflow:global_step/sec: 2.17318
INFO:tensorflow:loss = 2.400657, step = 3734 (46.017 sec)
INFO:tensorflow:lr = 0.00034344778 (46.016 sec)
INFO:tensorflow:global_step/sec: 2.2523
INFO:tensorflow:loss = 2.421836, step = 3834 (44.397 sec)
INFO:tensorflow:lr = 0.00034204862 (44.400 sec)
INFO:tensorflow:Saving checkpoints for 3912 into ../model/lstm_seq2seq/model.ckpt.
INFO:tensorflow:Loss for final step: 2.4021182.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-3912
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-3912
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing (Exact Match) Accuracy: 0.699
INFO:tensorflow:Best (Exact Match) Accuracy: 0.699
INFO:tensorflow:Calling model_fn.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
[<tf.Variable 'Embedding/dense/kernel:0' shape=(1024, 300) dtype=float32_ref>,
 <tf.Variable 'Embedding/dense/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Embedding/glove:0' shape=(8692, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/kernel:0' shape=(900, 1200) dtype=float32_ref>,
 <tf.Variable 'Encoder/lstm_fused_cell_1/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Encoder/state_fc/bias:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/memory_layer_1/kernel:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/kernel:0' shape=(1200, 1200) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/lstm_cell/bias:0' shape=(1200,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/query_layer/kernel:0' shape=(300, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/bahdanau_attention/attention_v:0' shape=(300,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/attention_wrapper/attention_layer_1/kernel:0' shape=(900, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/bias:0' shape=(8692,) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_W:0' shape=(600, 300) dtype=float32_ref>,
 <tf.Variable 'Decoder/decoder/tied_dense/proj_b:0' shape=(300,) dtype=float32_ref>]
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-3912
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 3912 into ../model/lstm_seq2seq/model.ckpt.
Reading ../data/train.tsv
INFO:tensorflow:loss = 2.4316266, step = 3912
INFO:tensorflow:lr = 0.0003409612
INFO:tensorflow:global_step/sec: 2.1214
INFO:tensorflow:loss = 2.3947659, step = 4012 (47.144 sec)
INFO:tensorflow:lr = 0.0003395722 (47.144 sec)
INFO:tensorflow:global_step/sec: 2.24698
INFO:tensorflow:loss = 2.402081, step = 4112 (44.502 sec)
INFO:tensorflow:lr = 0.0003381888 (44.503 sec)
INFO:tensorflow:global_step/sec: 2.18832
INFO:tensorflow:loss = 2.3932135, step = 4212 (45.700 sec)
INFO:tensorflow:lr = 0.00033681106 (45.700 sec)
INFO:tensorflow:global_step/sec: 2.21088
INFO:tensorflow:loss = 2.4088333, step = 4312 (45.229 sec)
INFO:tensorflow:lr = 0.00033543896 (45.228 sec)
INFO:tensorflow:global_step/sec: 2.14227
INFO:tensorflow:loss = 2.4013007, step = 4412 (46.682 sec)
INFO:tensorflow:lr = 0.00033407242 (46.683 sec)
INFO:tensorflow:global_step/sec: 2.17381
INFO:tensorflow:loss = 2.417449, step = 4512 (46.001 sec)
INFO:tensorflow:lr = 0.00033271144 (46.001 sec)
INFO:tensorflow:global_step/sec: 2.19726
INFO:tensorflow:loss = 2.4270375, step = 4612 (45.507 sec)
INFO:tensorflow:lr = 0.00033135602 (45.507 sec)
INFO:tensorflow:global_step/sec: 2.16973
INFO:tensorflow:loss = 2.3798676, step = 4712 (46.092 sec)
INFO:tensorflow:lr = 0.0003300061 (46.093 sec)
INFO:tensorflow:global_step/sec: 2.24348
INFO:tensorflow:loss = 2.4008048, step = 4812 (44.570 sec)
INFO:tensorflow:lr = 0.0003286617 (44.567 sec)
INFO:tensorflow:Saving checkpoints for 4890 into ../model/lstm_seq2seq/model.ckpt.
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
INFO:tensorflow:Loss for final step: 2.3781312.
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-4890
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Saver not created because there are no variables in the graph to restore
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from ../model/lstm_seq2seq/model.ckpt-4890
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Reading ../data/test.tsv
INFO:tensorflow:Evaluation: Testing (Exact Match) Accuracy: 0.700
INFO:tensorflow:Best (Exact Match) Accuracy: 0.700
INFO:tensorflow:Calling model_fn.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:38: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.