"""
We use following lines because we are running on Google Colab
If you are running notebook on a local computer, you don't need this cell
"""
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/tensorflow2/semantic_parsing/tree_slu/main')

Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).

%tensorflow_version 2.x
!pip install tensorflow-addons

TensorFlow 2.x selected.
Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (0.6.0)
Requirement already satisfied: tensorflow-gpu==2.0.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-addons) (2.0.0)
Requirement already satisfied: six>=1.10.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-addons) (1.13.0)
Requirement already satisfied: gast==0.2.2 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (0.2.2)
Requirement already satisfied: grpcio>=1.8.6 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.25.0)
Requirement already satisfied: tensorboard<2.1.0,>=2.0.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (2.0.1)
Requirement already satisfied: protobuf>=3.6.1 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (3.10.0)
Requirement already satisfied: keras-preprocessing>=1.0.5 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.1.0)
Requirement already satisfied: numpy<2.0,>=1.16.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.17.4)
Requirement already satisfied: wrapt>=1.11.1 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.11.2)
Requirement already satisfied: wheel>=0.26 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (0.33.6)
Requirement already satisfied: tensorflow-estimator<2.1.0,>=2.0.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (2.0.1)
Requirement already satisfied: google-pasta>=0.1.6 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (0.1.8)
Requirement already satisfied: absl-py>=0.7.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (0.8.1)
Requirement already satisfied: astor>=0.6.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (0.8.0)
Requirement already satisfied: opt-einsum>=2.3.2 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (3.1.0)
Requirement already satisfied: keras-applications>=1.0.8 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.0.8)
Requirement already satisfied: termcolor>=1.1.0 in /tensorflow-2.0.0/python3.6 (from tensorflow-gpu==2.0.0->tensorflow-addons) (1.1.0)
Requirement already satisfied: markdown>=2.6.8 in /tensorflow-2.0.0/python3.6 (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (3.1.1)
Requirement already satisfied: werkzeug>=0.11.15 in /tensorflow-2.0.0/python3.6 (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (0.16.0)
Requirement already satisfied: setuptools>=41.0.0 in /tensorflow-2.0.0/python3.6 (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (41.6.0)
Requirement already satisfied: google-auth<2,>=1.6.3 in /tensorflow-2.0.0/python3.6 (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (1.7.0)
Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /tensorflow-2.0.0/python3.6 (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (0.4.1)
Requirement already satisfied: h5py in /tensorflow-2.0.0/python3.6 (from keras-applications>=1.0.8->tensorflow-gpu==2.0.0->tensorflow-addons) (2.10.0)
Requirement already satisfied: cachetools<3.2,>=2.0.0 in /tensorflow-2.0.0/python3.6 (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (3.1.1)
Requirement already satisfied: pyasn1-modules>=0.2.1 in /tensorflow-2.0.0/python3.6 (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (0.2.7)
Requirement already satisfied: rsa<4.1,>=3.1.4 in /tensorflow-2.0.0/python3.6 (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (4.0)
Requirement already satisfied: requests-oauthlib>=0.7.0 in /tensorflow-2.0.0/python3.6 (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (1.3.0)
Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /tensorflow-2.0.0/python3.6 (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (0.4.7)
Requirement already satisfied: oauthlib>=3.0.0 in /tensorflow-2.0.0/python3.6 (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (3.1.0)
Requirement already satisfied: requests>=2.0.0 in /tensorflow-2.0.0/python3.6 (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (2.22.0)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /tensorflow-2.0.0/python3.6 (from requests>=2.0.0->requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (3.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /tensorflow-2.0.0/python3.6 (from requests>=2.0.0->requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (2019.9.11)
Requirement already satisfied: idna<2.9,>=2.5 in /tensorflow-2.0.0/python3.6 (from requests>=2.0.0->requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (2.8)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /tensorflow-2.0.0/python3.6 (from requests>=2.0.0->requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0.0->tensorflow-addons) (1.25.7)

import tensorflow as tf
import tensorflow_addons as tfa

import numpy as np
import pprint
import logging
import time
import nltk

print("TensorFlow Version", tf.__version__)
print('GPU Enabled:', tf.test.is_gpu_available())

TensorFlow Version 2.0.0
GPU Enabled: True

# stream data from text files
def data_generator(f_path, params):
  with open(f_path) as f:
    print('Reading', f_path)
    for line in f:
      text_raw, text_tokenized, label = line.split('\t')
      text_tokenized = text_tokenized.lower().split()
      label = label.replace('[', '[ ').lower().split()
      source = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in text_tokenized]
      target = [params['tgt2idx'].get(w, len(params['tgt2idx'])) for w in label]
      target_in = [1] + target
      target_out = target + [2]
      yield (source, target_in, target_out)

def dataset(is_training, params):
  _shapes = ([None], [None], [None])
  _types = (tf.int32, tf.int32, tf.int32)
  _pads = (0, 0, 0)
  
  if is_training:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['train_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.shuffle(params['buffer_size'])
    ds = ds.padded_batch(params['train_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  else:
    ds = tf.data.Dataset.from_generator(
      lambda: data_generator(params['test_path'], params),
      output_shapes = _shapes,
      output_types = _types,)
    ds = ds.padded_batch(params['eval_batch_size'], _shapes, _pads)
    ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
  
  return ds

class Embed(tf.keras.Model):
  def __init__(self):
    super().__init__()
    self.embedding = tf.Variable(np.load('../vocab/word.npy'),
                                 dtype=tf.float32,
                                 name='pretrained_embedding')
  
  def call(self, inputs):
    if inputs.dtype != tf.int32:
      inputs = tf.cast(inputs, tf.int32)
    x = tf.nn.embedding_lookup(self.embedding, inputs)
    return x

class Encoder(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])
    self.encoder = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(
      params['rnn_units'], return_state=True, return_sequences=True))
    self.state_fc = tf.keras.layers.Dense(params['rnn_units'], params['activation'], name='state_fc')
  
  def call(self, inputs, training):
    x = self.dropout(inputs, training=training)
    
    encoder_o, state_fw_h, state_fw_c, state_bw_h, state_bw_c = self.encoder(x)
    encoder_s = [
      self.state_fc(tf.concat((state_fw_h, state_bw_h), -1)),
      self.state_fc(tf.concat((state_fw_c, state_bw_c), -1)),]
    
    return encoder_o, encoder_s

class TiedDense(tf.keras.layers.Layer):
  def __init__(self, tied_embed, out_dim):
    super().__init__()
    self.tied_embed = tied_embed
    self.out_dim = out_dim
  
  def build(self, input_shape):
    self.bias = self.add_weight(name='bias',
                                shape=[self.out_dim],
                                trainable=True)
    """
    self.proj_W = self.add_weight(name='proj_W',
                                  shape=[2*params['rnn_units'], params['embed_dim']],
                                  trainable=True)
    self.proj_b = self.add_weight(name='proj_b',
                                  shape=[params['embed_dim']],
                                  trainable=True)
    """
    super().build(input_shape)
  
  @tf.function
  def call(self, inputs):
    """
    inputs = params['activation'](inputs @ self.proj_W + self.proj_b)
    """
    x = tf.matmul(inputs, self.tied_embed, transpose_b=True)
    x += self.bias
    return x
  
  def compute_output_shape(self, input_shape):
    return input_shape[:-1].concatenate(self.out_dim)

class Model(tf.keras.Model):
  def __init__(self, params):
    super().__init__()
    self.embed = Embed()

    self.encoder = Encoder(params)

    self.dropout = tf.keras.layers.Dropout(params['dropout_rate'])

    self.attn = tfa.seq2seq.BahdanauAttention(params['rnn_units'])

    self.decoder_cell = tfa.seq2seq.AttentionWrapper(
      tf.keras.layers.LSTMCell(params['rnn_units']),
      self.attn,
      attention_layer_size=params['rnn_units'])
    
    self.proj_layer = TiedDense(self.embed.embedding, len(params['tgt2idx'])+1)

    self.teach_forcing = tfa.seq2seq.BasicDecoder(
      self.decoder_cell,
      tfa.seq2seq.sampler.TrainingSampler(),
      output_layer = self.proj_layer)

    self.beam_search = tfa.seq2seq.BeamSearchDecoder(
      self.decoder_cell,
      beam_width = params['beam_width'],
      embedding_fn = lambda x: self.embed(x),
      output_layer = self.proj_layer,
      maximum_iterations = 80,)

  
  def call(self, inputs, training=True):
    if training:
      source, target_in = inputs
    else:
      source = inputs
    batch_sz = tf.shape(source)[0]

    encoder_o, encoder_s = self.encoder(self.embed(source), training=training)

    if training:
      self.attn([encoder_o, tf.math.count_nonzero(source, 1)], setup_memory=True)
      attn_state = self.decoder_cell.get_initial_state(batch_size=batch_sz, dtype=tf.float32)
      attn_state = attn_state.clone(cell_state=encoder_s)

      decoder_o, _, _ = self.teach_forcing(
        inputs = self.dropout(self.embed(target_in), training=training),
        initial_state = attn_state,
        sequence_length = tf.math.count_nonzero(target_in, 1, dtype=tf.int32))

      logits_or_ids = decoder_o.rnn_output
    else:
      encoder_o_t = tfa.seq2seq.tile_batch(encoder_o, params['beam_width'])
      encoder_len_t = tfa.seq2seq.tile_batch(tf.math.count_nonzero(source, 1), params['beam_width'])
      encoder_s_t = tfa.seq2seq.tile_batch(encoder_s, params['beam_width'])

      self.attn([encoder_o_t, encoder_len_t], setup_memory=True)
      attn_state = self.decoder_cell.get_initial_state(batch_size=batch_sz*params['beam_width'], dtype=tf.float32)
      attn_state = attn_state.clone(cell_state=encoder_s_t)

      decoder_o, _, _ = self.beam_search(
        None,
        start_tokens = tf.tile(tf.constant([1], tf.int32), [batch_sz]),
        end_token = 2,
        initial_state = attn_state,)

      logits_or_ids = decoder_o.predicted_ids[:, :, 0]

    return logits_or_ids

def get_vocab(f_path):
  word2idx = {}
  with open(f_path) as f:
    for i, line in enumerate(f):
      line = line.rstrip()
      word2idx[line] = i
  return word2idx

def is_descending(history: list) -> bool:
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True

params = {
    'train_path': '../data/train.tsv',
    'test_path': '../data/test.tsv',
    'vocab_src_path': '../vocab/source.txt',
    'vocab_tgt_path': '../vocab/target.txt',
    'model_path': '../model/',
    'dropout_rate': 0.2,
    'rnn_units': 300,
    'embed_dim': 300,
    'activation': tf.nn.relu,
    'beam_width': 5,
    'lr': 4e-4,
    'clip_norm': .1,
    'buffer_size': 31279,
    'train_batch_size': 32,
    'eval_batch_size': 128,
    'num_patience': 5,
}

params['tgt2idx'] = get_vocab(params['vocab_tgt_path'])
params['idx2tgt'] = {idx: tgt for tgt, idx in params['tgt2idx'].items()}

model = Model(params)
model.build(input_shape=[[None, None], [None, None]])
pprint.pprint([(v.name, v.shape) for v in model.trainable_variables])

[('pretrained_embedding:0', TensorShape([8692, 300])),
 ('encoder/bidirectional/forward_lstm/kernel:0', TensorShape([300, 1200])),
 ('encoder/bidirectional/forward_lstm/recurrent_kernel:0',
  TensorShape([300, 1200])),
 ('encoder/bidirectional/forward_lstm/bias:0', TensorShape([1200])),
 ('encoder/bidirectional/backward_lstm/kernel:0', TensorShape([300, 1200])),
 ('encoder/bidirectional/backward_lstm/recurrent_kernel:0',
  TensorShape([300, 1200])),
 ('encoder/bidirectional/backward_lstm/bias:0', TensorShape([1200])),
 ('encoder/state_fc/kernel:0', TensorShape([600, 300])),
 ('encoder/state_fc/bias:0', TensorShape([300])),
 ('BahdanauAttention/attention_v:0', TensorShape([300])),
 ('attention_wrapper/BahdanauAttention/kernel:0', TensorShape([300, 300])),
 ('BahdanauAttention/kernel:0', TensorShape([600, 300])),
 ('attention_wrapper/attention_layer/kernel:0', TensorShape([900, 300])),
 ('attention_wrapper/lstm_cell_3/kernel:0', TensorShape([600, 1200])),
 ('attention_wrapper/lstm_cell_3/recurrent_kernel:0', TensorShape([300, 1200])),
 ('attention_wrapper/lstm_cell_3/bias:0', TensorShape([1200])),
 ('tied_dense/bias:0', TensorShape([8692]))]

decay_lr = tf.optimizers.schedules.ExponentialDecay(params['lr'], 1000, 0.99)
optim = tf.optimizers.Adam(params['lr'])
global_step = 0

history_acc = []
best_acc = .0

t0 = time.time()
logger = logging.getLogger('tensorflow')
logger.propagate = False
logger.setLevel(logging.INFO)

def minimal_test(model, params):
  test_str = ['what', 'times', 'are', 'the', 'nutcracker', 'show', 'playing', 'near', 'me']
  test_arr = tf.convert_to_tensor([[params['tgt2idx'][w] for w in test_str]])
  generated = model(inputs=test_arr, training=False)

  print('-'*12)
  print('minimal test')
  print('utterance:', ' '.join(test_str))
  parsed = ' '.join([params['idx2tgt'][idx] for idx in generated[0].numpy()])
  parsed = parsed.replace('<end>', '').strip()
  print('parsed:', parsed)
  print()
  try:
    nltk.tree.Tree.fromstring(parsed.replace('[ ', '(').replace(' ]', ')')).pretty_print()
  except:
    pass
  print('-'*12)

def is_descending(history: list) -> bool:
  history = history[-(params['num_patience']+1):]
  for i in range(1, len(history)):
    if history[i-1] <= history[i]:
      return False
  return True

while True:
  # TRAINING
  is_training = True
  for i, (source, target_in, target_out) in enumerate(dataset(is_training=is_training, params=params)):
    with tf.GradientTape() as tape:
      logits_or_ids = model((source, target_in), training=is_training)
      
      loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
        labels=target_out, logits=logits_or_ids)
      weights = tf.cast(tf.sign(target_in), tf.float32)
      loss = tf.reduce_sum(loss * weights) / tf.reduce_sum(weights)

    variables = model.trainable_variables
    optim.lr.assign(decay_lr(global_step))
    grads = tape.gradient(loss, variables)
    grads, _ = tf.clip_by_global_norm(grads, params['clip_norm'])
    optim.apply_gradients(zip(grads, variables))
    
    if global_step % 50 == 0:
      logger.info("Step {} | Loss: {:.4f} | Spent: {:.1f} secs | LR: {:.6f}".format(
        global_step, loss.numpy().item(), time.time()-t0, optim.lr.numpy().item()))
      t0 = time.time()
    
    global_step += 1

  # EVALUATION
  is_training = False
  minimal_test(model, params)
  m = tf.keras.metrics.Mean()
  
  for i, (source, target_in, target_out) in enumerate(dataset(is_training=is_training, params=params)):
    generated = model(inputs=source, training=is_training)
    seq_lens = tf.argmax(tf.cast(tf.equal(target_out, 2), tf.int32), axis=1)
    for pred, tgt, seq_len in zip(generated.numpy(), target_out.numpy(), seq_lens.numpy()):
      matched = np.all(pred[:seq_len+1] == tgt[:seq_len+1])
      m.update_state(int(matched))
  
  acc = m.result().numpy()
  logger.info("Evaluation: Testing Exact Match Accuracy: {:.3f}".format(acc))
  history_acc.append(acc)

  if acc > best_acc:
    best_acc = acc
  logger.info("Best Accuracy: {:.3f}".format(best_acc))

  if len(history_acc) > params['num_patience'] and is_descending(history_acc):
    logger.info("Testing Accuracy not improved over {} epochs, Early Stop".format(params['num_patience']))
    break

Reading ../data/train.tsv
INFO:tensorflow:Step 0 | Loss: 9.2826 | Spent: 6.0 secs | LR: 0.000400
INFO:tensorflow:Step 50 | Loss: 3.4936 | Spent: 31.0 secs | LR: 0.000400
INFO:tensorflow:Step 100 | Loss: 2.5489 | Spent: 31.4 secs | LR: 0.000400
INFO:tensorflow:Step 150 | Loss: 1.8968 | Spent: 30.4 secs | LR: 0.000399
INFO:tensorflow:Step 200 | Loss: 1.5225 | Spent: 31.3 secs | LR: 0.000399
INFO:tensorflow:Step 250 | Loss: 1.5356 | Spent: 31.6 secs | LR: 0.000399
INFO:tensorflow:Step 300 | Loss: 1.1140 | Spent: 34.1 secs | LR: 0.000399
INFO:tensorflow:Step 350 | Loss: 0.9405 | Spent: 32.0 secs | LR: 0.000399
INFO:tensorflow:Step 400 | Loss: 0.6781 | Spent: 31.5 secs | LR: 0.000398
INFO:tensorflow:Step 450 | Loss: 0.6539 | Spent: 31.1 secs | LR: 0.000398
INFO:tensorflow:Step 500 | Loss: 0.7228 | Spent: 31.4 secs | LR: 0.000398
INFO:tensorflow:Step 550 | Loss: 0.5562 | Spent: 32.1 secs | LR: 0.000398
INFO:tensorflow:Step 600 | Loss: 0.5013 | Spent: 31.4 secs | LR: 0.000398
INFO:tensorflow:Step 650 | Loss: 0.3475 | Spent: 30.2 secs | LR: 0.000397
INFO:tensorflow:Step 700 | Loss: 0.4110 | Spent: 31.2 secs | LR: 0.000397
INFO:tensorflow:Step 750 | Loss: 0.4447 | Spent: 33.5 secs | LR: 0.000397
INFO:tensorflow:Step 800 | Loss: 0.6801 | Spent: 32.9 secs | LR: 0.000397
INFO:tensorflow:Step 850 | Loss: 1.3601 | Spent: 30.7 secs | LR: 0.000397
INFO:tensorflow:Step 900 | Loss: 0.3804 | Spent: 30.7 secs | LR: 0.000396
INFO:tensorflow:Step 950 | Loss: 0.1635 | Spent: 30.6 secs | LR: 0.000396
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are the nutcracker show playing near me ]

                   in:get_event                      
  ______________________|__________________________   
what times are the  nutcracker  show playing near  me

------------
Reading ../data/test.tsv

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:35: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Evaluation: Testing Exact Match Accuracy: 0.383
INFO:tensorflow:Best Accuracy: 0.383
Reading ../data/train.tsv
INFO:tensorflow:Step 1000 | Loss: 0.3383 | Spent: 167.3 secs | LR: 0.000396
INFO:tensorflow:Step 1050 | Loss: 0.3575 | Spent: 31.2 secs | LR: 0.000396
INFO:tensorflow:Step 1100 | Loss: 0.3602 | Spent: 33.0 secs | LR: 0.000396
INFO:tensorflow:Step 1150 | Loss: 0.2549 | Spent: 32.3 secs | LR: 0.000395
INFO:tensorflow:Step 1200 | Loss: 0.2287 | Spent: 31.8 secs | LR: 0.000395
INFO:tensorflow:Step 1250 | Loss: 0.3000 | Spent: 32.4 secs | LR: 0.000395
INFO:tensorflow:Step 1300 | Loss: 0.2924 | Spent: 31.4 secs | LR: 0.000395
INFO:tensorflow:Step 1350 | Loss: 0.1161 | Spent: 31.1 secs | LR: 0.000395
INFO:tensorflow:Step 1400 | Loss: 0.1842 | Spent: 32.1 secs | LR: 0.000394
INFO:tensorflow:Step 1450 | Loss: 0.2146 | Spent: 31.8 secs | LR: 0.000394
INFO:tensorflow:Step 1500 | Loss: 0.2259 | Spent: 30.4 secs | LR: 0.000394
INFO:tensorflow:Step 1550 | Loss: 0.1540 | Spent: 33.3 secs | LR: 0.000394
INFO:tensorflow:Step 1600 | Loss: 0.1853 | Spent: 31.5 secs | LR: 0.000394
INFO:tensorflow:Step 1650 | Loss: 0.1268 | Spent: 31.9 secs | LR: 0.000393
INFO:tensorflow:Step 1700 | Loss: 0.1582 | Spent: 31.1 secs | LR: 0.000393
INFO:tensorflow:Step 1750 | Loss: 0.1391 | Spent: 30.1 secs | LR: 0.000393
INFO:tensorflow:Step 1800 | Loss: 0.0759 | Spent: 31.9 secs | LR: 0.000393
INFO:tensorflow:Step 1850 | Loss: 0.0602 | Spent: 31.0 secs | LR: 0.000393
INFO:tensorflow:Step 1900 | Loss: 0.2162 | Spent: 32.2 secs | LR: 0.000392
INFO:tensorflow:Step 1950 | Loss: 0.0843 | Spent: 31.0 secs | LR: 0.000392
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:35: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Evaluation: Testing Exact Match Accuracy: 0.597
INFO:tensorflow:Best Accuracy: 0.597
Reading ../data/train.tsv
INFO:tensorflow:Step 2000 | Loss: 0.2686 | Spent: 186.5 secs | LR: 0.000392
INFO:tensorflow:Step 2050 | Loss: 0.1794 | Spent: 31.5 secs | LR: 0.000392
INFO:tensorflow:Step 2100 | Loss: 0.0995 | Spent: 30.3 secs | LR: 0.000392
INFO:tensorflow:Step 2150 | Loss: 0.2179 | Spent: 31.6 secs | LR: 0.000391
INFO:tensorflow:Step 2200 | Loss: 0.1103 | Spent: 30.6 secs | LR: 0.000391
INFO:tensorflow:Step 2250 | Loss: 0.0681 | Spent: 31.2 secs | LR: 0.000391
INFO:tensorflow:Step 2300 | Loss: 0.1054 | Spent: 30.5 secs | LR: 0.000391
INFO:tensorflow:Step 2350 | Loss: 0.1282 | Spent: 30.8 secs | LR: 0.000391
INFO:tensorflow:Step 2400 | Loss: 0.0614 | Spent: 32.4 secs | LR: 0.000390
INFO:tensorflow:Step 2450 | Loss: 0.0933 | Spent: 32.1 secs | LR: 0.000390
INFO:tensorflow:Step 2500 | Loss: 0.0860 | Spent: 32.1 secs | LR: 0.000390
INFO:tensorflow:Step 2550 | Loss: 0.1360 | Spent: 31.7 secs | LR: 0.000390
INFO:tensorflow:Step 2600 | Loss: 0.0816 | Spent: 31.0 secs | LR: 0.000390
INFO:tensorflow:Step 2650 | Loss: 0.1040 | Spent: 31.2 secs | LR: 0.000389
INFO:tensorflow:Step 2700 | Loss: 0.1258 | Spent: 30.6 secs | LR: 0.000389
INFO:tensorflow:Step 2750 | Loss: 0.0744 | Spent: 30.1 secs | LR: 0.000389
INFO:tensorflow:Step 2800 | Loss: 0.1592 | Spent: 32.1 secs | LR: 0.000389
INFO:tensorflow:Step 2850 | Loss: 0.1176 | Spent: 32.1 secs | LR: 0.000389
INFO:tensorflow:Step 2900 | Loss: 0.1056 | Spent: 31.9 secs | LR: 0.000389
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:35: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Evaluation: Testing Exact Match Accuracy: 0.651
INFO:tensorflow:Best Accuracy: 0.651
Reading ../data/train.tsv
INFO:tensorflow:Step 2950 | Loss: 0.0584 | Spent: 185.4 secs | LR: 0.000388
INFO:tensorflow:Step 3000 | Loss: 0.0577 | Spent: 32.3 secs | LR: 0.000388
INFO:tensorflow:Step 3050 | Loss: 0.0769 | Spent: 32.0 secs | LR: 0.000388
INFO:tensorflow:Step 3100 | Loss: 0.0682 | Spent: 32.6 secs | LR: 0.000388
INFO:tensorflow:Step 3150 | Loss: 0.0645 | Spent: 33.1 secs | LR: 0.000388
INFO:tensorflow:Step 3200 | Loss: 0.0889 | Spent: 30.9 secs | LR: 0.000387
INFO:tensorflow:Step 3250 | Loss: 0.0837 | Spent: 31.7 secs | LR: 0.000387
INFO:tensorflow:Step 3300 | Loss: 0.0779 | Spent: 30.9 secs | LR: 0.000387
INFO:tensorflow:Step 3350 | Loss: 0.0581 | Spent: 30.4 secs | LR: 0.000387
INFO:tensorflow:Step 3400 | Loss: 0.0877 | Spent: 31.9 secs | LR: 0.000387
INFO:tensorflow:Step 3450 | Loss: 0.0830 | Spent: 32.4 secs | LR: 0.000386
INFO:tensorflow:Step 3500 | Loss: 0.0820 | Spent: 30.9 secs | LR: 0.000386
INFO:tensorflow:Step 3550 | Loss: 0.0557 | Spent: 31.5 secs | LR: 0.000386
INFO:tensorflow:Step 3600 | Loss: 0.0908 | Spent: 32.1 secs | LR: 0.000386
INFO:tensorflow:Step 3650 | Loss: 0.0867 | Spent: 31.6 secs | LR: 0.000386
INFO:tensorflow:Step 3700 | Loss: 0.0269 | Spent: 32.9 secs | LR: 0.000385
INFO:tensorflow:Step 3750 | Loss: 0.0349 | Spent: 31.9 secs | LR: 0.000385
INFO:tensorflow:Step 3800 | Loss: 0.0666 | Spent: 31.4 secs | LR: 0.000385
INFO:tensorflow:Step 3850 | Loss: 0.0340 | Spent: 30.7 secs | LR: 0.000385
INFO:tensorflow:Step 3900 | Loss: 0.0743 | Spent: 31.5 secs | LR: 0.000385
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show playing ] [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                     in:get_event                                                                 
  ________________________|______________________________________________                          
 |     |    |             |                                         sl:location                   
 |     |    |             |                                              |                         
 |     |    |             |                                       in:get_location                 
 |     |    |             |                              ________________|_______________          
 |     |    |      sl:category_even              sl:search_radius                 sl:location_user
 |     |    |             t                             |                                |        
 |     |    |    _________|_________________            |                                |         
what times are the    nutcracker    show playing       near                              me       

------------
Reading ../data/test.tsv

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:35: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.

INFO:tensorflow:Evaluation: Testing Exact Match Accuracy: 0.669
INFO:tensorflow:Best Accuracy: 0.669
Reading ../data/train.tsv
INFO:tensorflow:Step 3950 | Loss: 0.0279 | Spent: 193.0 secs | LR: 0.000384
INFO:tensorflow:Step 4000 | Loss: 0.0257 | Spent: 31.7 secs | LR: 0.000384
INFO:tensorflow:Step 4050 | Loss: 0.0623 | Spent: 29.8 secs | LR: 0.000384
INFO:tensorflow:Step 4100 | Loss: 0.0587 | Spent: 31.0 secs | LR: 0.000384
INFO:tensorflow:Step 4150 | Loss: 0.0527 | Spent: 31.1 secs | LR: 0.000384
INFO:tensorflow:Step 4200 | Loss: 0.0429 | Spent: 31.4 secs | LR: 0.000383
INFO:tensorflow:Step 4250 | Loss: 0.0460 | Spent: 32.6 secs | LR: 0.000383
INFO:tensorflow:Step 4300 | Loss: 0.0385 | Spent: 30.1 secs | LR: 0.000383
INFO:tensorflow:Step 4350 | Loss: 0.0462 | Spent: 31.6 secs | LR: 0.000383
INFO:tensorflow:Step 4400 | Loss: 0.0607 | Spent: 31.6 secs | LR: 0.000383
INFO:tensorflow:Step 4450 | Loss: 0.0660 | Spent: 32.9 secs | LR: 0.000383
INFO:tensorflow:Step 4500 | Loss: 0.0995 | Spent: 31.2 secs | LR: 0.000382
INFO:tensorflow:Step 4550 | Loss: 0.0563 | Spent: 30.7 secs | LR: 0.000382
INFO:tensorflow:Step 4600 | Loss: 0.0716 | Spent: 30.6 secs | LR: 0.000382
INFO:tensorflow:Step 4650 | Loss: 0.0909 | Spent: 30.8 secs | LR: 0.000382
INFO:tensorflow:Step 4700 | Loss: 0.0768 | Spent: 30.1 secs | LR: 0.000382
INFO:tensorflow:Step 4750 | Loss: 0.0677 | Spent: 29.6 secs | LR: 0.000381
INFO:tensorflow:Step 4800 | Loss: 0.1531 | Spent: 29.8 secs | LR: 0.000381
INFO:tensorflow:Step 4850 | Loss: 0.2851 | Spent: 31.2 secs | LR: 0.000381
------------
minimal test
utterance: what times are the nutcracker show playing near me
parsed: [ in:get_event what times are [ sl:category_event the nutcracker show ] playing [ sl:location [ in:get_location [ sl:search_radius near ] [ sl:location_user me ] ] ] ]

                       in:get_event                                                                        
  __________________________|_____________________________________________________                          
 |     |    |     |                        |                                 sl:location                   
 |     |    |     |                        |                                      |                         
 |     |    |     |                        |                               in:get_location                 
 |     |    |     |                        |                      ________________|_______________          
 |     |    |     |                 sl:category_even      sl:search_radius                 sl:location_user
 |     |    |     |                        t                     |                                |        
 |     |    |     |          ______________|__________           |                                |         
what times are playing     the         nutcracker    show       near                              me       

------------
Reading ../data/test.tsv

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:35: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.